feat: new background work process

This removes all previous dependencies on RabbitMQ and the need to run separate cron and requeueing processes.

app/models/concerns/has_locking.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# This concern provides functionality for locking items along with additional functionality to handle
+# the concept of retrying items after a certain period of time. The following database columns are
+# required on the model
+#
+# * locked_by - A string column to store the name of the process that has locked the item
+# * locked_at - A datetime column to store the time the item was locked
+# * retry_after - A datetime column to store the time after which the item should be retried
+# * attempts - An integer column to store the number of attempts that have been made to process the item
+#
+# 'ready' means that it's ready to be processed.
+module HasLocking
+
+  extend ActiveSupport::Concern
+
+  included do
+    scope :unlocked, -> { where(locked_at: nil) }
+    scope :ready, -> { where("retry_after IS NULL OR retry_after < ?", Time.now) }
+  end
+
+  def ready?
+    retry_after.nil? || retry_after < Time.now
+  end
+
+  def unlock
+    self.locked_by = nil
+    self.locked_at = nil
+    update_columns(locked_by: nil, locked_at: nil)
+  end
+
+  def locked?
+    locked_at.present?
+  end
+
+  def retry_later(time = nil)
+    retry_time = time || calculate_retry_time(attempts, 5.minutes)
+    self.locked_by = nil
+    self.locked_at = nil
+    update_columns(locked_by: nil, locked_at: nil, retry_after: Time.now + retry_time, attempts: attempts + 1)
+  end
+
+  def calculate_retry_time(attempts, initial_period)
+    (1.3**attempts) * initial_period
+  end
+
+end

app/models/concerns/has_soft_destroy.rb

@@ -14,7 +14,6 @@ module HasSoftDestroy
     run_callbacks :soft_destroy do
       self.deleted_at = Time.now
       save!
-      ActionDeletionJob.queue(:main, type: self.class.name, id: id)
     end
   end
 

app/models/queued_message.rb

@@ -29,33 +29,18 @@
 class QueuedMessage < ApplicationRecord
 
   include HasMessage
+  include HasLocking
 
   belongs_to :server
   belongs_to :ip_address, optional: true
   belongs_to :user, optional: true
 
   before_create :allocate_ip_address
-  after_commit :queue, on: :create
 
-  scope :unlocked, -> { where(locked_at: nil) }
-  scope :retriable, -> { where("retry_after IS NULL OR retry_after < ?", Time.now) }
-  scope :requeueable, -> { where("retry_after IS NULL OR retry_after < ?", 30.seconds.ago) }
+  scope :ready_with_delayed_retry, -> { where("retry_after IS NULL OR retry_after < ?", 30.seconds.ago) }
 
-  def retriable?
-    retry_after.nil? || retry_after < Time.now
-  end
-
-  def queue
-    UnqueueMessageJob.queue(queue_name, id: id)
-  end
-
-  def queue!
-    update_column(:retry_after, nil)
-    queue
-  end
-
-  def queue_name
-    ip_address ? :"outgoing-#{ip_address.id}" : :main
+  def retry_now
+    update(retry_after: nil)
   end
 
   def send_bounce
@@ -70,40 +55,6 @@ class QueuedMessage < ApplicationRecord
     self.ip_address = pool.ip_addresses.select_by_priority
   end
 
-  def acquire_lock
-    time = Time.now
-    locker = Postal.locker_name
-    rows = self.class.where(id: id, locked_by: nil, locked_at: nil).update_all(locked_by: locker, locked_at: time)
-    if rows == 1
-      self.locked_by = locker
-      self.locked_at = time
-      true
-    else
-      false
-    end
-  end
-
-  def retry_later(time = nil)
-    retry_time = time || self.class.calculate_retry_time(attempts, 5.minutes)
-    self.locked_by = nil
-    self.locked_at = nil
-    update_columns(locked_by: nil, locked_at: nil, retry_after: Time.now + retry_time, attempts: attempts + 1)
-  end
-
-  def unlock
-    self.locked_by = nil
-    self.locked_at = nil
-    update_columns(locked_by: nil, locked_at: nil)
-  end
-
-  def self.calculate_retry_time(attempts, initial_period)
-    (1.3**attempts) * initial_period
-  end
-
-  def locked?
-    locked_at.present?
-  end
-
   def batchable_messages(limit = 10)
     unless locked?
       raise Postal::Error, "Must lock current message before locking any friends"
@@ -114,13 +65,9 @@ class QueuedMessage < ApplicationRecord
     else
       time = Time.now
       locker = Postal.locker_name
-      self.class.retriable.where(batch_key: batch_key, ip_address_id: ip_address_id, locked_by: nil, locked_at: nil).limit(limit).update_all(locked_by: locker, locked_at: time)
+      self.class.ready.where(batch_key: batch_key, ip_address_id: ip_address_id, locked_by: nil, locked_at: nil).limit(limit).update_all(locked_by: locker, locked_at: time)
       QueuedMessage.where(batch_key: batch_key, ip_address_id: ip_address_id, locked_by: locker, locked_at: time).where.not(id: id)
     end
   end
 
-  def self.requeue_all
-    unlocked.requeueable.each(&:queue)
-  end
-
 end

app/models/scheduled_task.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: scheduled_tasks
+#
+#  id             :bigint           not null, primary key
+#  name           :string(255)
+#  next_run_after :datetime
+#
+# Indexes
+#
+#  index_scheduled_tasks_on_name  (name) UNIQUE
+#
+class ScheduledTask < ApplicationRecord
+end

app/models/server.rb

@@ -206,7 +206,7 @@ class Server < ApplicationRecord
   end
 
   def queue_size
-    @queue_size ||= queued_messages.retriable.count
+    @queue_size ||= queued_messages.ready.count
   end
 
   def stats
@@ -222,7 +222,7 @@ class Server < ApplicationRecord
 
   # Return the domain which can be used to authenticate emails sent from the given e-mail address.
   #
-  #  @param address [String] an e-mail address
+  # @param address [String] an e-mail address
   # @return [Domain, nil] the domain to use for authentication
   def authenticated_domain_for_address(address)
     return nil if address.blank?

app/models/webhook_request.rb

@@ -5,21 +5,28 @@
 # Table name: webhook_requests
 #
 #  id          :integer          not null, primary key
+#  attempts    :integer          default(0)
+#  error       :text(65535)
+#  event       :string(255)
+#  locked_at   :datetime
+#  locked_by   :string(255)
+#  payload     :text(65535)
+#  retry_after :datetime
+#  url         :string(255)
+#  uuid        :string(255)
+#  created_at  :datetime
 #  server_id   :integer
 #  webhook_id  :integer
-#  url         :string(255)
-#  event       :string(255)
-#  uuid        :string(255)
-#  payload     :text(65535)
-#  attempts    :integer          default(0)
-#  retry_after :datetime
-#  error       :text(65535)
-#  created_at  :datetime
+#
+# Indexes
+#
+#  index_webhook_requests_on_locked_by  (locked_by)
 #
 
 class WebhookRequest < ApplicationRecord
 
   include HasUUID
+  include HasLocking
 
   RETRIES = { 1 => 2.minutes, 2 => 3.minutes, 3 => 6.minutes, 4 => 10.minutes, 5 => 15.minutes }.freeze
 
@@ -31,30 +38,9 @@ class WebhookRequest < ApplicationRecord
 
   serialize :payload, Hash
 
-  after_commit :queue, on: :create
-
-  def self.trigger(server, event, payload = {})
-    unless server.is_a?(Server)
-      server = Server.find(server.to_i)
-    end
-
-    webhooks = server.webhooks.enabled.includes(:webhook_events).references(:webhook_events).where("webhooks.all_events = ? OR webhook_events.event = ?", true, event)
-    webhooks.each do |webhook|
-      server.webhook_requests.create!(event: event, payload: payload, webhook: webhook, url: webhook.url)
-    end
-  end
-
-  def self.requeue_all
-    where("retry_after < ?", Time.now).find_each(&:queue)
-  end
-
-  def queue
-    WebhookDeliveryJob.queue(:main, id: id)
-  end
-
   def deliver
     payload = { event: event, timestamp: created_at.to_f, payload: self.payload, uuid: uuid }.to_json
-    Postal.logger.tagged(event: event, url: url, component: "webhooks") do
+    Postal.logger.tagged(event: event, url: url) do
       Postal.logger.info "Sending webhook request"
       result = Postal::HTTP.post(url, sign: true, json: payload, timeout: 5)
       self.attempts += 1
@@ -74,7 +60,7 @@ class WebhookRequest < ApplicationRecord
 
       if result[:code] >= 200 && result[:code] < 300
         Postal.logger.info "Received #{result[:code]} status code. That's OK."
-        destroy
+        destroy!
         webhook&.update_column(:last_used_at, Time.now)
         true
       else
@@ -82,14 +68,31 @@ class WebhookRequest < ApplicationRecord
         self.error = "Couldn't send to URL. Code received was #{result[:code]}"
         if retry_after
           Postal.logger.info "Will retry #{retry_after} (this was attempt #{self.attempts})"
-          save
+          self.locked_by = nil
+          self.locked_at = nil
+          save!
         else
           Postal.logger.info "Have tried #{self.attempts} times. Giving up."
-          destroy
+          destroy!
         end
         false
       end
     end
   end
 
+  class << self
+
+    def trigger(server, event, payload = {})
+      unless server.is_a?(Server)
+        server = Server.find(server.to_i)
+      end
+
+      webhooks = server.webhooks.enabled.includes(:webhook_events).references(:webhook_events).where("webhooks.all_events = ? OR webhook_events.event = ?", true, event)
+      webhooks.each do |webhook|
+        server.webhook_requests.create!(event: event, payload: payload, webhook: webhook, url: webhook.url)
+      end
+    end
+
+  end
+
 end

app/models/worker_role.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: worker_roles
+#
+#  id          :bigint           not null, primary key
+#  acquired_at :datetime
+#  role        :string(255)
+#  worker      :string(255)
+#
+# Indexes
+#
+#  index_worker_roles_on_role  (role) UNIQUE
+#
+class WorkerRole < ApplicationRecord
+
+  class << self
+
+    # Acquire or renew a lock for the given role.
+    #
+    # @param role [String] The name of the role to acquire
+    # @return [Symbol, false] True if the lock was acquired or renewed, false otherwise
+    def acquire(role)
+      # update our existing lock if we already have one
+      updates = where(role: role, worker: Postal.locker_name).update_all(acquired_at: Time.current)
+      return :renewed if updates.positive?
+
+      # attempt to steal a role from another worker
+      updates = where(role: role).where("acquired_at is null OR acquired_at < ?", 5.minutes.ago)
+                                 .update_all(acquired_at: Time.current, worker: Postal.locker_name)
+      return :stolen if updates.positive?
+
+      # attempt to create a new role for this worker
+      begin
+        create!(role: role, worker: Postal.locker_name, acquired_at: Time.current)
+        :created
+      rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid
+        false
+      end
+    end
+
+    # Release a lock for the given role for the current process.
+    #
+    # @param role [String] The name of the role to release
+    # @return [Boolean] True if the lock was released, false otherwise
+    def release(role)
+      updates = where(role: role, worker: Postal.locker_name).delete_all
+      updates.positive?
+    end
+
+  end
+
+end