feat: new background work process

This removes all previous dependencies on RabbitMQ and the need to run separate cron and requeueing processes.

lib/postal/config.rb

@@ -85,6 +85,9 @@ module Postal
     end
   end
 
+  # Return a generic logger for use generally throughout Postal.
+  #
+  # @return [Klogger::Logger] A logger instance
   def self.logger
     @logger ||= begin
       k = Klogger.new(nil, destination: Rails.env.test? ? "/dev/null" : $stdout, highlight: Rails.env.development?)
@@ -106,9 +109,14 @@ module Postal
   def self.locker_name
     string = process_name.dup
     string += " job:#{Thread.current[:job_id]}" if Thread.current[:job_id]
+    string += " thread:#{Thread.current.native_thread_id}"
     string
   end
 
+  def self.locker_name_with_suffix(suffix)
+    "#{locker_name} #{suffix}"
+  end
+
   def self.smtp_from_name
     config.smtp&.from_name || "Postal"
   end
@@ -175,7 +183,7 @@ module Postal
   end
 
   def self.graylog_logging_destination
-    @graylog_destination ||= begin
+    @graylog_logging_destination ||= begin
       notifier = GELF::Notifier.new(config.logging.graylog.host, config.logging.graylog.port, "WAN")
       proc do |_logger, payload, group_ids|
         short_message = payload.delete(:message) || "[message missing]"

lib/postal/job.rb

@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-require "nifty/utils/random_string"
-
-module Postal
-  class Job
-
-    def initialize(id, params = {})
-      @id = id
-      @params = params
-      on_initialize
-    end
-
-    attr_reader :id
-
-    def params
-      @params || {}
-    end
-
-    def on_initialize
-      # Called whenever the class is initialized. Can be overriden.
-    end
-
-    def on_error(exception)
-      # Called if there's an exception while processing the perform block.
-      # Receives the exception.
-    end
-
-    def perform
-    end
-
-    def log(text)
-      Worker.logger.info(text)
-    end
-
-    def self.queue(queue, params = {})
-      job_id = Nifty::Utils::RandomString.generate(length: 10).upcase
-      job_payload = { "params" => params, "class_name" => name, "id" => job_id, "queue" => queue }
-      Postal::Worker.job_queue(queue).publish(job_payload.to_json, persistent: false)
-      job_id
-    end
-
-  end
-end

lib/postal/message_db/message.rb

@@ -445,7 +445,11 @@ module Postal
       #
       def bounce!(bounce_message)
         create_delivery("Bounced", details: "We've received a bounce message for this e-mail. See <msg:#{bounce_message.id}> for details.")
-        SendWebhookJob.queue(:main, server_id: database.server_id, event: "MessageBounced", payload: { _original_message: id, _bounce: bounce_message.id })
+
+        WebhookRequest.trigger(server, "MessageBounced", {
+          original_message: webhook_hash,
+          bounce: bounce_message.webhook_hash
+        })
       end
 
       #
@@ -461,7 +465,12 @@ module Postal
       def create_load(request)
         update("loaded" => Time.now.to_f) if loaded.nil?
         database.insert(:loads, { message_id: id, ip_address: request.ip, user_agent: request.user_agent, timestamp: Time.now.to_f })
-        SendWebhookJob.queue(:main, server_id: database.server_id, event: "MessageLoaded", payload: { _message: id, ip_address: request.ip, user_agent: request.user_agent })
+
+        WebhookRequest.trigger(server, "MessageLoaded", {
+          message: webhook_hash,
+          ip_address: request.ip,
+          user_agent: request.user_agent
+        })
       end
 
       #

lib/postal/message_requeuer.rb

@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-module Postal
-  class MessageRequeuer
-
-    def run
-      Signal.trap("INT")  { @running ? @exit = true : Process.exit(0) }
-      Signal.trap("TERM") { @running ? @exit = true : Process.exit(0) }
-
-      log "Running message requeuer..."
-      loop do
-        @running = true
-        QueuedMessage.requeue_all
-        @running = false
-        check_exit
-        sleep 5
-      end
-    end
-
-    private
-
-    def log(text)
-      Postal.logger.info text, component: "message-requeuer"
-    end
-
-    def check_exit
-      return unless @exit
-
-      log "Exiting"
-      Process.exit(0)
-    end
-
-  end
-end

lib/postal/rabbit_mq.rb

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-require "postal/config"
-require "bunny"
-
-module Postal
-  module RabbitMQ
-
-    def self.create_connection
-      bunny_host = ["localhost"]
-
-      if Postal.config.rabbitmq&.host.is_a?(Array)
-        bunny_host = Postal.config.rabbitmq&.host
-      elsif Postal.config.rabbitmq&.host.is_a?(String)
-        bunny_host = [Postal.config.rabbitmq&.host]
-      end
-
-      conn = Bunny.new(
-        hosts: bunny_host,
-        port: Postal.config.rabbitmq&.port || 5672,
-        tls: Postal.config.rabbitmq&.tls || false,
-        verify_peer: Postal.config.rabbitmq&.verify_peer || true,
-        tls_ca_certificates: Postal.config.rabbitmq&.tls_ca_certificates || ["/etc/ssl/certs/ca-certificates.crt"],
-        username: Postal.config.rabbitmq&.username || "guest",
-        password: Postal.config.rabbitmq&.password || "guest",
-        vhost: Postal.config.rabbitmq&.vhost || nil
-      )
-      conn.start
-      conn
-    end
-
-    def self.create_channel
-      conn = create_connection
-      conn.create_channel(nil, Postal.config.workers.threads)
-    end
-
-  end
-end

lib/postal/tracking_middleware.rb

@@ -94,16 +94,20 @@ module Postal
           user_agent: request.user_agent,
           timestamp: time
         })
-        SendWebhookJob.queue(:main,
-                             server_id: message_db.server_id,
-                             event: "MessageLinkClicked",
-                             payload: {
-                               _message: link["message_id"],
-                               url: link["url"],
-                               token: link["token"],
-                               ip_address: request.ip,
-                               user_agent: request.user_agent
-                             })
+
+        begin
+          message_webhook_hash = message_db.message(link["message_id"]).webhook_hash
+          WebhookRequest.trigger(message_db.server, "MessageLinkClicked", {
+            message: message_webhook_hash,
+            url: link["url"],
+            token: link["token"],
+            ip_address: request.ip,
+            user_agent: request.user_agent
+          })
+        rescue Postal::MessageDB::Message::NotFound
+          # If we can't find the message that this link is associated with, we'll just ignore it
+          # and not trigger any webhooks.
+        end
       end
 
       [307, { "Location" => link["url"] }, ["Redirected to: #{link['url']}"]]

lib/postal/worker.rb

@@ -1,220 +0,0 @@
-# frozen_string_literal: true
-
-module Postal
-  class Worker
-
-    def initialize(queues)
-      @initial_queues = queues
-      @active_queues = {}
-      @process_name = $0
-      @running_jobs = []
-    end
-
-    def work
-      logger.info "Worker running with #{Postal.config.workers.threads} threads"
-
-      Signal.trap("INT")  do
-        @exit = true
-        set_process_name
-      end
-      Signal.trap("TERM") do
-        @exit = true
-        set_process_name
-      end
-
-      self.class.job_channel.prefetch(Postal.config.workers.threads)
-      @initial_queues.each { |queue| join_queue(queue) }
-
-      exit_checks = 0
-      loop do
-        if @exit && @running_jobs.empty?
-          logger.info "Exiting immediately because no jobs running"
-          exit 0
-        elsif @exit
-          if exit_checks >= 60
-            logger.info "Job did not finish in a timely manner. Exiting"
-            exit 0
-          end
-          if exit_checks.zero?
-            logger.info "Exit requested but job is running. Waiting for job to finish."
-          end
-          sleep 60
-          exit_checks += 1
-        else
-          manage_ip_queues
-          sleep 1
-        end
-      end
-    end
-
-    private
-
-    def receive_job(delivery_info, properties, message)
-      if message && message["class_name"]
-        @running_jobs << message["id"]
-        set_process_name
-        start_time = Time.now
-        Thread.current[:job_id] = message["id"]
-        logger.info "Processing job"
-        begin
-          klass = message["class_name"].constantize.new(message["id"], message["params"])
-          klass.perform
-          GC.start
-        rescue StandardError => e
-          klass.on_error(e) if defined?(klass)
-          logger.exception(e)
-          if defined?(Sentry)
-            Sentry.capture_exception(e, extra: { job_id: message["id"] })
-          end
-        ensure
-          logger.info "Finished job", time: (Time.now - start_time).to_i
-        end
-      end
-    ensure
-      Thread.current[:job_id] = nil
-      self.class.job_channel.ack(delivery_info.delivery_tag)
-      @running_jobs.delete(message["id"]) if message["id"]
-      set_process_name
-
-      if @exit && @running_jobs.empty?
-        logger.info "Exiting because all jobs have finished."
-        exit 0
-      end
-    end
-
-    def join_queue(queue)
-      if @active_queues[queue]
-        logger.error "attempted to join queue but already joined", queue: queue
-      else
-        consumer = self.class.job_queue(queue).subscribe(manual_ack: true) do |delivery_info, properties, body|
-          message = begin
-            JSON.parse(body)
-          rescue StandardError
-            nil
-          end
-
-          logger.tagged(job_id: message["id"], queue: queue, job_class: message["class_name"]) do
-            receive_job(delivery_info, properties, message)
-          end
-        end
-        @active_queues[queue] = consumer
-        logger.info "joined queue", queue: queue
-      end
-    end
-
-    def leave_queue(queue)
-      if consumer = @active_queues[queue]
-        consumer.cancel
-        @active_queues.delete(queue)
-        logger.info "left queue", queue: queue
-      else
-        logger.error "requested to leave queue, but not joined", queue: queue
-      end
-    end
-
-    def manage_ip_queues
-      @ip_queues ||= []
-      @ip_to_id_mapping ||= {}
-      @unassigned_ips ||= []
-      @pairs ||= {}
-      @counter ||= 0
-
-      if @counter >= 15
-        @ip_to_id_mapping = {}
-        @unassigned_ips = []
-        @counter = 0
-      else
-        @counter += 1
-      end
-
-      # Get all IP addresses on the system
-      current_ip_addresses = Socket.ip_address_list.map(&:ip_address)
-
-      # Map them to an actual ID in the database if we can and cache that
-      needed_ip_ids = []
-      current_ip_addresses.each do |ip|
-        need = nil
-        if id = @ip_to_id_mapping[ip]
-          # We know this IPs ID, we'll just use that.
-          need = id
-        elsif @unassigned_ips.include?(ip)
-          # We know this IP isn't valid. We don't need to do anything
-        elsif !self.class.local_ip?(ip) && ip_address = IPAddress.where("ipv4 = ? OR ipv6 = ?", ip, ip).first
-          # We need to look this up
-          @pairs[ip_address.ipv4] = ip_address.ipv6
-          @ip_to_id_mapping[ip] = ip_address.id
-          need = ip_address.id
-        else
-          @unassigned_ips << ip
-        end
-
-        next unless need
-
-        pair = @pairs[ip] || @pairs.key(ip)
-        if pair.nil? || current_ip_addresses.include?(pair)
-          needed_ip_ids << @ip_to_id_mapping[ip]
-        else
-          logger.info "Host has '#{ip}' but its pair (#{pair}) isn't here. Cannot add now."
-        end
-      end
-
-      # Make an array of needed queue names
-      # Work out what we need to actually do here
-      missing_queues = needed_ip_ids - @ip_queues
-      unwanted_queues = @ip_queues - needed_ip_ids
-      # Leave the queues we don't want any more
-      unwanted_queues.each do |id|
-        leave_queue("outgoing-#{id}")
-        @ip_queues.delete(id)
-        ip_addresses_to_clear = []
-        @ip_to_id_mapping.each do |iip, iid|
-          if id == iid
-            ip_addresses_to_clear << iip
-          end
-        end
-        ip_addresses_to_clear.each { |ip| @ip_to_id_mapping.delete(ip) }
-      end
-      # Join any missing queues
-      missing_queues.uniq.each do |id|
-        join_queue("outgoing-#{id}")
-        @ip_queues << id
-      end
-    end
-
-    def set_process_name
-      prefix = @process_name.to_s
-      prefix += " [exiting]" if @exit
-      if @running_jobs.empty?
-        $0 = "#{prefix} (idle)"
-      else
-        $0 = "#{prefix} (running #{@running_jobs.join(', ')})"
-      end
-    end
-
-    def logger
-      self.class.logger
-    end
-
-    class << self
-
-      def logger
-        Postal.logger
-      end
-
-      def job_channel
-        @job_channel ||= Postal::RabbitMQ.create_channel
-      end
-
-      def job_queue(name)
-        @job_queues ||= {}
-        @job_queues[name] ||= job_channel.queue("deliver-jobs-#{name}", durable: true, arguments: { "x-message-ttl" => 60_000 })
-      end
-
-      def local_ip?(ip)
-        !!(ip =~ /\A(127\.|fe80:|::)/)
-      end
-
-    end
-
-  end
-end