test: add tests for message unqueueing

This adds a comprehensive set of tests for the message unqueueing service.

Additionally, it improves how message databases are used for testing environments.

app/services/unqueue_message_service.rb

@@ -163,7 +163,10 @@ class UnqueueMessageService
         queued_message.message.inspect_message
         if queued_message.message.inspected
           is_spam = queued_message.message.spam_score > queued_message.server.spam_threshold
-          queued_message.message.update(spam: true) if is_spam
+          if is_spam
+            queued_message.message.update(spam: true)
+            log "message is spam (scored #{queued_message.message.spam_score}, threshold is #{queued_message.server.spam_threshold})"
+          end
           queued_message.message.append_headers(
             "X-Postal-Spam: #{queued_message.message.spam ? 'yes' : 'no'}",
             "X-Postal-Spam-Threshold: #{queued_message.server.spam_threshold}",
@@ -285,7 +288,7 @@ class UnqueueMessageService
           # If the message is a hard fail, send a bounce message for this message.
           log "sending a bounce because message hard failed"
           if bounce_id = queued_message.send_bounce
-            log_details += ". " unless log_details =~ /\.\z/
+            log_details += "." unless log_details =~ /\.\z/
             log_details += " Sent bounce message to sender (see message <msg:#{bounce_id}>)"
           end
         end
@@ -445,7 +448,8 @@ class UnqueueMessageService
       if recent_hard_fails >= 1 && queued_message.server.message_db.suppression_list.add(:recipient, queued_message.message.rcpt_to, reason: "too many hard fails")
         log "Added #{queued_message.message.rcpt_to} to suppression list because #{recent_hard_fails} hard fails in 24 hours"
         result.details += "." if result.details =~ /\.\z/
-        result.details += " Recipient added to suppression list (too many hard fails)."
+        result.details += " " if result.details.present?
+        result.details += "Recipient added to suppression list (too many hard fails)."
       end
     end
 
@@ -477,6 +481,7 @@ class UnqueueMessageService
     if defined?(Sentry)
       Sentry.capture_exception(e, extra: { server_id: queued_message.server_id, queued_message_id: queued_message.message_id })
     end
+
     queued_message.message&.create_delivery("Error",
                                             details: "An internal error occurred while sending " \
                                                      "this message. This message will be retried " \

lib/postal/message_db/database.rb

@@ -12,9 +12,10 @@ module Postal
 
       end
 
-      def initialize(organization_id, server_id)
+      def initialize(organization_id, server_id, database_name: nil)
         @organization_id = organization_id
         @server_id = server_id
+        @database_name = database_name
       end
 
       attr_reader :organization_id

lib/postal/message_db/message.rb

@@ -39,6 +39,10 @@ module Postal
         @attributes = attributes
       end
 
+      def reload
+        self.class.find_one(@database, @attributes["id"])
+      end
+
       #
       # Return the server for this message
       #
@@ -200,9 +204,9 @@ module Postal
       #
       #  Save this message
       #
-      def save
+      def save(queue_on_create: true)
         save_raw_message
-        persisted? ? _update : _create
+        persisted? ? _update : _create(queue: queue_on_create)
         self
       end
 
@@ -346,8 +350,14 @@ module Postal
       #
       # Create a new item in the message queue for this message
       #
-      def add_to_message_queue(options = {})
-        QueuedMessage.create!(message: self, server_id: @database.server_id, batch_key: batch_key, domain: recipient_domain, route_id: route_id, manual: options[:manual]).id
+      def add_to_message_queue(**options)
+        QueuedMessage.create!({
+          message: self,
+          server_id: @database.server_id,
+          batch_key: batch_key,
+          domain: recipient_domain,
+          route_id: route_id
+        }.merge(options))
       end
 
       #
@@ -572,7 +582,7 @@ module Postal
         @database.update("messages", @attributes.except(:id), where: { id: @attributes["id"] })
       end
 
-      def _create
+      def _create(queue: true)
         self.timestamp = Time.now.to_f if timestamp.blank?
         self.status = "Pending" if status.blank?
         self.token = Nifty::Utils::RandomString.generate(length: 12) if token.blank?
@@ -581,7 +591,7 @@ module Postal
         @database.statistics.increment_all(timestamp, scope)
         Statistic.global.increment!(:total_messages)
         Statistic.global.increment!("total_#{scope}".to_sym)
-        add_to_message_queue
+        add_to_message_queue if queue
       end
 
       def mail

lib/postal/rspec_helpers.rb

@@ -3,13 +3,6 @@
 module Postal
   module RspecHelpers
 
-    def with_global_server(&block)
-      server = Server.find(GLOBAL_SERVER.id)
-      block.call(server)
-    ensure
-      server.message_db.provisioner.clean
-    end
-
     def create_plain_text_message(server, text, to = "test@example.com", override_attributes = {})
       domain = create(:domain, owner: server)
       attributes = { from: "test@#{domain.name}", subject: "Test Plain Text Message" }.merge(override_attributes)

lib/postal/send_result.rb

@@ -13,5 +13,10 @@ module Postal
     attr_accessor :time
     attr_accessor :suppress_bounce
 
+    def initialize
+      @details = ""
+      yield self if block_given?
+    end
+
   end
 end

lib/test_logger.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+class TestLogger
+
+  def initialize
+    @log_lines = []
+    @group_set = Klogger::GroupSet.new
+    @print = false
+  end
+
+  def print!
+    @print = true
+  end
+
+  def add(level, message, **tags)
+    @group_set.groups.each do |group|
+      tags = group[:tags].merge(tags)
+    end
+
+    @log_lines << { level: level, message: message, tags: tags }
+    puts message if @print
+    true
+  end
+
+  [:info, :debug, :warn, :error].each do |level|
+    define_method(level) do |message, **tags|
+      add(level, message, **tags)
+    end
+  end
+
+  def tagged(**tags, &block)
+    @group_set.call_without_id(**tags, &block)
+  end
+
+  def log_line(match)
+    @log_lines.reverse.each do |log_line|
+      return log_line if match.is_a?(String) && log_line[:message] == match
+      return log_line if match.is_a?(Regexp) && log_line[:message] =~ match
+    end
+    nil
+  end
+
+  def has_logged?(match)
+    !!log_line(match)
+  end
+
+end

spec/app/models/outgoing_message_prototype_spec.rb

@@ -3,21 +3,20 @@
 require "rails_helper"
 
 describe OutgoingMessagePrototype do
+  let(:server) { create(:server) }
   it "should create a new message" do
-    with_global_server do |server|
-      domain = create(:domain, owner: server)
-      prototype = OutgoingMessagePrototype.new(server, "127.0.0.1", "TestSuite", {
-        from: "test@#{domain.name}",
-        to: "test@example.com",
-        subject: "Test Message",
-        plain_body: "A plain body!"
-      })
+    domain = create(:domain, owner: server)
+    prototype = OutgoingMessagePrototype.new(server, "127.0.0.1", "TestSuite", {
+      from: "test@#{domain.name}",
+      to: "test@example.com",
+      subject: "Test Message",
+      plain_body: "A plain body!"
+    })
 
-      expect(prototype.valid?).to be true
-      message = prototype.create_message("test@example.com")
-      expect(message).to be_a Hash
-      expect(message[:id]).to be_a Integer
-      expect(message[:token]).to be_a String
-    end
+    expect(prototype.valid?).to be true
+    message = prototype.create_message("test@example.com")
+    expect(message).to be_a Hash
+    expect(message[:id]).to be_a Integer
+    expect(message[:token]).to be_a String
   end
 end

spec/app/services/unqueue_message_service/incoming_messages_spec.rb

@@ -0,0 +1,743 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe UnqueueMessageService do
+  let(:server) { create(:server) }
+  let(:logger) { TestLogger.new }
+  let(:queued_message) { create(:queued_message, server: server) }
+  subject(:service) { described_class.new(queued_message: queued_message, logger: logger) }
+
+  # We're going to, for now, just stop the SMTP sender from doing anything here because
+  # we don't want to leak out of this test in to the real world.
+  before do
+    smtp_sender_mock = double("SMTPSender")
+    allow(Postal::SMTPSender).to receive(:new).and_return(smtp_sender_mock)
+    allow(smtp_sender_mock).to receive(:start)
+    allow(smtp_sender_mock).to receive(:finish)
+    allow(smtp_sender_mock).to receive(:send_message) do
+      puts "SMTP SENDING DETECTED!"
+    end
+  end
+
+  describe "#call" do
+    context "for an incoming message" do
+      let(:route) { create(:route, server: server) }
+      let(:message) { MessageFactory.incoming(server, route: route) }
+      let(:queued_message) { create(:queued_message, :locked, message: message) }
+
+      context "when the server is suspended" do
+        before do
+          allow(queued_message.server).to receive(:suspended?).and_return(true)
+        end
+
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/server is suspended/)
+        end
+
+        it "sets the message status to Held" do
+          service.call
+          expect(message.reload.status).to eq "Held"
+        end
+
+        it "creates a Held delivery" do
+          service.call
+          delivery = message.deliveries.last
+          expect(delivery).to have_attributes(status: "Held", details: /server has been suspended/i)
+        end
+
+        it "removes the queued message" do
+          service.call
+          expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        end
+      end
+
+      context "when the number of attempts is more than the maximum" do
+        let(:queued_message) { create(:queued_message, :locked, message: message, attempts: Postal.config.general.maximum_delivery_attempts + 1) }
+
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/message has reached maximum number of attempts/)
+        end
+
+        it "sends a bounce to the sender" do
+          expect(Postal::BounceMessage).to receive(:new).with(server, queued_message.message)
+          service.call
+        end
+
+        it "sets the message status to HardFail" do
+          service.call
+          expect(message.reload.status).to eq "HardFail"
+        end
+
+        it "creates a HardFail delivery" do
+          service.call
+          delivery = message.deliveries.last
+          expect(delivery).to have_attributes(status: "HardFail", details: /maximum number of delivery attempts.*bounce sent to sender/i)
+        end
+
+        it "removes the queued message" do
+          service.call
+          expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        end
+      end
+
+      context "when the message raw data has been removed" do
+        before do
+          message.raw_table = nil
+          message.save
+        end
+
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/raw message has been removed/)
+        end
+
+        it "sets the message status to HardFail" do
+          service.call
+          expect(message.reload.status).to eq "HardFail"
+        end
+
+        it "creates a HardFail delivery" do
+          service.call
+          delivery = message.deliveries.last
+          expect(delivery).to have_attributes(status: "HardFail", details: /Raw message has been removed/i)
+        end
+
+        it "removes the queued message" do
+          service.call
+          expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        end
+      end
+
+      context "when the message is a bounce for an existing message" do
+        let(:existing_message) { MessageFactory.outgoing(server) }
+
+        let(:message) do
+          MessageFactory.incoming(server) do |msg, mail|
+            msg.bounce = true
+            mail["X-Postal-MsgID"] = existing_message.token
+          end
+        end
+
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/message is a bounce/)
+        end
+
+        it "adds the original message as the bounce ID for the received message" do
+          service.call
+          expect(message.reload.bounce_for_id).to eq existing_message.id
+        end
+
+        it "sets the received message status to Processed" do
+          service.call
+          expect(message.reload.status).to eq "Processed"
+        end
+
+        it "creates a Processed delivery on the received message" do
+          service.call
+          delivery = message.deliveries.last
+          expect(delivery).to have_attributes(status: "Processed", details: /This has been detected as a bounce message for <msg:#{existing_message.id}>/i)
+        end
+
+        it "sets the existing message status to Bounced" do
+          service.call
+          expect(existing_message.reload.status).to eq "Bounced"
+        end
+
+        it "creates a Bounced delivery on the original message" do
+          service.call
+          delivery = existing_message.deliveries.last
+          expect(delivery).to have_attributes(status: "Bounced", details: /received a bounce message for this e-mail. See <msg:#{message.id}> for/i)
+        end
+
+        it "triggers a MessageBounced webhook event" do
+          expect(WebhookRequest).to receive(:trigger).with(server, "MessageBounced", {
+            original_message: kind_of(Hash),
+            bounce: kind_of(Hash)
+          })
+          service.call
+        end
+
+        it "removes the queued message" do
+          service.call
+          expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        end
+      end
+
+      context "when the message was a bounce but there's no return path for it" do
+        let(:message) do
+          MessageFactory.incoming(server) do |msg|
+            msg.bounce = true
+          end
+        end
+
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/no source messages found, hard failing/)
+        end
+
+        it "sets the message status to HardFail" do
+          service.call
+          expect(message.reload.status).to eq "HardFail"
+        end
+
+        it "creates a HardFail delivery" do
+          service.call
+          delivery = message.deliveries.last
+          expect(delivery).to have_attributes(status: "HardFail", details: /was a bounce but we couldn't link it with any outgoing message/i)
+        end
+
+        it "removes the queued message" do
+          service.call
+          expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        end
+      end
+
+      context "when the message is not a bounce" do
+        it "increments the stats for the server" do
+          expect { service.call }.to change { server.message_db.live_stats.total(5) }.by(1)
+        end
+
+        it "inspects the message and adds headers" do
+          expect { service.call }.to change { message.reload.inspected }.from(false).to(true)
+          new_message = message.reload
+          expect(new_message.headers).to match hash_including(
+            "x-postal-spam" => ["no"],
+            "x-postal-spam-threshold" => ["5.0"],
+            "x-postal-threat" => ["no"]
+          )
+        end
+
+        it "marks the message as spam if the spam score is higher than the server threshold" do
+          inspection_result = double("Result", spam_score: server.spam_threshold + 1, threat: false, threat_message: nil, spam_checks: [])
+          allow(Postal::MessageInspection).to receive(:scan).and_return(inspection_result)
+          service.call
+          expect(message.reload.spam).to be true
+        end
+      end
+
+      context "when the message has a spam score greater than the server's spam failure threshold" do
+        before do
+          inspection_result = double("Result", spam_score: 100, threat: false, threat_message: nil, spam_checks: [])
+          allow(Postal::MessageInspection).to receive(:scan).and_return(inspection_result)
+        end
+
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/message has a spam score higher than the server's maxmimum/)
+        end
+
+        it "sets the message status to HardFail" do
+          service.call
+          expect(message.reload.status).to eq "HardFail"
+        end
+
+        it "creates a HardFail delivery" do
+          service.call
+          delivery = message.deliveries.last
+          expect(delivery).to have_attributes(status: "HardFail", details: /spam score is higher than the failure threshold for this server/i)
+        end
+
+        it "removes the queued message" do
+          service.call
+          expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        end
+      end
+
+      context "when the server mode is Development and the message was not manually queued" do
+        before do
+          server.update!(mode: "Development")
+        end
+
+        after do
+          server.update!(mode: "Live")
+        end
+
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/server is in development mode/)
+        end
+
+        it "sets the message status to Held" do
+          service.call
+          expect(message.reload.status).to eq "Held"
+        end
+
+        it "creates a Held delivery" do
+          service.call
+          delivery = message.deliveries.last
+          expect(delivery).to have_attributes(status: "Held", details: /server is in development mode/i)
+        end
+
+        it "removes the queued message" do
+          service.call
+          expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        end
+      end
+
+      context "when there is no route for the incoming message" do
+        let(:route) { nil }
+
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/no route and\/or endpoint available for processing/i)
+        end
+
+        it "sets the message status to HardFail" do
+          service.call
+          expect(message.reload.status).to eq "HardFail"
+        end
+
+        it "creates a HardFail delivery" do
+          service.call
+          delivery = message.deliveries.last
+          expect(delivery).to have_attributes(status: "HardFail", details: /does not have a route and\/or endpoint available/i)
+        end
+
+        it "removes the queued message" do
+          service.call
+          expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        end
+      end
+
+      context "when the route's spam mode is Quarantine, the message is spam and not manually queued" do
+        let(:route) { create(:route, server: server, spam_mode: "Quarantine") }
+
+        before do
+          inspection_result = double("Result", spam_score: server.spam_threshold + 1, threat: false, threat_message: nil, spam_checks: [])
+          allow(Postal::MessageInspection).to receive(:scan).and_return(inspection_result)
+        end
+
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/message is spam and route says to quarantine spam message/i)
+        end
+
+        it "sets the message status to Held" do
+          service.call
+          expect(message.reload.status).to eq "Held"
+        end
+
+        it "creates a Held delivery" do
+          service.call
+          delivery = message.deliveries.last
+          expect(delivery).to have_attributes(status: "Held", details: /message placed into quarantine/i)
+        end
+
+        it "removes the queued message" do
+          service.call
+          expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        end
+      end
+
+      context "when the route's spam mode is Fail, the message is spam and not manually queued" do
+        let(:route) { create(:route, server: server, spam_mode: "Fail") }
+
+        before do
+          inspection_result = double("Result", spam_score: server.spam_threshold + 1, threat: false, threat_message: nil, spam_checks: [])
+          allow(Postal::MessageInspection).to receive(:scan).and_return(inspection_result)
+        end
+
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/message is spam and route says to fail spam message/i)
+        end
+
+        it "sets the message status to HardFail" do
+          service.call
+          expect(message.reload.status).to eq "HardFail"
+        end
+
+        it "creates a HardFail delivery" do
+          service.call
+          delivery = message.deliveries.last
+          expect(delivery).to have_attributes(status: "HardFail", details: /message is spam and the route specifies it should be failed/i)
+        end
+
+        it "removes the queued message" do
+          service.call
+          expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        end
+      end
+
+      context "when the route's mode is Accept" do
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/route says to accept without endpoint/i)
+        end
+
+        it "sets the message status to Processed" do
+          service.call
+          expect(message.reload.status).to eq "Processed"
+        end
+
+        it "creates a Processed delivery" do
+          service.call
+          delivery = message.deliveries.last
+          expect(delivery).to have_attributes(status: "Processed", details: /message has been accepted but not sent to any endpoints/i)
+        end
+
+        it "removes the queued message" do
+          service.call
+          expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        end
+      end
+
+      context "when the route's mode is Hold" do
+        let(:route) { create(:route, server: server, mode: "Hold") }
+
+        context "when the message was queued manually" do
+          let(:queued_message) { create(:queued_message, :locked, server: server, message: message, manual: true) }
+
+          it "logs" do
+            service.call
+            expect(logger).to have_logged(/route says to hold and message was queued manually/i)
+          end
+
+          it "sets the message status to Processed" do
+            service.call
+            expect(message.reload.status).to eq "Processed"
+          end
+
+          it "creates a Processed delivery" do
+            service.call
+            delivery = message.deliveries.last
+            expect(delivery).to have_attributes(status: "Processed", details: /message has been processed/i)
+          end
+
+          it "removes the queued message" do
+            service.call
+            expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+          end
+        end
+
+        context "when the message was not queued manually" do
+          let(:queued_message) { create(:queued_message, :locked, server: server, message: message, manual: false) }
+
+          it "logs" do
+            service.call
+            expect(logger).to have_logged(/route says to hold, marking as held/i)
+          end
+
+          it "sets the message status to Held" do
+            service.call
+            expect(message.reload.status).to eq "Held"
+          end
+
+          it "creates a Held delivery" do
+            service.call
+            delivery = message.deliveries.last
+            expect(delivery).to have_attributes(status: "Held", details: /message has been accepted but not sent to any endpoints/i)
+          end
+
+          it "removes the queued message" do
+            service.call
+            expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+          end
+        end
+      end
+
+      context "when the route's mode is Bounce" do
+        let(:route) { create(:route, server: server, mode: "Bounce") }
+
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/route says to bounce/i)
+        end
+
+        it "sends a bounce" do
+          expect(Postal::BounceMessage).to receive(:new).with(server, queued_message.message)
+          service.call
+        end
+
+        it "sets the message status to HardFail" do
+          service.call
+          expect(message.reload.status).to eq "HardFail"
+        end
+
+        it "creates a HardFail delivery" do
+          service.call
+          delivery = message.deliveries.last
+          expect(delivery).to have_attributes(status: "HardFail", details: /message has been bounced because/i)
+        end
+
+        it "removes the queued message" do
+          service.call
+          expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        end
+      end
+
+      context "when the route's mode is Reject" do
+        let(:route) { create(:route, server: server, mode: "Reject") }
+
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/route says to bounce/i)
+        end
+
+        it "sends a bounce" do
+          expect(Postal::BounceMessage).to receive(:new).with(server, queued_message.message)
+          service.call
+        end
+
+        it "sets the message status to HardFail" do
+          service.call
+          expect(message.reload.status).to eq "HardFail"
+        end
+
+        it "creates a HardFail delivery" do
+          service.call
+          delivery = message.deliveries.last
+          expect(delivery).to have_attributes(status: "HardFail", details: /message has been bounced because/i)
+        end
+
+        it "removes the queued message" do
+          service.call
+          expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        end
+      end
+
+      context "when the route's endpoint is an HTTP endpoint" do
+        let(:endpoint) { create(:http_endpoint, server: server) }
+        let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) }
+
+        it "sends the message to the HTTPSender" do
+          http_sender_double = double("HTTPSender")
+          expect(Postal::HTTPSender).to receive(:new).with(endpoint).and_return(http_sender_double)
+          expect(http_sender_double).to receive(:start).with(no_args)
+          expect(http_sender_double).to receive(:finish).with(no_args)
+          expect(http_sender_double).to receive(:send_message).with(queued_message.message).and_return(Postal::SendResult.new)
+          service.call
+        end
+      end
+
+      context "when the route's endpoint is an SMTP endpoint" do
+        let(:endpoint) { create(:smtp_endpoint, server: server) }
+        let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) }
+
+        it "sends the message to the SMTPSender" do
+          smtp_sender_double = double("SMTPSender")
+          expect(smtp_sender_double).to receive(:start).with(no_args)
+          expect(smtp_sender_double).to receive(:finish).with(no_args)
+          expect(smtp_sender_double).to receive(:send_message).with(queued_message.message).and_return(Postal::SendResult.new)
+          expect(Postal::SMTPSender).to receive(:new).with(message.recipient_domain, nil, { servers: [endpoint] }).and_return(smtp_sender_double)
+          service.call
+        end
+      end
+
+      context "when the route's endpoint is an Address endpoint" do
+        let(:endpoint) { create(:address_endpoint, server: server) }
+        let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) }
+
+        it "sends the message to the SMTPSender" do
+          smtp_sender_double = double("SMTPSender")
+          expect(smtp_sender_double).to receive(:start).with(no_args)
+          expect(smtp_sender_double).to receive(:finish).with(no_args)
+          expect(smtp_sender_double).to receive(:send_message).with(queued_message.message).and_return(Postal::SendResult.new)
+          expect(Postal::SMTPSender).to receive(:new).with(endpoint.domain, nil, { force_rcpt_to: endpoint.address }).and_return(smtp_sender_double)
+          service.call
+        end
+      end
+
+      context "when the route's endpoint is an unknown endpoint" do
+        let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: create(:webhook, server: server)) }
+
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/invalid endpoint for route/i)
+        end
+
+        it "sets the message status to HardFail" do
+          service.call
+          expect(message.reload.status).to eq "HardFail"
+        end
+
+        it "creates a HardFail delivery" do
+          service.call
+          delivery = message.deliveries.last
+          expect(delivery).to have_attributes(status: "HardFail", details: /invalid endpoint for route/i)
+        end
+
+        it "removes the queued message" do
+          service.call
+          expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        end
+      end
+
+      context "when the message has been sent to a sender" do
+        let(:endpoint) { create(:smtp_endpoint, server: server) }
+        let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) }
+
+        let(:send_result) do
+          Postal::SendResult.new do |result|
+            result.type = "Sent"
+            result.details = "Sent successfully"
+          end
+        end
+
+        before do
+          smtp_sender_mock = double("SMTPSender")
+          allow(Postal::SMTPSender).to receive(:new).and_return(smtp_sender_mock)
+          allow(smtp_sender_mock).to receive(:start)
+          allow(smtp_sender_mock).to receive(:finish)
+          allow(smtp_sender_mock).to receive(:send_message).and_return(send_result)
+        end
+
+        context "when the sender returns a HardFail and bounces are suppressed" do
+          before do
+            send_result.type = "HardFail"
+            send_result.suppress_bounce = true
+          end
+
+          it "logs" do
+            service.call
+            expect(logger).to have_logged(/suppressing bounce message after hard fail/)
+          end
+
+          it "does not send a bounce" do
+            allow(Postal::BounceMessage).to receive(:new)
+            service.call
+            expect(Postal::BounceMessage).to_not have_received(:new)
+          end
+        end
+
+        context "when the sender returns a HardFail and bounces should be sent" do
+          before do
+            send_result.type = "HardFail"
+            send_result.details = "Failed to send message"
+          end
+
+          it "logs" do
+            service.call
+            expect(logger).to have_logged(/sending a bounce because message hard failed/)
+          end
+
+          it "sends a bounce" do
+            expect(Postal::BounceMessage).to receive(:new).with(server, queued_message.message)
+            service.call
+          end
+
+          it "sets the message status to HardFail" do
+            service.call
+            expect(message.reload.status).to eq "HardFail"
+          end
+
+          it "creates a delivery with the details and a suffix about the bounce message" do
+            service.call
+            delivery = message.deliveries.last
+            expect(delivery).to have_attributes(status: "HardFail", details: /Failed to send message. Sent bounce message to sender \(see message <msg:\d+>\)/i)
+          end
+        end
+
+        it "creates a delivery with the result from the sender" do
+          send_result.output = "some output here"
+          send_result.secure = true
+          send_result.log_id = "12345"
+          send_result.time = 2.32
+
+          service.call
+          delivery = message.deliveries.last
+          expect(delivery).to have_attributes(status: "Sent",
+                                              details: "Sent successfully",
+                                              output: "some output here",
+                                              sent_with_ssl: true,
+                                              log_id: "12345",
+                                              time: 2.32)
+        end
+
+        context "when the sender wants to retry" do
+          before do
+            send_result.type = "SoftFail"
+            send_result.retry = true
+          end
+
+          it "logs" do
+            service.call
+            expect(logger).to have_logged(/message requeued for trying later, at/i)
+          end
+
+          it "sets the message status to SoftFail" do
+            service.call
+            expect(message.reload.status).to eq "SoftFail"
+          end
+
+          it "updates the queued message with a new retry time" do
+            Timecop.freeze do
+              retry_time = 5.minutes.from_now.change(usec: 0)
+              service.call
+              expect(queued_message.reload.retry_after).to eq retry_time
+            end
+          end
+
+          it "allocates a new IP address to send the message from and updates the queued message" do
+            expect(queued_message).to receive(:allocate_ip_address)
+            service.call
+          end
+
+          it "does not remove the queued message" do
+            service.call
+            expect(queued_message.reload).to be_present
+          end
+        end
+
+        context "when the sender does not want a retry" do
+          it "logs" do
+            service.call
+            expect(logger).to have_logged(/message processing completed/i)
+          end
+
+          it "sets the message status to Sent" do
+            service.call
+            expect(message.reload.status).to eq "Sent"
+          end
+
+          it "marks the endpoint as used" do
+            route.endpoint.update!(last_used_at: nil)
+            Timecop.freeze do
+              expect { service.call }.to change { route.endpoint.reload.last_used_at.to_i }.from(0).to(Time.now.to_i)
+            end
+          end
+
+          it "removes the queued message" do
+            service.call
+            expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+          end
+        end
+      end
+
+      context "when an exception occurrs during processing" do
+        let(:endpoint) { create(:smtp_endpoint, server: server) }
+        let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) }
+
+        before do
+          smtp_sender_mock = double("SMTPSender")
+          allow(Postal::SMTPSender).to receive(:new).and_return(smtp_sender_mock)
+          allow(smtp_sender_mock).to receive(:start)
+          allow(smtp_sender_mock).to receive(:finish)
+          allow(smtp_sender_mock).to receive(:send_message) do
+            1 / 0
+          end
+        end
+
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/internal error: ZeroDivisionError/i)
+        end
+
+        it "creates an Error delivery" do
+          service.call
+          delivery = message.deliveries.last
+          expect(delivery).to have_attributes(status: "Error", details: /internal error/i)
+        end
+
+        it "marks the message for retrying later" do
+          service.call
+          expect(queued_message.reload.retry_after).to be_present
+        end
+      end
+    end
+  end
+end

spec/app/services/unqueue_message_service/outgoing_message_spec.rb

@@ -0,0 +1,600 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe UnqueueMessageService do
+  let(:server) { create(:server) }
+  let(:logger) { TestLogger.new }
+  let(:send_result) do
+    Postal::SendResult.new do |r|
+      r.type = "Sent"
+    end
+  end
+  subject(:service) { described_class.new(queued_message: queued_message, logger: logger) }
+
+  # We're going to, for now, just stop the SMTP sender from doing anything here because
+  # we don't want to leak out of this test in to the real world.
+  before do
+    smtp_sender_mock = double("SMTPSender")
+    allow(Postal::SMTPSender).to receive(:new).and_return(smtp_sender_mock)
+    allow(smtp_sender_mock).to receive(:start)
+    allow(smtp_sender_mock).to receive(:finish)
+    allow(smtp_sender_mock).to receive(:send_message).and_return(send_result)
+  end
+
+  context "for an outgoing message" do
+    let(:domain) { create(:domain, server: server) }
+    let(:credential) { create(:credential, server: server) }
+    let(:message) { MessageFactory.outgoing(server, domain: domain, credential: credential) }
+    let(:queued_message) { create(:queued_message, :locked, message: message) }
+
+    context "when the server is suspended" do
+      let(:server) { create(:server, :suspended) }
+
+      it "logs" do
+        service.call
+        expect(logger).to have_logged(/server is suspended/)
+      end
+
+      it "sets the message status to Held" do
+        service.call
+        expect(message.reload.status).to eq "Held"
+      end
+
+      it "creates a Hold delivery" do
+        service.call
+        delivery = message.deliveries.last
+        expect(delivery).to have_attributes(status: "Held", details: /server has been suspended/i)
+      end
+
+      it "removes the queued message" do
+        service.call
+        expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+      end
+    end
+
+    context "when the number of attempts is more than the maximum" do
+      let(:queued_message) { create(:queued_message, :locked, message: message, attempts: Postal.config.general.maximum_delivery_attempts + 1) }
+
+      it "logs" do
+        service.call
+        expect(logger).to have_logged(/message has reached maximum number of attempts/)
+      end
+
+      it "adds the recipient to the suppression list and logs this" do
+        Timecop.freeze do
+          service.call
+          entry = server.message_db.suppression_list.get(:recipient, message.rcpt_to)
+          expect(entry).to match hash_including(
+            "address" => message.rcpt_to,
+            "type" => "recipient",
+            "reason" => "too many soft fails"
+          )
+        end
+      end
+
+      it "sets the message status to Held" do
+        service.call
+        expect(message.reload.status).to eq "HardFail"
+      end
+
+      it "creates a HardFail delivery" do
+        service.call
+        delivery = message.deliveries.last
+        expect(delivery).to have_attributes(status: "HardFail", details: /maximum number of delivery attempts.*added [\w.@]+ to suppression list/i)
+      end
+
+      it "removes the queued message" do
+        service.call
+        expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+      end
+    end
+
+    context "when the message raw data has been removed" do
+      before do
+        message.raw_table = nil
+        message.save
+      end
+
+      it "logs" do
+        service.call
+        expect(logger).to have_logged(/raw message has been removed/)
+      end
+
+      it "sets the message status to Held" do
+        service.call
+        expect(message.reload.status).to eq "HardFail"
+      end
+
+      it "creates a HardFail delivery" do
+        service.call
+        delivery = message.deliveries.last
+        expect(delivery).to have_attributes(status: "HardFail", details: /Raw message has been removed/i)
+      end
+
+      it "removes the queued message" do
+        service.call
+        expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+      end
+    end
+
+    context "when the domain belonging to the message no longer exists" do
+      before do
+        domain.destroy
+      end
+
+      it "logs" do
+        service.call
+        expect(logger).to have_logged(/message has no domain/)
+      end
+
+      it "sets the message status to HardFail" do
+        service.call
+        expect(message.reload.status).to eq "HardFail"
+      end
+
+      it "creates a HardFail delivery" do
+        service.call
+        delivery = message.deliveries.last
+        expect(delivery).to have_attributes(status: "HardFail", details: /Message's domain no longer exist/i)
+      end
+
+      it "removes the queued message" do
+        service.call
+        expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+      end
+    end
+
+    context "when the message has no rcpt to address" do
+      before do
+        message.update(rcpt_to: "")
+      end
+
+      it "logs" do
+        service.call
+        expect(logger).to have_logged(/message has no 'to' address/)
+      end
+
+      it "sets the message status to HardFail" do
+        service.call
+        expect(message.reload.status).to eq "HardFail"
+      end
+
+      it "creates a HardFail delivery" do
+        service.call
+        delivery = message.deliveries.last
+        expect(delivery).to have_attributes(status: "HardFail", details: /Message doesn't have an RCPT to/i)
+      end
+
+      it "removes the queued message" do
+        service.call
+        expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+      end
+    end
+
+    context "when the message has a x-postal-tag header" do
+      let(:message) do
+        MessageFactory.outgoing(server, domain: domain) do |_msg, mail|
+          mail["x-postal-tag"] = "example-tag"
+        end
+      end
+
+      it "logs" do
+        service.call
+        expect(logger).to have_logged(/added tag: example-tag/)
+      end
+
+      it "adds the tag to the message object" do
+        service.call
+        expect(message.reload.tag).to eq("example-tag")
+      end
+    end
+
+    context "when the credential says to hold the message" do
+      let(:credential) { create(:credential, hold: true) }
+
+      context "when the message was queued manually" do
+        let(:queued_message) { create(:queued_message, :locked, message: message, manual: true) }
+
+        it "does not hold the message" do
+          service.call
+          deliveries = message.deliveries.find { |d| d.status == "Held" }
+          expect(deliveries).to be_nil
+        end
+      end
+
+      context "when the message was not queued manually" do
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/credential wants us to hold messages/)
+        end
+
+        it "sets the message status to Held" do
+          service.call
+          expect(message.reload.status).to eq "Held"
+        end
+
+        it "creates a Held delivery" do
+          service.call
+          delivery = message.deliveries.last
+          expect(delivery).to have_attributes(status: "Held", details: /Credential is configured to hold all messages authenticated/i)
+        end
+
+        it "removes the queued message" do
+          service.call
+          expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        end
+      end
+    end
+
+    context "when the rcpt address is on the suppression list" do
+      before do
+        server.message_db.suppression_list.add(:recipient, message.rcpt_to, reason: "testing")
+      end
+
+      context "when the message was queued manually" do
+        let(:queued_message) { create(:queued_message, :locked, message: message, manual: true) }
+
+        it "does not hold the message" do
+          service.call
+          deliveries = message.deliveries.find { |d| d.status == "Held" }
+          expect(deliveries).to be_nil
+        end
+      end
+
+      context "when the message was not queued manually" do
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/recipient is on the suppression list/)
+        end
+
+        it "sets the message status to Held" do
+          service.call
+          expect(message.reload.status).to eq "Held"
+        end
+
+        it "creates a Held delivery" do
+          service.call
+          delivery = message.deliveries.last
+          expect(delivery).to have_attributes(status: "Held", details: /Recipient \(#{message.rcpt_to}\) is on the suppression list/i)
+        end
+
+        it "removes the queued message" do
+          service.call
+          expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        end
+      end
+    end
+
+    context "when the message content has not been parsed" do
+      it "parses the content" do
+        mocked_parser = double("Result")
+        allow(mocked_parser).to receive(:actioned?).and_return(false)
+        allow(mocked_parser).to receive(:tracked_links).and_return(0)
+        allow(mocked_parser).to receive(:tracked_images).and_return(0)
+        expect(Postal::MessageParser).to receive(:new).with(kind_of(Postal::MessageDB::Message)).and_return(mocked_parser)
+        service.call
+        reloaded_message = message.reload
+        expect(reloaded_message.parsed).to eq 1
+        expect(reloaded_message.tracked_links).to eq 0
+        expect(reloaded_message.tracked_images).to eq 0
+      end
+    end
+
+    context "when the server has an outbound spam threshold configured" do
+      let(:server) { create(:server, outbound_spam_threshold: 5.0) }
+
+      it "logs" do
+        service.call
+        expect(logger).to have_logged(/inspecting message/)
+        expect(logger).to have_logged(/message inspected successfully/)
+      end
+
+      it "inspects the message" do
+        inspection_result = double("Result", spam_score: 1.0, threat: false, threat_message: nil, spam_checks: [])
+        expect(Postal::MessageInspection).to receive(:scan).and_return(inspection_result)
+        service.call
+      end
+
+      context "when the message spam score is higher than the threshold" do
+        before do
+          inspection_result = double("Result", spam_score: 6.0, threat: false, threat_message: nil, spam_checks: [])
+          allow(Postal::MessageInspection).to receive(:scan).and_return(inspection_result)
+        end
+
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/message is spam/)
+        end
+
+        it "sets the spam boolean on the message" do
+          service.call
+          expect(message.reload.spam).to be true
+        end
+
+        it "sets the message status to HardFail" do
+          service.call
+          expect(message.reload.status).to eq "HardFail"
+        end
+
+        it "creates a HardFail delivery" do
+          service.call
+          delivery = message.deliveries.last
+          expect(delivery).to have_attributes(status: "HardFail", details: /Message is likely spam. Threshold is 5.0 and the message scored 6.0/i)
+        end
+
+        it "removes the queued message" do
+          service.call
+          expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        end
+      end
+    end
+
+    context "when the server does not have a outbound spam threshold configured" do
+      it "does not inspect the message" do
+        expect(Postal::MessageInspection).to_not receive(:scan)
+        service.call
+      end
+    end
+
+    context "when the message already has an x-postal-msgid header" do
+      let(:message) do
+        MessageFactory.outgoing(server, domain: domain, credential: credential) do |_, mail|
+          mail["x-postal-msgid"] = "existing-id"
+        end
+      end
+
+      it "does not another one" do
+        service.call
+        expect(message.reload.headers["x-postal-msgid"]).to eq ["existing-id"]
+      end
+
+      it "does not add dkim headers" do
+        service.call
+        expect(message.reload.headers["dkim-signature"]).to be_nil
+      end
+    end
+
+    context "when the message does not have a x-postal-msgid header" do
+      it "adds it" do
+        service.call
+        expect(message.reload.headers["x-postal-msgid"]).to match [match(/[a-zA-Z0-9]{12}/)]
+      end
+
+      it "adds a dkim header" do
+        service.call
+        expect(message.reload.headers["dkim-signature"]).to match [match(/\Av=1; a=rsa-sha256/)]
+      end
+    end
+
+    context "when the server has exceeded its send limit" do
+      let(:server) { create(:server, send_limit: 5) }
+
+      before do
+        5.times { server.message_db.live_stats.increment("outgoing") }
+      end
+
+      it "updates the time the limit was exceeded" do
+        expect { service.call }.to change { server.reload.send_limit_exceeded_at }.from(nil).to(kind_of(Time))
+      end
+
+      it "logs" do
+        service.call
+        expect(logger).to have_logged(/server send limit has been exceeded/)
+      end
+
+      it "sets the message status to Held" do
+        service.call
+        expect(message.reload.status).to eq "Held"
+      end
+
+      it "creates a Held delivery" do
+        service.call
+        delivery = message.deliveries.last
+        expect(delivery).to have_attributes(status: "Held", details: /Message held because send limit \(5\) has been reached/i)
+      end
+
+      it "removes the queued message" do
+        service.call
+        expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+      end
+    end
+
+    context "when the server is approaching its send limit" do
+      let(:server) { create(:server, send_limit: 10) }
+
+      before do
+        9.times { server.message_db.live_stats.increment("outgoing") }
+      end
+
+      it "updates the time the limit was being approached" do
+        expect { service.call }.to change { server.reload.send_limit_approaching_at }.from(nil).to(kind_of(Time))
+      end
+
+      it "does not set the exceeded time" do
+        expect { service.call }.to_not change { server.reload.send_limit_exceeded_at } # rubocop:disable Lint/AmbiguousBlockAssociation
+      end
+    end
+
+    context "when the server is not exceeded or approaching its limit" do
+      let(:server) { create(:server, :exceeded_send_limit, send_limit: 10) }
+
+      it "clears the approaching and exceeded limits" do
+        service.call
+        server.reload
+        expect(server.send_limit_approaching_at).to be_nil
+        expect(server.send_limit_exceeded_at).to be_nil
+      end
+    end
+
+    context "when the server is in development mode" do
+      let(:server) { create(:server, mode: "Development") }
+
+      context "when the message was queued manually" do
+        let(:queued_message) { create(:queued_message, :locked, message: message, manual: true) }
+
+        it "does not hold the message" do
+          service.call
+          deliveries = message.deliveries.find { |d| d.status == "Held" }
+          expect(deliveries).to be_nil
+        end
+      end
+
+      context "when the message was not queued manually" do
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/server is in development mode/)
+        end
+
+        it "sets the message status to Held" do
+          service.call
+          expect(message.reload.status).to eq "Held"
+        end
+
+        it "creates a Held delivery" do
+          service.call
+          delivery = message.deliveries.last
+          expect(delivery).to have_attributes(status: "Held", details: /Server is in development mode/i)
+        end
+
+        it "removes the queued message" do
+          service.call
+          expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        end
+      end
+    end
+
+    context "when there are no other impediments" do
+      it "increments the live stats" do
+        expect { service.call }.to change { server.message_db.live_stats.total(60) }.from(0).to(1)
+      end
+
+      context "when there is an IP address assigned to the queued message" do
+        let(:ip) { create(:ip_address) }
+        let(:queued_message) { create(:queued_message, :locked, message: message, ip_address: ip) }
+
+        it "sends the message to the SMTP sender with the IP" do
+          service.call
+          expect(Postal::SMTPSender).to have_received(:new).with(message.recipient_domain, ip)
+        end
+      end
+
+      context "when there is no IP address assigned to the queued message" do
+        it "sends the message to the SMTP sender without an IP" do
+          service.call
+          expect(Postal::SMTPSender).to have_received(:new).with(message.recipient_domain, nil)
+        end
+      end
+
+      context "when the message hard fails" do
+        before do
+          send_result.type = "HardFail"
+        end
+
+        context "when the recipient has got no hard fails in the last 24 hours" do
+          it "does not add to the suppression list" do
+            service.call
+            expect(server.message_db.suppression_list.all_with_pagination(1)[:total]).to eq 0
+          end
+        end
+
+        context "when the recipient has more than one hard fail in the last 24 hours" do
+          before do
+            2.times do
+              MessageFactory.outgoing(server, domain: domain, credential: credential) do |msg|
+                msg.status = "HardFail"
+              end
+            end
+          end
+
+          it "logs" do
+            service.call
+            expect(logger).to have_logged(/added #{message.rcpt_to} to suppression list because 2 hard fails in 24 hours/i)
+          end
+
+          it "adds the recipient to the suppression list" do
+            service.call
+            entry = server.message_db.suppression_list.get(:recipient, message.rcpt_to)
+            expect(entry).to match hash_including(
+              "address" => message.rcpt_to,
+              "type" => "recipient",
+              "reason" => "too many hard fails"
+            )
+          end
+        end
+      end
+
+      context "when the message is sent manually and the recipient is on the suppression list" do
+        let(:queued_message) { create(:queued_message, :locked, message: message, manual: true) }
+
+        before do
+          server.message_db.suppression_list.add(:recipient, message.rcpt_to, reason: "testing")
+        end
+
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/removed #{message.rcpt_to} from suppression list/)
+        end
+
+        it "removes them from the suppression list" do
+          service.call
+          expect(server.message_db.suppression_list.get(:recipient, message.rcpt_to)).to be_nil
+        end
+
+        it "adds the details to the result details" do
+          service.call
+          expect(send_result.details).to include("Recipient removed from suppression list")
+        end
+      end
+
+      it "creates a delivery with the appropriate details" do
+        send_result.details = "Sent successfully to mx.example.com"
+        service.call
+        delivery = message.deliveries.last
+        expect(delivery).to have_attributes(status: "Sent", details: "Sent successfully to mx.example.com")
+      end
+
+      context "if the message should be retried" do
+        before do
+          send_result.type = "SoftFail"
+          send_result.retry = true
+        end
+
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/message requeued for trying later/)
+        end
+
+        it "sets the message status to SoftFail" do
+          service.call
+          expect(message.reload.status).to eq "SoftFail"
+        end
+
+        it "updates the retry time on the queued message" do
+          Timecop.freeze do
+            retry_time = 5.minutes.from_now.change(usec: 0)
+            service.call
+            expect(queued_message.reload.retry_after).to eq retry_time
+          end
+        end
+      end
+
+      context "if the message should not be retried" do
+        it "logs" do
+          service.call
+          expect(logger).to have_logged(/message processing complete/)
+        end
+
+        it "sets the message status to Sent" do
+          service.call
+          expect(message.reload.status).to eq "Sent"
+        end
+
+        it "removes the queued message" do
+          service.call
+          expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+        end
+      end
+    end
+  end
+end

spec/app/services/unqueue_message_service_spec.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe UnqueueMessageService do
+  let(:server) { create(:server) }
+  let(:logger) { TestLogger.new }
+  let(:queued_message) { create(:queued_message, server: server) }
+  subject(:service) { described_class.new(queued_message: queued_message, logger: logger) }
+
+  describe "#call" do
+    context "when the backend message does not exist" do
+      it "deletes the queued message" do
+        service.call
+        expect(logger).to have_logged(/unqueue because backend message has been removed/)
+        expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound)
+      end
+    end
+
+    context "when the message is not ready for processing" do
+      let(:message) { MessageFactory.outgoing(server) }
+      let(:queued_message) { create(:queued_message, :retry_in_future, message: message) }
+
+      it "does not do anything" do
+        service.call
+        expect(logger).to have_logged(/skipping because message isn't ready for processing/)
+      end
+    end
+
+    context "when there are other messages to batch with this one" do
+      context "when the backend message of a sub-message has been removed" do
+        it "removes the queued message for that message"
+      end
+    end
+  end
+end

spec/app/services/webhook_delivery_service_spec.rb

@@ -3,7 +3,7 @@
 require "rails_helper"
 
 RSpec.describe WebhookDeliveryService do
-  let(:server) { GLOBAL_SERVER }
+  let(:server) { create(:server) }
   let(:webhook) { create(:webhook, server: server) }
   let(:webhook_request) { create(:webhook_request, :locked, webhook: webhook) }
 
@@ -16,10 +16,6 @@ RSpec.describe WebhookDeliveryService do
     stub_request(:post, webhook.url).to_return(status: response_status, body: response_body)
   end
 
-  after do
-    server.message_db.provisioner.clean
-  end
-
   describe "#call" do
     it "sends a request to the webhook's url" do
       service.call

spec/factories/address_endpoint_factory.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :address_endpoint do
+    server
+    sequence(:address) { |n| "test#{n}@example.com" }
+  end
+end

spec/factories/queued_message_factory.rb

@@ -27,14 +27,33 @@
 #
 FactoryBot.define do
   factory :queued_message do
-    server
-    message_id { 1234 }
     domain { "example.com" }
-    batch_key { nil }
+
+    transient do
+      message { nil }
+    end
+
+    after(:build) do |message, evaluator|
+      if evaluator.message
+        message.server = evaluator.message.server
+        message.message_id = evaluator.message.id
+        message.batch_key = evaluator.message.batch_key
+        message.domain = evaluator.message.recipient_domain
+        message.route_id = evaluator.message.route_id
+      else
+        message.server ||= create(:server)
+        message.message_id ||= 0
+      end
+    end
 
     trait :locked do
       locked_by { "worker1" }
       locked_at { 5.minutes.ago }
     end
+
+    trait :retry_in_future do
+      attempts { 2 }
+      retry_after { 1.hour.from_now }
+    end
   end
 end

spec/factories/server_factory.rb

@@ -53,5 +53,10 @@ FactoryBot.define do
     trait :suspended do
       suspended_at { Time.current }
     end
+
+    trait :exceeded_send_limit do
+      send_limit_approaching_at { 5.minutes.ago }
+      send_limit_exceeded_at { 1.minute.ago }
+    end
   end
 end

spec/factories/smtp_endpoint_factory.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :smtp_endpoint do
+    server
+    name { "Example SMTP Endpoint" }
+    hostname { "example.com" }
+    ssl_mode { "None" }
+    port { 25 }
+  end
+end

spec/helpers/message_db_mocking.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module GlobalMessageDB
+
+  class << self
+
+    def find_or_create
+      return @db if @db
+
+      @db = Postal::MessageDB::Database.new(1, 1, database_name: "postal-test-message-db")
+      @db.provisioner.provision
+    end
+
+    def exists?
+      !@db.nil?
+    end
+
+  end
+
+end
+
+RSpec.configure do |config|
+  config.before(:example) do
+    @mocked_message_dbs = []
+    allow_any_instance_of(Server).to receive(:message_db).and_wrap_original do |m|
+      GlobalMessageDB.find_or_create
+
+      message_db = m.call
+      @mocked_message_dbs << message_db
+      allow(message_db).to receive(:database_name).and_return("postal-test-message-db")
+      message_db
+    end
+  end
+
+  config.after(:example) do
+    if GlobalMessageDB.exists? && @mocked_message_dbs.present?
+      GlobalMessageDB.find_or_create.provisioner.clean
+      @mocked_message_dbs = []
+    end
+  end
+end

spec/helpers/message_factory.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+# This class can be used to generate a message which can be used for the purposes of
+# testing within the given server.
+class MessageFactory
+
+  def initialize(server)
+    @server = server
+  end
+
+  def incoming(route: nil, &block)
+    @message = @server.message_db.new_message
+    @message.scope = "incoming"
+    @message.rcpt_to = "test@example.com"
+    @message.mail_from = "john@example.com"
+
+    if route
+      @message.rcpt_to = route.description
+      @message.route_id = route.id
+    end
+
+    create_message(&block)
+  end
+
+  def outgoing(domain: nil, credential: nil, &block)
+    @message = @server.message_db.new_message
+    @message.scope = "outgoing"
+    @message.rcpt_to = "john@example.com"
+    @message.mail_from = "test@example.com"
+
+    if domain
+      @message.mail_from = "test@#{domain.name}"
+      @message.domain_id = domain.id
+    end
+
+    if credential
+      @message.credential_id = credential.id
+    end
+
+    create_message(&block)
+  end
+
+  class << self
+
+    def incoming(server, **kwargs, &block)
+      new(server).incoming(**kwargs, &block)
+    end
+
+    def outgoing(server, **kwargs, &block)
+      new(server).outgoing(**kwargs, &block)
+    end
+
+  end
+
+  private
+
+  def create_message
+    mail = create_mail(@message.rcpt_to, @message.mail_from)
+
+    if block_given?
+      yield @message, mail
+    end
+
+    @message.raw_message = mail.to_s
+    @message.save(queue_on_create: false)
+    @message
+  end
+
+  def create_mail(to, from)
+    mail = Mail.new
+    mail.to = to
+    mail.from = from
+    mail.subject = "An example message"
+    mail.body = "Hello world!"
+    mail
+  end
+
+end

spec/lib/postal/message_db/database_spec.rb

@@ -4,7 +4,8 @@ require "rails_helper"
 
 describe Postal::MessageDB::Database do
   context "when provisioned" do
-    subject(:database) { GLOBAL_SERVER.message_db }
+    let(:server) { create(:server) }
+    subject(:database) { server.message_db }
 
     it "should be a message db" do
       expect(database).to be_a Postal::MessageDB::Database

spec/lib/postal/message_parser_spec.rb

@@ -3,25 +3,23 @@
 require "rails_helper"
 
 describe Postal::MessageParser do
+  let(:server) { create(:server) }
+
   it "should not do anything when there are no tracking domains" do
-    with_global_server do |server|
-      expect(server.track_domains.size).to eq 0
-      message = create_plain_text_message(server, "Hello world!", "test@example.com")
-      parser = Postal::MessageParser.new(message)
-      expect(parser.actioned?).to be false
-      expect(parser.tracked_links).to eq 0
-      expect(parser.tracked_images).to eq 0
-    end
+    expect(server.track_domains.size).to eq 0
+    message = create_plain_text_message(server, "Hello world!", "test@example.com")
+    parser = Postal::MessageParser.new(message)
+    expect(parser.actioned?).to be false
+    expect(parser.tracked_links).to eq 0
+    expect(parser.tracked_images).to eq 0
   end
 
   it "should replace links in messages" do
-    with_global_server do |server|
-      message = create_plain_text_message(server, "Hello world! http://github.com/atech/postal", "test@example.com")
-      create(:track_domain, server: server, domain: message.domain)
-      parser = Postal::MessageParser.new(message)
-      expect(parser.actioned?).to be true
-      expect(parser.new_body).to match(/^Hello world! https:\/\/click\.#{message.domain.name}/)
-      expect(parser.tracked_links).to eq 1
-    end
+    message = create_plain_text_message(server, "Hello world! http://github.com/atech/postal", "test@example.com")
+    create(:track_domain, server: server, domain: message.domain)
+    parser = Postal::MessageParser.new(message)
+    expect(parser.actioned?).to be true
+    expect(parser.new_body).to match(/^Hello world! https:\/\/click\.#{message.domain.name}/)
+    expect(parser.tracked_links).to eq 1
   end
 end

spec/lib/postal/smtp_server/client/finished_spec.rb

@@ -7,7 +7,7 @@ module Postal
 
     describe Client do
       let(:ip_address) { "1.2.3.4" }
-      let(:server) { GLOBAL_SERVER } # We'll use the global server instance for this
+      let(:server) { create(:server) }
       subject(:client) { described_class.new(ip_address) }
 
       let(:credential) { create(:credential, server: server, type: "SMTP") }
@@ -22,10 +22,6 @@ module Postal
         client.handle("RCPT TO: #{rcpt_to}")
       end
 
-      after do
-        server.message_db.provisioner.clean
-      end
-
       describe "when finished sending data" do
         context "when the data is larger than the maximum message size" do
           it "returns an error and resets the state" do

spec/rails_helper.rb

@@ -14,8 +14,10 @@ DatabaseCleaner.allow_remote_database_url = true
 ActiveRecord::Base.logger = Logger.new("/dev/null")
 
 Dir[File.expand_path("factories/*.rb", __dir__)].each { |f| require f }
+Dir[File.expand_path("helpers/**/*.rb", __dir__)].each { |f| require f }
 
 ActiveRecord::Migration.maintain_test_schema!
+
 RSpec.configure do |config|
   config.use_transactional_fixtures = true
   config.infer_spec_type_from_file_location!
@@ -25,29 +27,9 @@ RSpec.configure do |config|
   config.before(:suite) do
     # Test that the factories are working as they should and then clean up before getting started on
     # the rest of the suite.
-    begin
-      DatabaseCleaner.start
-      FactoryBot.lint
-    ensure
-      DatabaseCleaner.clean
-    end
-
-    # We're going to create a global server that can be used by any tests.
-    # Because the mail databases don't use any transactions, all data left in the
-    # database will be left there unless removed.
     DatabaseCleaner.start
-
-    # rubocop:disable Lint/ConstantDefinitionInBlock
-    GLOBAL_SERVER = FactoryBot.create(:server, provision_database: true)
-    # rubocop:enable Lint/ConstantDefinitionInBlock
-  end
-
-  config.after(:suite) do
-    # Remove the global server after the suite has finished running and then
-    # clean the database in case it left anything lying around.
-    if defined?(GLOBAL_SERVER)
-      GLOBAL_SERVER.destroy
-      DatabaseCleaner.clean
-    end
+    FactoryBot.lint
+  ensure
+    DatabaseCleaner.clean
   end
 end