chore(main): release 3.3.1 (#2890)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

.github/workflows/ci.yml

@@ -63,9 +63,14 @@ jobs:
     name: Release (branch)
     runs-on: ubuntu-latest
     needs: [build]
-    if: startsWith(github.ref, 'refs/heads/')
+    if: >-
+      startsWith(github.ref, 'refs/heads/') &&
+      startsWith(github.ref, 'refs/heads/release-please-') != true &&
+      startsWith(github.ref, 'refs/heads/dependabot/') != true
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
       - uses: docker/setup-qemu-action@v3
       - uses: docker/setup-buildx-action@v2
       - uses: docker/login-action@v2
@@ -73,22 +78,35 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - id: tag
+      - id: info
         run: |
-          TAG="${GITHUB_REF#refs/heads/}"
-          if [ -z "$TAG" ]; then exit 1; fi
-          if [[ $TAG == "main" ]]; then TAG="latest"; else TAG="branch-${TAG}"; fi
-          echo "tag=${TAG}" >> $GITHUB_OUTPUT
+          IMAGE=ghcr.io/postalserver/postal
+
+          REF="${GITHUB_REF#refs/heads/}"
+          if [ -z "$REF" ]; then exit 1; fi
+
+          VER="$(git describe --tags 2>/dev/null)"
+          echo "version=${VER}" >> "$GITHUB_OUTPUT"
+          echo "branch=${REF}" >> "$GITHUB_OUTPUT"
+
+          echo 'tags<<EOF' >> "$GITHUB_OUTPUT"
+          if [[ "$REF" == "main" ]]; then
+            echo "${IMAGE}:latest" >> "$GITHUB_OUTPUT"
+          else
+            echo "${IMAGE}:branch-${REF}" >> "$GITHUB_OUTPUT"
+          fi
+          echo 'EOF' >> "$GITHUB_OUTPUT"
       - uses: docker/build-push-action@v4
         with:
           push: true
-          tags: |
-            ghcr.io/postalserver/postal:${{ steps.tag.outputs.tag }}
-            ghcr.io/postalserver/postal:commit-${{ github.sha }}
+          tags: ${{ steps.info.outputs.tags }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
           target: full
           platforms: linux/amd64
+          build-args: |
+            VERSION=${{ steps.info.outputs.version }}
+            BRANCH=${{ steps.info.outputs.branch }}
 
   publish-image:
     name: Publish Image

.gitignore

@@ -29,6 +29,7 @@ vendor/bundle
 
 Procfile.local
 VERSION
+BRANCH
 
 .rubocop-https*
 .env*

.release-please-manifest.json

@@ -1,3 +1,3 @@
 {
-  ".": "3.2.0"
+  ".": "3.3.1"
 }

CHANGELOG.md

@@ -2,6 +2,75 @@
 
 This file contains all the latest changes and updates to Postal.
 
+## [3.3.1](https://github.com/postalserver/postal/compare/3.3.0...3.3.1) (2024-03-21)
+
+
+### Bug Fixes
+
+* **smtp-sender:** ensure relays without a host are excluded ([3a56ec8](https://github.com/postalserver/postal/commit/3a56ec8a74950e0162d98f1af5f58a67a82d6455))
+* **smtp-sender:** fixes `SMTPSender.smtp_relays` ([b3264b9](https://github.com/postalserver/postal/commit/b3264b942776e254d3c351c94c435d172a514e18))
+
+
+### Miscellaneous Chores
+
+* **container:** add the branch name to the container ([bee5098](https://github.com/postalserver/postal/commit/bee509832edc151d97fe5bfc48c4973452873fc8))
+* **github-actions:** don't generate commit- tags ([d65bbe0](https://github.com/postalserver/postal/commit/d65bbe0579037c5df962a18134bc007f5159d7e5))
+* **github-actions:** don't run for dependabot or release-please PRs and fetch whole repo ([adaf2b0](https://github.com/postalserver/postal/commit/adaf2b07502e9ed91290873ad8465051c6fd814f))
+* **github-actions:** include a version string on branch-*/latest images ([64bc7dc](https://github.com/postalserver/postal/commit/64bc7dcf7c0a8e006ab6eb6e8b4a52ad5e7e6528))
+* **ui:** display branch in footer if present ([1823617](https://github.com/postalserver/postal/commit/18236171ebc398c157f2e61b15c7df9f91205284))
+
+
+### Code Refactoring
+
+* remove moonrope but maintain legacy API actions ([#2889](https://github.com/postalserver/postal/issues/2889)) ([4d9654d](https://github.com/postalserver/postal/commit/4d9654dac47d59c760be96388d0421de74d3e6ac))
+
+## [3.3.0](https://github.com/postalserver/postal/compare/3.2.2...3.3.0) (2024-03-18)
+
+
+### Features
+
+* **prometheus:** add `postal_message_queue_latency` metric ([ee8d829](https://github.com/postalserver/postal/commit/ee8d829a854f91e476167869cafe35c2d37bb314))
+* **worker:** allow number of threads to be configured ([7e2accc](https://github.com/postalserver/postal/commit/7e2acccd1ebd80750a3ebdb96cb5c36b5263cc24))
+* **worker:** scale connection pool with worker threads ([ea542a0](https://github.com/postalserver/postal/commit/ea542a0694b3465b04fd3ebc439837df414deb1e))
+
+
+### Bug Fixes
+
+* **message-dequeuer:** ability to disable batching ([4fcb9e9](https://github.com/postalserver/postal/commit/4fcb9e9a2e34be5aa4bdf13f0529f40e564b72b4))
+
+
+### Miscellaneous Chores
+
+* **config-docs:** update docs for latest oidc defaults ([364eba6](https://github.com/postalserver/postal/commit/364eba6c5fce2f08a36489f42856ad5024a2062c))
+* **config-docs:** update proxy protocol to mention v1 ([45dd8aa](https://github.com/postalserver/postal/commit/45dd8aaac56f15481cb7bf9081401cb28dc1e707))
+
+## [3.2.2](https://github.com/postalserver/postal/compare/3.2.1...3.2.2) (2024-03-14)
+
+
+### Bug Fixes
+
+* don't use authentication on org & server deletion ([be45652](https://github.com/postalserver/postal/commit/be456523dd3aacb5c3eb45c9261da97ebffe603c))
+* **smtp-server:** fixes proxy protocol ([9240612](https://github.com/postalserver/postal/commit/92406129cfcf1a06499a6f5aa18c73f1d6195793))
+
+
+### Miscellaneous Chores
+
+* allow config location message to be suppressed ([f760cdb](https://github.com/postalserver/postal/commit/f760cdb5a1d53e9c30ee495d129cbf12603a3cbd))
+* hide further config messages ([1c67f72](https://github.com/postalserver/postal/commit/1c67f72209c93404d7024ce3d15f6f54f2d707c4))
+* suppress config location on default-dkim-record ([aa76aae](https://github.com/postalserver/postal/commit/aa76aae2322af41af1bd60cfe1d69a11ac76324e))
+
+
+### Tests
+
+* add tests for the legacy API ([3d208d6](https://github.com/postalserver/postal/commit/3d208d632f4fc8a4adbfdb2bf4b377271eae6692))
+
+## [3.2.1](https://github.com/postalserver/postal/compare/3.2.0...3.2.1) (2024-03-13)
+
+
+### Bug Fixes
+
+* fixes `postal default-dkim-record` ([58dddeb](https://github.com/postalserver/postal/commit/58dddebeb81dc6fab945d2b10a91588eddc471c2))
+
 ## [3.2.0](https://github.com/postalserver/postal/compare/3.1.1...3.2.0) (2024-03-13)
 
 

Dockerfile

@@ -3,7 +3,7 @@ FROM ruby:3.2.2-bullseye AS base
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN apt-get update \
   && apt-get install -y --no-install-recommends \
-  software-properties-common dirmngr apt-transport-https \
+      software-properties-common dirmngr apt-transport-https \
   && (curl -sL https://deb.nodesource.com/setup_20.x | bash -) \
   && rm -rf /var/lib/apt/lists/*
 
@@ -43,8 +43,10 @@ COPY ./docker/wait-for.sh /docker-entrypoint.sh
 COPY --chown=postal . .
 
 # Export the version
-ARG VERSION=unspecified
-RUN echo $VERSION > VERSION
+ARG VERSION=null
+ARG BRANCH=null
+RUN echo $VERSION > VERSION \
+  && echo $BRANCH > BRANCH
 
 # Set paths for when running in a container
 ENV POSTAL_CONFIG_FILE_PATH=/config/postal.yml

Gemfile

@@ -18,7 +18,6 @@ gem "kaminari"
 gem "klogger-logger"
 gem "konfig-config", "~> 3.0"
 gem "mail"
-gem "moonrope"
 gem "mysql2"
 gem "nifty-utils"
 gem "nilify_blanks"

Gemfile.lock

@@ -103,7 +103,6 @@ GEM
       database_cleaner-core (~> 2.0.0)
     database_cleaner-core (2.0.1)
     date (3.3.4)
-    deep_merge (1.2.2)
     diff-lcs (1.5.0)
     domain_name (0.6.20240107)
     dotenv (3.0.2)
@@ -184,10 +183,6 @@ GEM
     mini_mime (1.1.5)
     mini_portile2 (2.8.5)
     minitest (5.22.2)
-    moonrope (2.0.2)
-      deep_merge (~> 1.0)
-      json
-      rack (>= 1.4)
     mysql2 (0.5.6)
     net-http (0.4.1)
       uri
@@ -423,7 +418,6 @@ DEPENDENCIES
   klogger-logger
   konfig-config (~> 3.0)
   mail
-  moonrope
   mysql2
   nifty-utils
   nilify_blanks

api/authenticator.rb

@@ -1,31 +0,0 @@
-# frozen_string_literal: true
-
-authenticator :server do
-  friendly_name "Server Authenticator"
-  header "X-Server-API-Key", "The API token for a server that you wish to authenticate with.", example: "f29a45f0d4e1744ebaee"
-  error "InvalidServerAPIKey", "The API token provided in X-Server-API-Key was not valid.", attributes: { token: "The token that was looked up" }
-  error "ServerSuspended", "The mail server has been suspended"
-  lookup do
-    if key = request.headers["X-Server-API-Key"]
-      if credential = Credential.where(type: "API", key: key).first
-        if credential.server.suspended?
-          error "ServerSuspended"
-        else
-          credential.use
-          credential
-        end
-      else
-        error "InvalidServerAPIKey", token: key
-      end
-    end
-  end
-  rule :default, "AccessDenied", "Must be authenticated as a server." do
-    identity.is_a?(Credential)
-  end
-end
-
-authenticator :anonymous do
-  rule :default, "MustNotBeAuthenticated", "Must not be authenticated." do
-    identity.nil?
-  end
-end

api/controllers/messages_api_controller.rb

@@ -1,41 +0,0 @@
-# frozen_string_literal: true
-
-controller :messages do
-  friendly_name "Messages API"
-  description "This API allows you to access message details"
-  authenticator :server
-
-  action :message do
-    title "Return message details"
-    description "Returns all details about a message"
-    param :id, "The ID of the message", type: Integer, required: true
-    returns Hash, structure: :message, structure_opts: { paramable: { expansions: false } }
-    error "MessageNotFound", "No message found matching provided ID", attributes: { id: "The ID of the message" }
-    action do
-      begin
-        message = identity.server.message(params.id)
-      rescue Postal::MessageDB::Message::NotFound
-        error "MessageNotFound", id: params.id
-      end
-      structure :message, message, return: true
-    end
-  end
-
-  action :deliveries do
-    title "Return deliveries for a message"
-    description "Returns an array of deliveries which have been attempted for this message"
-    param :id, "The ID of the message", type: Integer, required: true
-    returns Array, structure: :delivery, structure_opts: { full: true }
-    error "MessageNotFound", "No message found matching provided ID", attributes: { id: "The ID of the message" }
-    action do
-      begin
-        message = identity.server.message(params.id)
-      rescue Postal::MessageDB::Message::NotFound
-        error "MessageNotFound", id: params.id
-      end
-      message.deliveries.map do |d|
-        structure :delivery, d
-      end
-    end
-  end
-end

api/controllers/send_api_controller.rb

@@ -1,112 +0,0 @@
-# frozen_string_literal: true
-
-controller :send do
-  friendly_name "Send API"
-  description "This API allows you to send messages"
-  authenticator :server
-
-  action :message do
-    title "Send a message"
-    description "This action allows you to send a message by providing the appropriate options"
-    # Acceptable Parameters
-    param :to, "The e-mail addresses of the recipients (max 50)", type: Array
-    param :cc, "The e-mail addresses of any CC contacts (max 50)", type: Array
-    param :bcc, "The e-mail addresses of any BCC contacts (max 50)", type: Array
-    param :from, "The e-mail address for the From header", type: String
-    param :sender, "The e-mail address for the Sender header", type: String
-    param :subject, "The subject of the e-mail", type: String
-    param :tag, "The tag of the e-mail", type: String
-    param :reply_to, "Set the reply-to address for the mail", type: String
-    param :plain_body, "The plain text body of the e-mail", type: String
-    param :html_body, "The HTML body of the e-mail", type: String
-    param :attachments, "An array of attachments for this e-mail", type: Array
-    param :headers, "A hash of additional headers", type: Hash
-    param :bounce, "Is this message a bounce?", type: :boolean
-    # Errors
-    error "ValidationError", "The provided data was not sufficient to send an email", attributes: { errors: "A hash of error details" }
-    error "NoRecipients", "There are no recipients defined to receive this message"
-    error "NoContent", "There is no content defined for this e-mail"
-    error "TooManyToAddresses", "The maximum number of To addresses has been reached (maximum 50)"
-    error "TooManyCCAddresses", "The maximum number of CC addresses has been reached (maximum 50)"
-    error "TooManyBCCAddresses", "The maximum number of BCC addresses has been reached (maximum 50)"
-    error "FromAddressMissing", "The From address is missing and is required"
-    error "UnauthenticatedFromAddress", "The From address is not authorised to send mail from this server"
-    error "AttachmentMissingName", "An attachment is missing a name"
-    error "AttachmentMissingData", "An attachment is missing data"
-    # Return
-    returns Hash
-    # Action
-    action do
-      attributes = {}
-      attributes[:to] = params.to
-      attributes[:cc] = params.cc
-      attributes[:bcc] = params.bcc
-      attributes[:from] = params.from
-      attributes[:sender] = params.sender
-      attributes[:subject] = params.subject
-      attributes[:reply_to] = params.reply_to
-      attributes[:plain_body] = params.plain_body
-      attributes[:html_body] = params.html_body
-      attributes[:bounce] = params.bounce ? true : false
-      attributes[:tag] = params.tag
-      attributes[:custom_headers] = params.headers
-      attributes[:attachments] = []
-      (params.attachments || []).each do |attachment|
-        next unless attachment.is_a?(Hash)
-
-        attributes[:attachments] << { name: attachment["name"], content_type: attachment["content_type"], data: attachment["data"], base64: true }
-      end
-      message = OutgoingMessagePrototype.new(identity.server, request.ip, "api", attributes)
-      message.credential = identity
-      if message.valid?
-        result = message.create_messages
-        { message_id: message.message_id, messages: result }
-      else
-        error message.errors.first
-      end
-    end
-  end
-
-  action :raw do
-    title "Send a raw RFC2822 message"
-    description "This action allows you to send us a raw RFC2822 formatted message along with the recipients that it should be sent to. This is similar to sending a message through our SMTP service."
-    param :mail_from, "The address that should be logged as sending the message", type: String, required: true
-    param :rcpt_to, "The addresses this message should be sent to", type: Array, required: true
-    param :data, "A base64 encoded RFC2822 message to send", type: String, required: true
-    param :bounce, "Is this message a bounce?", type: :boolean
-    returns Hash
-    error "UnauthenticatedFromAddress", "The From address is not authorised to send mail from this server"
-    action do
-      # Decode the raw message
-      raw_message = Base64.decode64(params.data)
-
-      # Parse through mail to get the from/sender headers
-      mail = Mail.new(raw_message.split("\r\n\r\n", 2).first)
-      from_headers = { "from" => mail.from, "sender" => mail.sender }
-      authenticated_domain = identity.server.find_authenticated_domain_from_headers(from_headers)
-
-      # If we're not authenticated, don't continue
-      if authenticated_domain.nil?
-        error "UnauthenticatedFromAddress"
-      end
-
-      # Store the result ready to return
-      result = { message_id: nil, messages: {} }
-      params.rcpt_to.uniq.each do |rcpt_to|
-        message = identity.server.message_db.new_message
-        message.rcpt_to = rcpt_to
-        message.mail_from = params.mail_from
-        message.raw_message = raw_message
-        message.received_with_ssl = true
-        message.scope = "outgoing"
-        message.domain_id = authenticated_domain.id
-        message.credential_id = identity.id
-        message.bounce = params.bounce
-        message.save
-        result[:message_id] = message.message_id if result[:message_id].nil?
-        result[:messages][rcpt_to] = { id: message.id, token: message.token }
-      end
-      result
-    end
-  end
-end

api/structures/delivery_api_structure.rb

@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-structure :delivery do
-  basic :id
-  basic :status
-  basic :details
-  basic :output, value: proc { o.output&.strip }
-  basic :sent_with_ssl, value: proc { o.sent_with_ssl }
-  basic :log_id
-  basic :time, value: proc { o.time&.to_f }
-  basic :timestamp, value: proc { o.timestamp.to_f }
-end

api/structures/message_api_structure.rb

@@ -1,68 +0,0 @@
-# frozen_string_literal: true
-
-structure :message do
-  basic :id
-  basic :token
-
-  expansion(:status) do
-    {
-      status: o.status,
-      last_delivery_attempt: o.last_delivery_attempt&.to_f,
-      held: o.held,
-      hold_expiry: o.hold_expiry&.to_f
-    }
-  end
-
-  expansion(:details) do
-    {
-      rcpt_to: o.rcpt_to,
-      mail_from: o.mail_from,
-      subject: o.subject,
-      message_id: o.message_id,
-      timestamp: o.timestamp.to_f,
-      direction: o.scope,
-      size: o.size,
-      bounce: o.bounce,
-      bounce_for_id: o.bounce_for_id,
-      tag: o.tag,
-      received_with_ssl: o.received_with_ssl
-    }
-  end
-
-  expansion(:inspection) do
-    {
-      inspected: o.inspected,
-      spam: o.spam,
-      spam_score: o.spam_score.to_f,
-      threat: o.threat,
-      threat_details: o.threat_details
-    }
-  end
-
-  expansion(:plain_body) { o.plain_body }
-
-  expansion(:html_body) { o.html_body }
-
-  expansion(:attachments) do
-    o.attachments.map do |attachment|
-      {
-        filename: attachment.filename.to_s,
-        content_type: attachment.mime_type,
-        data: Base64.encode64(attachment.body.to_s),
-        size: attachment.body.to_s.bytesize,
-        hash: Digest::SHA1.hexdigest(attachment.body.to_s)
-      }
-    end
-  end
-
-  expansion(:headers) { o.headers }
-
-  expansion(:raw_message) { Base64.encode64(o.raw_message) }
-
-  expansion(:activity_entries) do
-    {
-      loads: o.loads,
-      clicks: o.clicks
-    }
-  end
-end

app/controllers/legacy_api/base_controller.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+module LegacyAPI
+  # The Legacy API is the Postal v1 API which existed from the start with main
+  # aim of allowing e-mails to sent over HTTP rather than SMTP. The API itself
+  # did not feature much functionality. This API was implemented using Moonrope
+  # which was a self documenting API tool, however, is now no longer maintained.
+  # In light of that, these controllers now implement the same functionality as
+  # the original Moonrope API without the actual requirement to use any of the
+  # Moonrope components.
+  #
+  # Important things to note about the API:
+  #
+  #   * Moonrope allow params to be provided as JSON in the body of the request
+  #     along with the application/json content type. It also allowed for params
+  #     to be sent in the 'params' parameter when using the
+  #     application/x-www-form-urlencoded content type. Both methods are supported.
+  #
+  #   * Authentication is performed using a X-Server-API-Key variable.
+  #
+  #   * The method used to make the request is not important. Most clients use POST
+  #     but other methods should be supported. The routing for this legacvy
+  #     API supports GET, POST, PUT and PATCH.
+  #
+  #   * The status code for responses will always be 200 OK. The actual status of
+  #     a request is determined by the value of the 'status' attribute in the
+  #     returned JSON.
+  class BaseController < ActionController::Base
+
+    skip_before_action :set_browser_id
+    skip_before_action :verify_authenticity_token
+
+    before_action :start_timer
+    before_action :authenticate_as_server
+
+    private
+
+    # The Moonrope API spec allows for parameters to be provided in the body
+    # along with the application/json content type or they can be provided,
+    # as JSON, in the 'params' parameter when used with the
+    # application/x-www-form-urlencoded content type. This legacy API needs
+    # support both options for maximum compatibility.
+    #
+    # @return [Hash]
+    def api_params
+      if request.headers["content-type"] =~ /\Aapplication\/json/
+        return params.to_unsafe_hash
+      end
+
+      if params["params"].present?
+        return JSON.parse(params["params"])
+      end
+
+      {}
+    end
+
+    # The API returns a length of time to complete a request. We'll start
+    # a timer when the request starts and then use this method to calculate
+    # the time taken to complete the request.
+    #
+    # @return [void]
+    def start_timer
+      @start_time = Time.now.to_f
+    end
+
+    # The only method available to authenticate to the legacy API is using a
+    # credential from the server itself. This method will attempt to find
+    # that credential from the X-Server-API-Key header and will set the
+    # current_credential instance variable if a token is valid. Otherwise it
+    # will render an error to halt execution.
+    #
+    # @return [void]
+    def authenticate_as_server
+      key = request.headers["X-Server-API-Key"]
+      if key.blank?
+        render_error "AccessDenied",
+                     message: "Must be authenticated as a server."
+        return
+      end
+
+      credential = Credential.where(type: "API", key: key).first
+      if credential.nil?
+        render_error "InvalidServerAPIKey",
+                     message: "The API token provided in X-Server-API-Key was not valid.",
+                     token: key
+        return
+      end
+
+      if credential.server.suspended?
+        render_error "ServerSuspended"
+        return
+      end
+
+      credential.use
+      @current_credential = credential
+    end
+
+    # Render a successful response to the client
+    #
+    # @param [Hash] data
+    # @return [void]
+    def render_success(data)
+      render json: { status: "success",
+                     time: (Time.now.to_f - @start_time).round(3),
+                     flags: {},
+                     data: data }
+    end
+
+    # Render an error response to the client
+    #
+    # @param [String] code
+    # @param [Hash] data
+    # @return [void]
+    def render_error(code, data = {})
+      render json: { status: "error",
+                     time: (Time.now.to_f - @start_time).round(3),
+                     flags: {},
+                     data: data.merge(code: code) }
+    end
+
+    # Render a parameter error response to the client
+    #
+    # @param [String] message
+    # @return [void]
+    def render_parameter_error(message)
+      render json: { status: "parameter-error",
+                     time: (Time.now.to_f - @start_time).round(3),
+                     flags: {},
+                     data: { message: message } }
+    end
+
+  end
+end

app/controllers/legacy_api/messages_controller.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module LegacyAPI
+  class MessagesController < BaseController
+
+    # Returns details about a given message
+    #
+    #   URL:            /api/v1/messages/message
+    #
+    #   Parameters:     id              => REQ: The ID of the message
+    #                   _expansions     => An array of types of details t
+    #                                      to return
+    #
+    #   Response:       A hash containing message information
+    #                   OR an error if the message does not exist.
+    #
+    def message
+      if api_params["id"].blank?
+        render_parameter_error "`id` parameter is required but is missing"
+        return
+      end
+
+      message = @current_credential.server.message(api_params["id"])
+      message_hash = { id: message.id, token: message.token }
+      expansions = api_params["_expansions"]
+
+      if expansions.include?("status")
+        message_hash[:status] = {
+          status: message.status,
+          last_delivery_attempt: message.last_delivery_attempt&.to_f,
+          held: message.held,
+          hold_expiry: message.hold_expiry&.to_f
+        }
+      end
+
+      if expansions.include?("details")
+        message_hash[:details] = {
+          rcpt_to: message.rcpt_to,
+          mail_from: message.mail_from,
+          subject: message.subject,
+          message_id: message.message_id,
+          timestamp: message.timestamp.to_f,
+          direction: message.scope,
+          size: message.size,
+          bounce: message.bounce,
+          bounce_for_id: message.bounce_for_id,
+          tag: message.tag,
+          received_with_ssl: message.received_with_ssl
+        }
+      end
+
+      if expansions.include?("inspection")
+        message_hash[:inspection] = {
+          inspected: message.inspected,
+          spam: message.spam,
+          spam_score: message.spam_score.to_f,
+          threat: message.threat,
+          threat_details: message.threat_details
+        }
+      end
+
+      if expansions.include?("plain_body")
+        message_hash[:plain_body] = message.plain_body
+      end
+
+      if expansions.include?("html_body")
+        message_hash[:html_body] = message.html_body
+      end
+
+      if expansions.include?("attachments")
+        message_hash[:attachments] = message.attachments.map do |attachment|
+          {
+            filename: attachment.filename.to_s,
+            content_type: attachment.mime_type,
+            data: Base64.encode64(attachment.body.to_s),
+            size: attachment.body.to_s.bytesize,
+            hash: Digest::SHA1.hexdigest(attachment.body.to_s)
+          }
+        end
+      end
+
+      if expansions.include?("headers")
+        message_hash[:headers] = message.headers
+      end
+
+      if expansions.include?("raw_message")
+        message_hash[:raw_message] = Base64.encode64(message.raw_message)
+      end
+
+      if expansions.include?("activity_entries")
+        message_hash[:activity_entries] = {
+          loads: message.loads,
+          clicks: message.clicks
+        }
+      end
+
+      render_success message_hash
+    rescue Postal::MessageDB::Message::NotFound
+      render_error "MessageNotFound",
+                   message: "No message found matching provided ID",
+                   id: api_params["id"]
+    end
+
+    # Returns all the deliveries for a given message
+    #
+    #   URL:            /api/v1/messages/deliveries
+    #
+    #   Parameters:     id              => REQ: The ID of the message
+    #
+    #   Response:       A array of hashes containing delivery information
+    #                   OR an error if the message does not exist.
+    #
+    def deliveries
+      if api_params["id"].blank?
+        render_parameter_error "`id` parameter is required but is missing"
+        return
+      end
+
+      message = @current_credential.server.message(api_params["id"])
+      deliveries = message.deliveries.map do |d|
+        {
+          id: d.id,
+          status: d.status,
+          details: d.details,
+          output: d.output&.strip,
+          sent_with_ssl: d.sent_with_ssl,
+          log_id: d.log_id,
+          time: d.time&.to_f,
+          timestamp: d.timestamp.to_f
+        }
+      end
+      render_success deliveries
+    rescue Postal::MessageDB::Message::NotFound
+      render_error "MessageNotFound",
+                   message: "No message found matching provided ID",
+                   id: api_params["id"]
+    end
+
+  end
+end

app/controllers/legacy_api/send_controller.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module LegacyAPI
+  class SendController < BaseController
+
+    ERROR_MESSAGES = {
+      "NoRecipients" => "There are no recipients defined to receive this message",
+      "NoContent" => "There is no content defined for this e-mail",
+      "TooManyToAddresses" => "The maximum number of To addresses has been reached (maximum 50)",
+      "TooManyCCAddresses" => "The maximum number of CC addresses has been reached (maximum 50)",
+      "TooManyBCCAddresses" => "The maximum number of BCC addresses has been reached (maximum 50)",
+      "FromAddressMissing" => "The From address is missing and is required",
+      "UnauthenticatedFromAddress" => "The From address is not authorised to send mail from this server",
+      "AttachmentMissingName" => "An attachment is missing a name",
+      "AttachmentMissingData" => "An attachment is missing data"
+    }.freeze
+
+    # Send a message with the given options
+    #
+    #   URL:            /api/v1/send/message
+    #
+    #   Parameters:     to              => REQ: An array of emails addresses
+    #                   cc              => An array of email addresses to CC
+    #                   bcc             => An array of email addresses to BCC
+    #                   from            => The name/email to send the email from
+    #                   sender          => The name/email of the 'Sender'
+    #                   reply_to        => The name/email of the 'Reply-to'
+    #                   plain_body      => The plain body
+    #                   html_body       => The HTML body
+    #                   bounce          => Is this message a bounce?
+    #                   tag             => A custom tag to add to the message
+    #                   custom_headers  => A hash of custom headers
+    #                   attachments     => An array of attachments
+    #                                      (name, content_type and data (base64))
+    #
+    #   Response:       A array of hashes containing message information
+    #                   OR an error if there is an issue sending the message
+    #
+    def message
+      attributes = {}
+      attributes[:to] = api_params["to"]
+      attributes[:cc] = api_params["cc"]
+      attributes[:bcc] = api_params["bcc"]
+      attributes[:from] = api_params["from"]
+      attributes[:sender] = api_params["sender"]
+      attributes[:subject] = api_params["subject"]
+      attributes[:reply_to] = api_params["reply_to"]
+      attributes[:plain_body] = api_params["plain_body"]
+      attributes[:html_body] = api_params["html_body"]
+      attributes[:bounce] = api_params["bounce"] ? true : false
+      attributes[:tag] = api_params["tag"]
+      attributes[:custom_headers] = api_params["headers"] if api_params["headers"]
+      attributes[:attachments] = []
+
+      (api_params["attachments"] || []).each do |attachment|
+        next unless attachment.is_a?(Hash)
+
+        attributes[:attachments] << { name: attachment["name"], content_type: attachment["content_type"], data: attachment["data"], base64: true }
+      end
+
+      message = OutgoingMessagePrototype.new(@current_credential.server, request.ip, "api", attributes)
+      message.credential = @current_credential
+      if message.valid?
+        result = message.create_messages
+        render_success message_id: message.message_id, messages: result
+      else
+        render_error message.errors.first, message: ERROR_MESSAGES[message.errors.first]
+      end
+    end
+
+    # Send a message by providing a raw message
+    #
+    #   URL:            /api/v1/send/raw
+    #
+    #   Parameters:     rcpt_to         => REQ: An array of email addresses to send
+    #                                      the message to
+    #                   mail_from       => REQ: the address to send the email from
+    #                   data            => REQ: base64-encoded mail data
+    #
+    #   Response:       A array of hashes containing message information
+    #                   OR an error if there is an issue sending the message
+    #
+    def raw
+      unless api_params["rcpt_to"].is_a?(Array)
+        render_parameter_error "`rcpt_to` parameter is required but is missing"
+        return
+      end
+
+      if api_params["mail_from"].blank?
+        render_parameter_error "`mail_from` parameter is required but is missing"
+        return
+      end
+
+      if api_params["data"].blank?
+        render_parameter_error "`data` parameter is required but is missing"
+        return
+      end
+
+      # Decode the raw message
+      raw_message = Base64.decode64(api_params["data"])
+
+      # Parse through mail to get the from/sender headers
+      mail = Mail.new(raw_message.split("\r\n\r\n", 2).first)
+      from_headers = { "from" => mail.from, "sender" => mail.sender }
+      authenticated_domain = @current_credential.server.find_authenticated_domain_from_headers(from_headers)
+
+      # If we're not authenticated, don't continue
+      if authenticated_domain.nil?
+        render_error "UnauthenticatedFromAddress"
+        return
+      end
+
+      # Store the result ready to return
+      result = { message_id: nil, messages: {} }
+      if api_params["rcpt_to"].is_a?(Array)
+        api_params["rcpt_to"].uniq.each do |rcpt_to|
+          message = @current_credential.server.message_db.new_message
+          message.rcpt_to = rcpt_to
+          message.mail_from = api_params["mail_from"]
+          message.raw_message = raw_message
+          message.received_with_ssl = true
+          message.scope = "outgoing"
+          message.domain_id = authenticated_domain.id
+          message.credential_id = @current_credential.id
+          message.bounce = api_params["bounce"] ? true : false
+          message.save
+          result[:message_id] = message.message_id if result[:message_id].nil?
+          result[:messages][rcpt_to] = { id: message.id, token: message.token }
+        end
+      end
+      render_success result
+    end
+
+  end
+end

app/controllers/organizations_controller.rb

@@ -43,10 +43,11 @@ class OrganizationsController < ApplicationController
   end
 
   def destroy
-    unless current_user.authenticate(params[:password])
+    if params[:confirm_text].blank? || params[:confirm_text].downcase.strip != organization.name.downcase.strip
       respond_to do |wants|
-        wants.html { redirect_to organization_delete_path(@organization), alert: "The password you entered was not valid. Please check and try again." }
-        wants.json { render json: { alert: "The password you entered was invalid. Please check and try again." } }
+        alert_text = "The text you entered does not match the organization name. Please check and try again."
+        wants.html { redirect_to organization_delete_path(@organization), alert: alert_text }
+        wants.json { render json: { alert: alert_text } }
       end
       return
     end

app/controllers/servers_controller.rb

@@ -65,17 +65,15 @@ class ServersController < ApplicationController
   end
 
   def destroy
-    unless current_user.authenticate(params[:password])
+    if params[:confirm_text].blank? || params[:confirm_text].downcase.strip != @server.name.downcase.strip
       respond_to do |wants|
-        wants.html do
-          redirect_to [:delete, organization, @server], alert: "The password you entered was not valid. Please check and try again."
-        end
-        wants.json do
-          render json: { alert: "The password you entere was invalid. Please check and try again" }
-        end
+        alert_text = "The text you entered does not match the server name. Please check and try again."
+        wants.html { redirect_to organization_delete_path(@organization), alert: alert_text }
+        wants.json { render json: { alert: alert_text } }
       end
       return
     end
+
     @server.soft_destroy
     redirect_to_with_json organization_root_path(organization), notice: "#{@server.name} has been deleted successfully"
   end

app/helpers/application_helper.rb

@@ -100,4 +100,11 @@ module ApplicationHelper
     end.html_safe
   end
 
+  def postal_version_string
+    string = Postal.version
+    string += " (#{Postal.branch})" if Postal.branch &&
+                                       Postal.branch != "main"
+    string
+  end
+
 end

app/lib/message_dequeuer/initial_processor.rb

@@ -3,6 +3,8 @@
 module MessageDequeuer
   class InitialProcessor < Base
 
+    include HasPrometheusMetrics
+
     attr_accessor :send_result
 
     def process
@@ -10,6 +12,7 @@ module MessageDequeuer
         logger.info "starting message unqueue"
         begin
           catch_stops do
+            increment_dequeue_metric
             check_message_exists
             check_message_is_ready
             find_other_messages_for_batch
@@ -17,7 +20,7 @@ module MessageDequeuer
             # Process the original message and then all of those
             # found for batching.
             process_message(@queued_message)
-            @other_messages.each { |message| process_message(message) }
+            @other_messages&.each { |message| process_message(message) }
           end
         ensure
           @state.finished
@@ -28,6 +31,13 @@ module MessageDequeuer
 
     private
 
+    def increment_dequeue_metric
+      time_in_queue = Time.now.to_f - @queued_message.created_at.to_f
+      log "queue latency is #{time_in_queue}s"
+      observe_prometheus_histogram :postal_message_queue_latency,
+                                   time_in_queue
+    end
+
     def check_message_exists
       return if @queued_message.message
 
@@ -45,6 +55,8 @@ module MessageDequeuer
     end
 
     def find_other_messages_for_batch
+      return unless Postal::Config.postal.batch_queued_messages?
+
       @other_messages = @queued_message.batchable_messages(100)
       log "found #{@other_messages.size} associated messages to process at the same time", batch_key: @queued_message.batch_key
     rescue StandardError

app/lib/signer.rb

@@ -1,66 +0,0 @@
-# frozen_string_literal: true
-
-require "base64"
-
-class Signer
-
-  # Create a new Signer
-  #
-  # @param [OpenSSL::PKey::RSA] private_key The private key to use for signing
-  # @return [Signer]
-  def initialize(private_key)
-    @private_key = private_key
-  end
-
-  # Return the private key
-  #
-  # @return [OpenSSL::PKey::RSA]
-  attr_reader :private_key
-
-  # Return the public key for the private key
-  #
-  # @return [OpenSSL::PKey::RSA]
-  def public_key
-    @private_key.public_key
-  end
-
-  # Sign the given data
-  #
-  # @param [String] data The data to sign
-  # @return [String] The signature
-  def sign(data)
-    private_key.sign(OpenSSL::Digest.new("SHA256"), data)
-  end
-
-  # Sign the given data and return a Base64-encoded signature
-  #
-  # @param [String] data The data to sign
-  # @return [String] The Base64-encoded signature
-  def sign64(data)
-    Base64.strict_encode64(sign(data))
-  end
-
-  # Return a JWK for the private key
-  #
-  # @return [JWT::JWK] The JWK
-  def jwk
-    @jwk ||= JWT::JWK.new(private_key, { use: "sig", alg: "RS256" })
-  end
-
-  # Sign the given data using SHA1 (for legacy use)
-  #
-  # @param [String] data The data to sign
-  # @return [String] The signature
-  def sha1_sign(data)
-    private_key.sign(OpenSSL::Digest.new("SHA1"), data)
-  end
-
-  # Sign the given data using SHA1 (for legacy use) and return a Base64-encoded string
-  #
-  # @param [String] data The data to sign
-  # @return [String] The signature
-  def sha1_sign64(data)
-    Base64.strict_encode64(sha1_sign(data))
-  end
-
-end

app/lib/smtp_server/client.rb

@@ -116,18 +116,19 @@ module SMTPServer
     private
 
     def proxy(data)
+      # inet-protocol, client-ip, proxy-ip, client-port, proxy-port
       if m = data.match(/\APROXY (.+) (.+) (.+) (.+) (.+)\z/)
         @ip_address = m[2]
         check_ip_address
         @state = :welcome
         logger&.debug "\e[35mClient identified as #{@ip_address}\e[0m"
         increment_command_count("PROXY")
-        "220 #{Postal::Config.postal.smtp_hostname} ESMTP Postal/#{id}"
-      else
-        @finished = true
-        increment_error_count("proxy-error")
-        "502 Proxy Error"
+        return "220 #{Postal::Config.postal.smtp_hostname} ESMTP Postal/#{trace_id}"
       end
+
+      @finished = true
+      increment_error_count("proxy-error")
+      "502 Proxy Error"
     end
 
     def quit

app/lib/worker/process.rb

@@ -45,7 +45,9 @@ module Worker
     ].freeze
 
     # @param [Integer] thread_count The number of worker threads to run in this process
-    def initialize(thread_count: 2, work_sleep_time: 5, task_sleep_time: 60)
+    def initialize(thread_count: Postal::Config.worker.threads,
+                   work_sleep_time: 5,
+                   task_sleep_time: 60)
       @thread_count = thread_count
       @exit_pipe_read, @exit_pipe_write = IO.pipe
       @work_sleep_time = work_sleep_time
@@ -58,6 +60,7 @@ module Worker
     def run
       logger.tagged(component: "worker") do
         setup_traps
+        ensure_connection_pool_size_is_suitable
         start_work_threads
         start_tasks_thread
         wait_for_threads
@@ -94,6 +97,23 @@ module Worker
       @exit_pipe_read.wait_readable(wait_time) ? true : false
     end
 
+    # Ensure that the connection pool is big enough for the number of threads
+    # configured.
+    #
+    # @return [void]
+    def ensure_connection_pool_size_is_suitable
+      current_pool_size = ActiveRecord::Base.connection_pool.size
+      desired_pool_size = @thread_count + 3
+
+      return if current_pool_size >= desired_pool_size
+
+      logger.warn "number of worker threads (#{@thread_count}) is more  " \
+                  "than the db connection pool size (#{current_pool_size}+3), " \
+                  "increasing connection pool size to #{desired_pool_size}"
+
+      Postal.change_database_connection_pool_size(desired_pool_size)
+    end
+
     # Wait for all threads to complete
     #
     # @return [void]
@@ -288,6 +308,9 @@ module Worker
       register_prometheus_histogram :postal_worker_task_runtime,
                                     docstring: "The time taken to process tasks",
                                     labels: [:task]
+
+      register_prometheus_histogram :postal_message_queue_latency,
+                                    docstring: "The length of time between a message being queued and being dequeued"
     end
 
   end

app/models/outgoing_message_prototype.rb

@@ -99,7 +99,7 @@ class OutgoingMessagePrototype
       {
         name: attachment[:name],
         content_type: attachment[:content_type] || "application/octet-stream",
-        data: attachment[:base64] ? Base64.decode64(attachment[:data]) : attachment[:data]
+        data: attachment[:base64] && attachment[:data] ? Base64.decode64(attachment[:data]) : attachment[:data]
       }
     end
   end

app/senders/smtp_sender.rb

@@ -244,11 +244,11 @@ class SMTPSender < BaseSender
       relays = Postal::Config.postal.smtp_relays
       return nil if relays.nil?
 
-      relays.map do |relay|
+      relays = relays.filter_map do |relay|
         next unless relay.host.present?
 
-        SMTPClient::Server.new(relay.host, relay.port, ssl_mode: relay.ssl_mode)
-      end.compact
+        SMTPClient::Server.new(relay.host, port: relay.port, ssl_mode: relay.ssl_mode)
+      end
 
       @smtp_relays = relays.empty? ? nil : relays
     end

app/views/layouts/application.html.haml

@@ -55,6 +55,8 @@
         %footer.siteContent__footer
           %ul.footer__links
             %li.footer__name
-              Powered by #{link_to "Postal", "https://postalserver.io", target: '_blank'} #{Postal.version}.
+              Powered by
+              #{link_to "Postal", "https://postalserver.io", target: '_blank'}
+              #{postal_version_string}
             %li= link_to "Documentation", "https://docs.postalserver.io", target: '_blank'
             %li= link_to "Ask for help", "https://discussions.postalserver.io", target: '_blank'

app/views/organizations/delete.html.haml

@@ -13,12 +13,11 @@
     all its mail servers & data will be deleted from our systems.
   .dangerZone
     %p.pageContent__text.u-margin
-      To continue to delete this organization, please enter your password in the field below and press
+      To continue to delete this organization, please enter the name of the organization in the field below and press
       continue. <b class='u-red'>There will be no other confirmations.</b>
     = form_tag [organization, :delete], :method => :delete, :remote => true do
       = hidden_field_tag 'return_to', params[:return_to]
       %p.u-margin
-        = password_field_tag "password", '', :class => 'input input--text input--danger'
+        = text_field_tag "confirm_text", '', :class => 'input input--text input--danger'
       .buttonSet.u-center
         = submit_tag "Delete this organization, mail servers and all messages", :class => 'button button--danger js-form-submit'
-

app/views/servers/delete.html.haml

@@ -10,12 +10,11 @@
     immediately.
   .dangerZone
     %p.pageContent__text.u-margin
-      To continue to remove this server, please enter your password in the field below and press
+      To continue to remove this server, please enter the server name in the field below and press
       continue. <b class='u-red'>There will be no other confirmations.</b>
     = form_tag [organization, @server], :remote => true, :method => :delete do
       = hidden_field_tag 'return_to', params[:return_to]
       %p.u-margin
-        = password_field_tag "password", '', :class => 'input input--text input--danger'
+        = text_field_tag "confirm_text", '', :class => 'input input--text input--danger'
       .buttonSet.u-center
         = submit_tag "Delete this mail server and all messages", :class => 'button button--danger'
-

config/routes.rb

@@ -1,6 +1,12 @@
 # frozen_string_literal: true
 
 Rails.application.routes.draw do
+  # Legacy API Routes
+  match "/api/v1/send/message" => "legacy_api/send#message", via: [:get, :post, :patch, :put]
+  match "/api/v1/send/raw" => "legacy_api/send#raw", via: [:get, :post, :patch, :put]
+  match "/api/v1/messages/message" => "legacy_api/messages#message", via: [:get, :post, :patch, :put]
+  match "/api/v1/messages/deliveries" => "legacy_api/messages#deliveries", via: [:get, :post, :patch, :put]
+
   scope "org/:org_permalink", as: "organization" do
     resources :domains, only: [:index, :new, :create, :destroy] do
       match :verify, on: :member, via: [:get, :post]

doc/config/environment-variables.md

@@ -19,11 +19,13 @@ This document contains all the environment variables which are available for thi
 | `POSTAL_SMTP_RELAYS` | Array of strings | An array of SMTP relays in the format of smtp://host:port | [] |
 | `POSTAL_TRUSTED_PROXIES` | Array of strings | An array of IP addresses to trust for proxying requests to Postal (in addition to localhost addresses) | [] |
 | `POSTAL_QUEUED_MESSAGE_LOCK_STALE_DAYS` | Integer | The number of days after which to consider a lock as stale. Messages with stale locks will be removed and not retried. | 1 |
+| `POSTAL_BATCH_QUEUED_MESSAGES` | Boolean | When enabled queued messages will be de-queued in batches based on their destination | true |
 | `WEB_SERVER_DEFAULT_PORT` | Integer | The default port the web server should listen on unless overriden by the PORT environment variable | 5000 |
 | `WEB_SERVER_DEFAULT_BIND_ADDRESS` | String | The default bind address the web server should listen on unless overriden by the BIND_ADDRESS environment variable | 127.0.0.1 |
 | `WEB_SERVER_MAX_THREADS` | Integer | The maximum number of threads which can be used by the web server | 5 |
 | `WORKER_DEFAULT_HEALTH_SERVER_PORT` | Integer | The default port for the worker health server to listen on | 9090 |
 | `WORKER_DEFAULT_HEALTH_SERVER_BIND_ADDRESS` | String | The default bind address for the worker health server to listen on | 127.0.0.1 |
+| `WORKER_THREADS` | Integer | The number of threads to execute within each worker | 2 |
 | `MAIN_DB_HOST` | String | Hostname for the main MariaDB server | localhost |
 | `MAIN_DB_PORT` | Integer | The MariaDB port to connect to | 3306 |
 | `MAIN_DB_USERNAME` | String | The MariaDB username | postal |
@@ -53,7 +55,7 @@ This document contains all the environment variables which are available for thi
 | `SMTP_SERVER_TLS_PRIVATE_KEY_PATH` | String | The path to the SMTP server's TLS private key | $config-file-root/smtp.key |
 | `SMTP_SERVER_TLS_CIPHERS` | String | Override ciphers to use for SSL |  |
 | `SMTP_SERVER_SSL_VERSION` | String | The SSL versions which are supported | SSLv23 |
-| `SMTP_SERVER_PROXY_PROTOCOL` | Boolean | Enable proxy protocol for use behind some load balancers | false |
+| `SMTP_SERVER_PROXY_PROTOCOL` | Boolean | Enable proxy protocol for use behind some load balancers (supports proxy protocol v1 only) | false |
 | `SMTP_SERVER_LOG_CONNECTIONS` | Boolean | Enable connection logging | false |
 | `SMTP_SERVER_MAX_MESSAGE_SIZE` | Integer | The maximum message size to accept from the SMTP server (in MB) | 14 |
 | `SMTP_SERVER_LOG_IP_ADDRESS_EXCLUSION_MATCHER` | String | A regular expression to use to exclude connections from logging |  |
@@ -98,13 +100,14 @@ This document contains all the environment variables which are available for thi
 | `MIGRATION_WAITER_ATTEMPTS` | Integer | The number of attempts to try waiting for migrations to complete before start | 120 |
 | `MIGRATION_WAITER_SLEEP_TIME` | Integer | The number of seconds to wait between each migration check | 2 |
 | `OIDC_ENABLED` | Boolean | Enable OIDC authentication | false |
+| `OIDC_LOCAL_AUTHENTICATION_ENABLED` | Boolean | When enabled, users with passwords will still be able to login locally. If disable, only OpenID Connect will be available. | true |
 | `OIDC_NAME` | String | The name of the OIDC provider as shown in the UI | OIDC Provider |
 | `OIDC_ISSUER` | String | The OIDC issuer URL |  |
 | `OIDC_IDENTIFIER` | String | The client ID for OIDC |  |
 | `OIDC_SECRET` | String | The client secret for OIDC |  |
-| `OIDC_SCOPES` | Array of strings | Scopes to request from the OIDC server. | openid |
+| `OIDC_SCOPES` | Array of strings | Scopes to request from the OIDC server. | openid,email |
 | `OIDC_UID_FIELD` | String | The field to use to determine the user's UID | sub |
-| `OIDC_EMAIL_ADDRESS_FIELD` | String | The field to use to determine the user's email address | sub |
+| `OIDC_EMAIL_ADDRESS_FIELD` | String | The field to use to determine the user's email address | email |
 | `OIDC_NAME_FIELD` | String | The field to use to determine the user's name | name |
 | `OIDC_DISCOVERY` | Boolean | Enable discovery to determine endpoints from .well-known/openid-configuration from the Issuer | true |
 | `OIDC_AUTHORIZATION_ENDPOINT` | String | The authorize endpoint on the authorization server (only used when discovery is false) |  |

doc/config/yaml.yml

@@ -31,6 +31,8 @@ postal:
   trusted_proxies: []
   # The number of days after which to consider a lock as stale. Messages with stale locks will be removed and not retried.
   queued_message_lock_stale_days: 1
+  # When enabled queued messages will be de-queued in batches based on their destination
+  batch_queued_messages: true
 
 web_server:
   # The default port the web server should listen on unless overriden by the PORT environment variable
@@ -45,6 +47,8 @@ worker:
   default_health_server_port: 9090
   # The default bind address for the worker health server to listen on
   default_health_server_bind_address: 127.0.0.1
+  # The number of threads to execute within each worker
+  threads: 2
 
 main_db:
   # Hostname for the main MariaDB server
@@ -113,7 +117,7 @@ smtp_server:
   tls_ciphers: 
   # The SSL versions which are supported
   ssl_version: SSLv23
-  # Enable proxy protocol for use behind some load balancers
+  # Enable proxy protocol for use behind some load balancers (supports proxy protocol v1 only)
   proxy_protocol: false
   # Enable connection logging
   log_connections: false
@@ -223,6 +227,8 @@ migration_waiter:
 oidc:
   # Enable OIDC authentication
   enabled: false
+  # When enabled, users with passwords will still be able to login locally. If disable, only OpenID Connect will be available.
+  local_authentication_enabled: true
   # The name of the OIDC provider as shown in the UI
   name: OIDC Provider
   # The OIDC issuer URL
@@ -233,11 +239,11 @@ oidc:
   secret: 
   # Scopes to request from the OIDC server.
   scopes:
-    - openid
+    - openid,email
   # The field to use to determine the user's UID
   uid_field: sub
   # The field to use to determine the user's email address
-  email_address_field: sub
+  email_address_field: email
   # The field to use to determine the user's name
   name_field: name
   # Enable discovery to determine endpoints from .well-known/openid-configuration from the Issuer

lib/postal/config.rb

@@ -16,11 +16,14 @@ require_relative "error"
 require_relative "version"
 require_relative "config_schema"
 require_relative "legacy_config_source"
+require_relative "signer"
 
 module Postal
 
   class << self
 
+    attr_writer :current_process_type
+
     # Return the path to the config file
     #
     # @return [String]
@@ -37,30 +40,36 @@ module Postal
       Dotenv.load(".env")
       sources << Konfig::Sources::Environment.new(ENV)
 
+      silence_config_messages = ENV.fetch("SILENCE_POSTAL_CONFIG_MESSAGES", "false") == "true"
+
       # If a config file exists, we need to load that. Config files can
       # either be legacy (v1) or new (v2). Any file without a 'version'
       # key is a legacy file whereas new-style config files will include
       # the 'version: 2' key/value.
       if File.file?(config_file_path)
-        puts "Loading config from #{config_file_path}"
+        unless silence_config_messages
+          warn "Loading config from #{config_file_path}"
+        end
 
         config_file = File.read(config_file_path)
         yaml = YAML.safe_load(config_file)
         config_version = yaml["version"] || 1
         case config_version
         when 1
-          puts "WARNING: Using legacy config file format. Upgrade your postal.yml to use"
-          puts "version 2 of the Postal configuration or configure using environment"
-          puts "variables. See https://postalserver.io/config-v2 for details."
+          unless silence_config_messages
+            warn "WARNING: Using legacy config file format. Upgrade your postal.yml to use"
+            warn "version 2 of the Postal configuration or configure using environment"
+            warn "variables. See https://postalserver.io/config-v2 for details."
+          end
           sources << LegacyConfigSource.new(yaml)
         when 2
           sources << Konfig::Sources::YAML.new(config_file)
         else
           raise "Invalid version specified in Postal config file. Must be 1 or 2."
         end
-      else
-        puts "No configuration file found at #{config_file_path}"
-        puts "Only using environment variables for configuration"
+      elsif !silence_config_messages
+        warn "No configuration file found at #{config_file_path}"
+        warn "Only using environment variables for configuration"
       end
 
       # Build configuration with the provided sources.
@@ -129,6 +138,33 @@ module Postal
       end
     end
 
+    # Change the connection pool size to the given size.
+    #
+    # @param new_size [Integer]
+    # @return [void]
+    def change_database_connection_pool_size(new_size)
+      ActiveRecord::Base.connection_pool.disconnect!
+
+      config = ActiveRecord::Base.configurations
+                                 .configs_for(env_name: Rails.env)
+                                 .first
+                                 .configuration_hash
+
+      ActiveRecord::Base.establish_connection(config.merge(pool: new_size))
+    end
+
+    # Return the branch name which created this release
+    #
+    # @return [String, nil]
+    def branch
+      return @branch if instance_variable_defined?("@branch")
+
+      @branch = begin
+        path = Rails.root.join("BRANCH")
+        File.read(path).strip if File.exist?(path)
+      end
+    end
+
   end
 
   Config = initialize_config

lib/postal/config_schema.rb

@@ -96,6 +96,11 @@ module Postal
         description "The number of days after which to consider a lock as stale. Messages with stale locks will be removed and not retried."
         default 1
       end
+
+      boolean :batch_queued_messages do
+        description "When enabled queued messages will be de-queued in batches based on their destination"
+        default true
+      end
     end
 
     group :web_server do
@@ -125,6 +130,11 @@ module Postal
         description "The default bind address for the worker health server to listen on"
         default "127.0.0.1"
       end
+
+      integer :threads do
+        description "The number of threads to execute within each worker"
+        default 2
+      end
     end
 
     group :main_db do
@@ -279,7 +289,7 @@ module Postal
       end
 
       boolean :proxy_protocol do
-        description "Enable proxy protocol for use behind some load balancers"
+        description "Enable proxy protocol for use behind some load balancers (supports proxy protocol v1 only)"
         default false
       end
 

lib/postal/signer.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "base64"
+module Postal
+  class Signer
+
+    # Create a new Signer
+    #
+    # @param [OpenSSL::PKey::RSA] private_key The private key to use for signing
+    # @return [Signer]
+    def initialize(private_key)
+      @private_key = private_key
+    end
+
+    # Return the private key
+    #
+    # @return [OpenSSL::PKey::RSA]
+    attr_reader :private_key
+
+    # Return the public key for the private key
+    #
+    # @return [OpenSSL::PKey::RSA]
+    def public_key
+      @private_key.public_key
+    end
+
+    # Sign the given data
+    #
+    # @param [String] data The data to sign
+    # @return [String] The signature
+    def sign(data)
+      private_key.sign(OpenSSL::Digest.new("SHA256"), data)
+    end
+
+    # Sign the given data and return a Base64-encoded signature
+    #
+    # @param [String] data The data to sign
+    # @return [String] The Base64-encoded signature
+    def sign64(data)
+      Base64.strict_encode64(sign(data))
+    end
+
+    # Return a JWK for the private key
+    #
+    # @return [JWT::JWK] The JWK
+    def jwk
+      @jwk ||= JWT::JWK.new(private_key, { use: "sig", alg: "RS256" })
+    end
+
+    # Sign the given data using SHA1 (for legacy use)
+    #
+    # @param [String] data The data to sign
+    # @return [String] The signature
+    def sha1_sign(data)
+      private_key.sign(OpenSSL::Digest.new("SHA1"), data)
+    end
+
+    # Sign the given data using SHA1 (for legacy use) and return a Base64-encoded string
+    #
+    # @param [String] data The data to sign
+    # @return [String] The signature
+    def sha1_sign64(data)
+      Base64.strict_encode64(sha1_sign(data))
+    end
+
+  end
+end

script/default_dkim_record.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
 
+ENV["SILENCE_POSTAL_CONFIG_LOCATION_MESSAGE"] = "true"
 require File.expand_path("../lib/postal/config", __dir__)
 puts Postal.rp_dkim_dns_record

script/generate_api_docs.sh

@@ -1,21 +0,0 @@
-#!/bin/bash
-
-set -e
-
-if [ ! -d /tmp/postal-api/.git ];
-then
-  git clone git@github.com:atech/postal-api /tmp/postal-api
-else
-  git -C /tmp/postal-api reset --hard HEAD
-  git -C /tmp/postal-api pull origin master
-fi
-
-rm -Rf /tmp/postal-api/*
-
-bundle exec moonrope api /tmp/postal-api
-
-cd /tmp/postal-api
-
-git add .
-git commit -m "update docs"
-git push origin master

spec/apis/legacy_api/messages/deliveries_spec.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe "Legacy Messages API", type: :request do
+  describe "/api/v1/messages/deliveries" do
+    context "when no authentication is provided" do
+      it "returns an error" do
+        post "/api/v1/messages/deliveries"
+        expect(response.status).to eq 200
+        parsed_body = JSON.parse(response.body)
+        expect(parsed_body["status"]).to eq "error"
+        expect(parsed_body["data"]["code"]).to eq "AccessDenied"
+      end
+    end
+
+    context "when the credential does not match anything" do
+      it "returns an error" do
+        post "/api/v1/messages/deliveries", headers: { "x-server-api-key" => "invalid" }
+        expect(response.status).to eq 200
+        parsed_body = JSON.parse(response.body)
+        expect(parsed_body["status"]).to eq "error"
+        expect(parsed_body["data"]["code"]).to eq "InvalidServerAPIKey"
+      end
+    end
+
+    context "when the credential belongs to a suspended server" do
+      it "returns an error" do
+        server = create(:server, :suspended)
+        credential = create(:credential, server: server)
+        post "/api/v1/messages/deliveries", headers: { "x-server-api-key" => credential.key }
+        expect(response.status).to eq 200
+        parsed_body = JSON.parse(response.body)
+        expect(parsed_body["status"]).to eq "error"
+        expect(parsed_body["data"]["code"]).to eq "ServerSuspended"
+      end
+    end
+
+    context "when the credential is valid" do
+      let(:server) { create(:server) }
+      let(:credential) { create(:credential, server: server) }
+
+      context "when no message ID is provided" do
+        it "returns an error" do
+          post "/api/v1/messages/deliveries", headers: { "x-server-api-key" => credential.key }
+          expect(response.status).to eq 200
+          parsed_body = JSON.parse(response.body)
+          expect(parsed_body["status"]).to eq "parameter-error"
+          expect(parsed_body["data"]["message"]).to match(/`id` parameter is required but is missing/)
+        end
+      end
+
+      context "when the message ID does not exist" do
+        it "returns an error" do
+          post "/api/v1/messages/deliveries",
+               headers: { "x-server-api-key" => credential.key,
+                          "content-type" => "application/json" },
+               params: { id: 123 }.to_json
+          expect(response.status).to eq 200
+          parsed_body = JSON.parse(response.body)
+          expect(parsed_body["status"]).to eq "error"
+          expect(parsed_body["data"]["code"]).to eq "MessageNotFound"
+          expect(parsed_body["data"]["id"]).to eq 123
+        end
+      end
+
+      context "when the message ID exists" do
+        let(:server) { create(:server) }
+        let(:credential) { create(:credential, server: server) }
+        let(:message) { MessageFactory.outgoing(server) }
+
+        before do
+          message.create_delivery("SoftFail", details: "no server found",
+                                              output: "404",
+                                              sent_with_ssl: true,
+                                              log_id: "1234",
+                                              time: 1.2)
+          message.create_delivery("Sent", details: "sent successfully",
+                                          output: "200",
+                                          sent_with_ssl: false,
+                                          log_id: "5678",
+                                          time: 2.2)
+        end
+
+        before do
+          post "/api/v1/messages/deliveries",
+               headers: { "x-server-api-key" => credential.key,
+                          "content-type" => "application/json" },
+               params: { id: message.id }.to_json
+        end
+
+        it "returns an array of deliveries" do
+          expect(response.status).to eq 200
+          parsed_body = JSON.parse(response.body)
+          expect(parsed_body["status"]).to eq "success"
+          expect(parsed_body["data"]).to match([
+                                                 { "id" => kind_of(Integer),
+                                                   "status" => "SoftFail",
+                                                   "details" => "no server found",
+                                                   "output" => "404",
+                                                   "sent_with_ssl" => true,
+                                                   "log_id" => "1234",
+                                                   "time" => 1.2,
+                                                   "timestamp" => kind_of(Float) },
+                                                 { "id" => kind_of(Integer),
+                                                   "status" => "Sent",
+                                                   "details" => "sent successfully",
+                                                   "output" => "200",
+                                                   "sent_with_ssl" => false,
+                                                   "log_id" => "5678",
+                                                   "time" => 2.2,
+                                                   "timestamp" => kind_of(Float) },
+                                               ])
+        end
+      end
+    end
+  end
+end

spec/apis/legacy_api/messages/message_spec.rb

@@ -0,0 +1,270 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe "Legacy Messages API", type: :request do
+  describe "/api/v1/messages/message" do
+    context "when no authentication is provided" do
+      it "returns an error" do
+        post "/api/v1/messages/message"
+        expect(response.status).to eq 200
+        parsed_body = JSON.parse(response.body)
+        expect(parsed_body["status"]).to eq "error"
+        expect(parsed_body["data"]["code"]).to eq "AccessDenied"
+      end
+    end
+
+    context "when the credential does not match anything" do
+      it "returns an error" do
+        post "/api/v1/messages/message", headers: { "x-server-api-key" => "invalid" }
+        expect(response.status).to eq 200
+        parsed_body = JSON.parse(response.body)
+        expect(parsed_body["status"]).to eq "error"
+        expect(parsed_body["data"]["code"]).to eq "InvalidServerAPIKey"
+      end
+    end
+
+    context "when the credential belongs to a suspended server" do
+      it "returns an error" do
+        server = create(:server, :suspended)
+        credential = create(:credential, server: server)
+        post "/api/v1/messages/message", headers: { "x-server-api-key" => credential.key }
+        expect(response.status).to eq 200
+        parsed_body = JSON.parse(response.body)
+        expect(parsed_body["status"]).to eq "error"
+        expect(parsed_body["data"]["code"]).to eq "ServerSuspended"
+      end
+    end
+
+    context "when the credential is valid" do
+      let(:server) { create(:server) }
+      let(:credential) { create(:credential, server: server) }
+
+      context "when no message ID is provided" do
+        it "returns an error" do
+          post "/api/v1/messages/message", headers: { "x-server-api-key" => credential.key }
+          expect(response.status).to eq 200
+          parsed_body = JSON.parse(response.body)
+          expect(parsed_body["status"]).to eq "parameter-error"
+          expect(parsed_body["data"]["message"]).to match(/`id` parameter is required but is missing/)
+        end
+      end
+
+      context "when the message ID does not exist" do
+        it "returns an error" do
+          post "/api/v1/messages/message",
+               headers: { "x-server-api-key" => credential.key,
+                          "content-type" => "application/json" },
+               params: { id: 123 }.to_json
+          expect(response.status).to eq 200
+          parsed_body = JSON.parse(response.body)
+          expect(parsed_body["status"]).to eq "error"
+          expect(parsed_body["data"]["code"]).to eq "MessageNotFound"
+        end
+      end
+
+      context "when the message ID exists" do
+        let(:server) { create(:server) }
+        let(:credential) { create(:credential, server: server) }
+        let(:message) { MessageFactory.outgoing(server) }
+        let(:expansions) { [] }
+
+        before do
+          post "/api/v1/messages/message",
+               headers: { "x-server-api-key" => credential.key,
+                          "content-type" => "application/json" },
+               params: { id: message.id, _expansions: expansions }.to_json
+        end
+
+        context "when no expansions are requested" do
+          it "returns details about the message" do
+            expect(response.status).to eq 200
+            parsed_body = JSON.parse(response.body)
+            expect(parsed_body["status"]).to eq "success"
+            expect(parsed_body["data"]).to match({
+              "id" => message.id,
+              "token" => message.token
+            })
+          end
+        end
+
+        context "when the status expansion is requested" do
+          let(:expansions) { ["status"] }
+
+          it "returns details about the message" do
+            expect(response.status).to eq 200
+            parsed_body = JSON.parse(response.body)
+            expect(parsed_body["status"]).to eq "success"
+            expect(parsed_body["data"]).to match({
+              "id" => message.id,
+              "token" => message.token,
+              "status" => { "held" => false,
+                            "hold_expiry" => nil,
+                            "last_delivery_attempt" => nil,
+                            "status" => "Pending" }
+            })
+          end
+        end
+
+        context "when the details expansion is requested" do
+          let(:expansions) { ["details"] }
+
+          it "returns details about the message" do
+            expect(response.status).to eq 200
+            parsed_body = JSON.parse(response.body)
+            expect(parsed_body["status"]).to eq "success"
+            expect(parsed_body["data"]).to match({
+              "id" => message.id,
+              "token" => message.token,
+              "details" => { "bounce" => false,
+                             "bounce_for_id" => 0,
+                             "direction" => "outgoing",
+                             "mail_from" => "test@example.com",
+                             "message_id" => message.message_id,
+                             "rcpt_to" => "john@example.com",
+                             "received_with_ssl" => nil,
+                             "size" => kind_of(String),
+                             "subject" => "An example message",
+                             "tag" => nil,
+                             "timestamp" => kind_of(Float) }
+            })
+          end
+        end
+
+        context "when the details expansion is requested" do
+          let(:expansions) { ["inspection"] }
+
+          it "returns details about the message" do
+            expect(response.status).to eq 200
+            parsed_body = JSON.parse(response.body)
+            expect(parsed_body["status"]).to eq "success"
+            expect(parsed_body["data"]).to match({
+              "id" => message.id,
+              "token" => message.token,
+              "inspection" => { "inspected" => false,
+                                "spam" => false,
+                                "spam_score" => 0.0,
+                                "threat" => false,
+                                "threat_details" => nil }
+            })
+          end
+        end
+
+        context "when the body expansions are requested" do
+          let(:expansions) { %w[plain_body html_body] }
+
+          it "returns details about the message" do
+            expect(response.status).to eq 200
+            parsed_body = JSON.parse(response.body)
+            expect(parsed_body["status"]).to eq "success"
+            expect(parsed_body["data"]).to match({
+              "id" => message.id,
+              "token" => message.token,
+              "plain_body" => message.plain_body,
+              "html_body" => message.html_body
+            })
+          end
+        end
+
+        context "when the attachments expansions is requested" do
+          let(:message) do
+            MessageFactory.outgoing(server) do |_, mail|
+              mail.attachments["example.txt"] = "hello world!"
+            end
+          end
+          let(:expansions) { ["attachments"] }
+
+          it "returns details about the message" do
+            expect(response.status).to eq 200
+            parsed_body = JSON.parse(response.body)
+            expect(parsed_body["status"]).to eq "success"
+            expect(parsed_body["data"]).to match({
+              "id" => message.id,
+              "token" => message.token,
+              "attachments" => [
+                {
+                  "content_type" => "text/plain",
+                  "data" => Base64.encode64("hello world!"),
+                  "filename" => "example.txt",
+                  "hash" => Digest::SHA1.hexdigest("hello world!"),
+                  "size" => 12
+                },
+              ]
+            })
+          end
+        end
+
+        context "when the headers expansions is requested" do
+          let(:expansions) { ["headers"] }
+
+          it "returns details about the message" do
+            expect(response.status).to eq 200
+            parsed_body = JSON.parse(response.body)
+            expect(parsed_body["status"]).to eq "success"
+            expect(parsed_body["data"]).to match({
+              "id" => message.id,
+              "token" => message.token,
+              "headers" => message.headers
+            })
+          end
+        end
+
+        context "when the raw_message expansions is requested" do
+          let(:expansions) { ["raw_message"] }
+
+          it "returns details about the message" do
+            expect(response.status).to eq 200
+            parsed_body = JSON.parse(response.body)
+            expect(parsed_body["status"]).to eq "success"
+            expect(parsed_body["data"]).to match({
+              "id" => message.id,
+              "token" => message.token,
+              "raw_message" => Base64.encode64(message.raw_message)
+            })
+          end
+        end
+
+        context "when the activity_entries expansions is requested" do
+          let(:message) do
+            MessageFactory.outgoing(server) do |msg|
+              msg.create_load(double("request", ip: "1.2.3.4", user_agent: "user agent"))
+              link = msg.create_link("https://example.com")
+              link_id = msg.database.select(:links, where: { token: link }).first["id"]
+              msg.database.insert(:clicks, {
+                message_id: msg.id,
+                link_id: link_id,
+                ip_address: "1.2.3.4",
+                user_agent: "user agent",
+                timestamp: Time.now.to_f
+              })
+            end
+          end
+          let(:expansions) { ["activity_entries"] }
+
+          it "returns details about the message" do
+            expect(response.status).to eq 200
+            parsed_body = JSON.parse(response.body)
+            expect(parsed_body["status"]).to eq "success"
+            expect(parsed_body["data"]).to match({
+              "id" => message.id,
+              "token" => message.token,
+              "activity_entries" => {
+                "loads" => [{
+                  "ip_address" => "1.2.3.4",
+                  "user_agent" => "user agent",
+                  "timestamp" => match(/\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z\z/)
+                }],
+                "clicks" => [{
+                  "url" => "https://example.com",
+                  "ip_address" => "1.2.3.4",
+                  "user_agent" => "user agent",
+                  "timestamp" => match(/\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z\z/)
+                }]
+              }
+            })
+          end
+        end
+      end
+    end
+  end
+end

spec/apis/legacy_api/send/message_spec.rb

@@ -0,0 +1,237 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe "Legacy Send API", type: :request do
+  describe "/api/v1/send/message" do
+    context "when no authentication is provided" do
+      it "returns an error" do
+        post "/api/v1/send/message"
+        expect(response.status).to eq 200
+        parsed_body = JSON.parse(response.body)
+        expect(parsed_body["status"]).to eq "error"
+        expect(parsed_body["data"]["code"]).to eq "AccessDenied"
+      end
+    end
+
+    context "when the credential does not match anything" do
+      it "returns an error" do
+        post "/api/v1/send/message", headers: { "x-server-api-key" => "invalid" }
+        expect(response.status).to eq 200
+        parsed_body = JSON.parse(response.body)
+        expect(parsed_body["status"]).to eq "error"
+        expect(parsed_body["data"]["code"]).to eq "InvalidServerAPIKey"
+      end
+    end
+
+    context "when the credential belongs to a suspended server" do
+      it "returns an error" do
+        server = create(:server, :suspended)
+        credential = create(:credential, server: server)
+        post "/api/v1/send/message", headers: { "x-server-api-key" => credential.key }
+        expect(response.status).to eq 200
+        parsed_body = JSON.parse(response.body)
+        expect(parsed_body["status"]).to eq "error"
+        expect(parsed_body["data"]["code"]).to eq "ServerSuspended"
+      end
+    end
+
+    context "when the credential is valid" do
+      let(:server) { create(:server) }
+      let(:credential) { create(:credential, server: server) }
+      let(:domain) { create(:domain, owner: server) }
+
+      context "when parameters are provided in a JSON body" do
+        let(:default_params) do
+          {
+            to: ["test@example.com"],
+            cc: ["cc@example.com"],
+            bcc: ["bcc@example.com"],
+            from: "test@#{domain.name}",
+            sender: "sender@#{domain.name}",
+            tag: "test-tag",
+            reply_to: "reply@example.com",
+            plain_body: "plain text",
+            html_body: "<p>html</p>",
+            attachments: [{ name: "test1.txt", content_type: "text/plain", data: Base64.encode64("hello world 1") },
+                          { name: "test2.txt", content_type: "text/plain", data: Base64.encode64("hello world 2") },],
+            headers: { "x-test-header-1" => "111", "x-test-header-2" => "222" },
+            bounce: false,
+            subject: "Test"
+          }
+        end
+        let(:params) { default_params }
+
+        before do
+          post "/api/v1/send/message",
+               headers: { "x-server-api-key" => credential.key,
+                          "content-type" => "application/json" },
+               params: params.to_json
+        end
+
+        context "when no recipients are provided" do
+          let(:params) { default_params.merge(to: [], cc: [], bcc: []) }
+
+          it "returns an error" do
+            parsed_body = JSON.parse(response.body)
+            expect(parsed_body["status"]).to eq "error"
+            expect(parsed_body["data"]["code"]).to eq "NoRecipients"
+            expect(parsed_body["data"]["message"]).to match(/there are no recipients defined to receive this message/i)
+          end
+        end
+
+        context "when no content is provided" do
+          let(:params) { default_params.merge(html_body: nil, plain_body: nil) }
+
+          it "returns an error" do
+            parsed_body = JSON.parse(response.body)
+            expect(parsed_body["status"]).to eq "error"
+            expect(parsed_body["data"]["code"]).to eq "NoContent"
+            expect(parsed_body["data"]["message"]).to match(/there is no content defined for this e-mail/i)
+          end
+        end
+
+        context "when the number of 'To' recipients exceeds the maximum" do
+          let(:params) { default_params.merge(to: ["a@a.com"] * 51) }
+
+          it "returns an error" do
+            parsed_body = JSON.parse(response.body)
+            expect(parsed_body["status"]).to eq "error"
+            expect(parsed_body["data"]["code"]).to eq "TooManyToAddresses"
+            expect(parsed_body["data"]["message"]).to match(/the maximum number of To addresses has been reached/i)
+          end
+        end
+
+        context "when the number of 'CC' recipients exceeds the maximum" do
+          let(:params) { default_params.merge(cc: ["a@a.com"] * 51) }
+
+          it "returns an error" do
+            parsed_body = JSON.parse(response.body)
+            expect(parsed_body["status"]).to eq "error"
+            expect(parsed_body["data"]["code"]).to eq "TooManyCCAddresses"
+            expect(parsed_body["data"]["message"]).to match(/the maximum number of CC addresses has been reached/i)
+          end
+        end
+
+        context "when the number of 'BCC' recipients exceeds the maximum" do
+          let(:params) { default_params.merge(bcc: ["a@a.com"] * 51) }
+
+          it "returns an error" do
+            parsed_body = JSON.parse(response.body)
+            expect(parsed_body["status"]).to eq "error"
+            expect(parsed_body["data"]["code"]).to eq "TooManyBCCAddresses"
+            expect(parsed_body["data"]["message"]).to match(/the maximum number of BCC addresses has been reached/i)
+          end
+        end
+
+        context "when the 'From' address is missing" do
+          let(:params) { default_params.merge(from: nil) }
+
+          it "returns an error" do
+            parsed_body = JSON.parse(response.body)
+            expect(parsed_body["status"]).to eq "error"
+            expect(parsed_body["data"]["code"]).to eq "FromAddressMissing"
+            expect(parsed_body["data"]["message"]).to match(/the from address is missing and is required/i)
+          end
+        end
+
+        context "when the 'From' address is not authorised" do
+          let(:params) { default_params.merge(from: "test@another.com") }
+
+          it "returns an error" do
+            parsed_body = JSON.parse(response.body)
+            expect(parsed_body["status"]).to eq "error"
+            expect(parsed_body["data"]["code"]).to eq "UnauthenticatedFromAddress"
+            expect(parsed_body["data"]["message"]).to match(/the from address is not authorised to send mail from this server/i)
+          end
+        end
+
+        context "when an attachment is missing a name" do
+          let(:params) { default_params.merge(attachments: [{ name: nil, content_type: "text/plain", data: Base64.encode64("hello world 1") }]) }
+
+          it "returns an error" do
+            parsed_body = JSON.parse(response.body)
+            expect(parsed_body["status"]).to eq "error"
+            expect(parsed_body["data"]["code"]).to eq "AttachmentMissingName"
+            expect(parsed_body["data"]["message"]).to match(/an attachment is missing a name/i)
+          end
+        end
+
+        context "when an attachment is missing data" do
+          let(:params) { default_params.merge(attachments: [{ name: "test1.txt", content_type: "text/plain", data: nil }]) }
+
+          it "returns an error" do
+            parsed_body = JSON.parse(response.body)
+            expect(parsed_body["status"]).to eq "error"
+            expect(parsed_body["data"]["code"]).to eq "AttachmentMissingData"
+            expect(parsed_body["data"]["message"]).to match(/an attachment is missing data/i)
+          end
+        end
+
+        context "when an attachment entry is not a hash" do
+          let(:params) { default_params.merge(attachments: [123, "string"]) }
+
+          it "continues as if it wasn't there" do
+            parsed_body = JSON.parse(response.body)
+            ["test@example.com", "cc@example.com", "bcc@example.com"].each do |rcpt_to|
+              message_id = parsed_body["data"]["messages"][rcpt_to]["id"]
+              message = server.message(message_id)
+              expect(message.attachments).to be_empty
+            end
+          end
+        end
+
+        context "when given a complete email to send" do
+          it "returns details of the messages created" do
+            parsed_body = JSON.parse(response.body)
+            expect(parsed_body["status"]).to eq "success"
+            expect(parsed_body["data"]["messages"]).to match({
+              "test@example.com" => { "id" => kind_of(Integer), "token" => /\A[a-zA-Z0-9]{16}\z/ },
+              "cc@example.com" => { "id" => kind_of(Integer), "token" => /\A[a-zA-Z0-9]{16}\z/ },
+              "bcc@example.com" => { "id" => kind_of(Integer), "token" => /\A[a-zA-Z0-9]{16}\z/ }
+            })
+          end
+
+          it "adds an appropriate received header" do
+            parsed_body = JSON.parse(response.body)
+            message_id = parsed_body["data"]["messages"]["test@example.com"]["id"]
+            message = server.message(message_id)
+            expect(message.headers["received"].first).to match(/\Afrom api/)
+          end
+
+          it "creates appropriate message objects" do
+            parsed_body = JSON.parse(response.body)
+            ["test@example.com", "cc@example.com", "bcc@example.com"].each do |rcpt_to|
+              message_id = parsed_body["data"]["messages"][rcpt_to]["id"]
+              message = server.message(message_id)
+              expect(message).to have_attributes(
+                server: server,
+                rcpt_to: rcpt_to,
+                mail_from: params[:from],
+                subject: params[:subject],
+                message_id: kind_of(String),
+                timestamp: kind_of(Time),
+                domain_id: domain.id,
+                credential_id: credential.id,
+                bounce: false,
+                tag: params[:tag],
+                headers: hash_including("x-test-header-1" => ["111"],
+                                        "x-test-header-2" => ["222"],
+                                        "sender" => [params[:sender]],
+                                        "to" => ["test@example.com"],
+                                        "cc" => ["cc@example.com"],
+                                        "reply-to" => ["reply@example.com"]),
+                plain_body: params[:plain_body],
+                html_body: params[:html_body],
+                attachments: [
+                  have_attributes(content_type: /\Atext\/plain/, filename: "test1.txt", body: have_attributes(to_s: "hello world 1")),
+                  have_attributes(content_type: /\Atext\/plain/, filename: "test2.txt", body: have_attributes(to_s: "hello world 2")),
+                ]
+              )
+            end
+          end
+        end
+      end
+    end
+  end
+end

spec/apis/legacy_api/send/raw_spec.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+RSpec.describe "Legacy Send API", type: :request do
+  describe "/api/v1/send/raw" do
+    context "when no authentication is provided" do
+      it "returns an error" do
+        post "/api/v1/send/raw"
+        expect(response.status).to eq 200
+        parsed_body = JSON.parse(response.body)
+        expect(parsed_body["status"]).to eq "error"
+        expect(parsed_body["data"]["code"]).to eq "AccessDenied"
+      end
+    end
+
+    context "when the credential does not match anything" do
+      it "returns an error" do
+        post "/api/v1/send/raw", headers: { "x-server-api-key" => "invalid" }
+        expect(response.status).to eq 200
+        parsed_body = JSON.parse(response.body)
+        expect(parsed_body["status"]).to eq "error"
+        expect(parsed_body["data"]["code"]).to eq "InvalidServerAPIKey"
+      end
+    end
+
+    context "when the credential belongs to a suspended server" do
+      it "returns an error" do
+        server = create(:server, :suspended)
+        credential = create(:credential, server: server)
+        post "/api/v1/send/raw", headers: { "x-server-api-key" => credential.key }
+        expect(response.status).to eq 200
+        parsed_body = JSON.parse(response.body)
+        expect(parsed_body["status"]).to eq "error"
+        expect(parsed_body["data"]["code"]).to eq "ServerSuspended"
+      end
+    end
+
+    context "when the credential is valid" do
+      let(:server) { create(:server) }
+      let(:credential) { create(:credential, server: server) }
+      let(:domain) { create(:domain, owner: server) }
+      let(:data) do
+        mail = Mail.new
+        mail.to = "test1@example.com"
+        mail.from = "test@#{domain.name}"
+        mail.subject = "test"
+        mail.text_part = Mail::Part.new
+        mail.text_part.body = "plain text"
+        mail.html_part = Mail::Part.new
+        mail.html_part.content_type = "text/html; charset=UTF-8"
+        mail.html_part.body = "<p>html</p>"
+        mail
+      end
+      let(:default_params) do
+        {
+          mail_from: "test@#{domain.name}",
+          rcpt_to: ["test1@example.com", "test2@example.com"],
+          data: Base64.encode64(data.to_s),
+          bounce: false
+        }
+      end
+      let(:content_type) { "application/json" }
+      let(:params) { default_params }
+
+      before do
+        post "/api/v1/send/raw",
+             headers: { "x-server-api-key" => credential.key,
+                        "content-type" => content_type },
+             params: content_type == "application/json" ? params.to_json : params
+      end
+
+      context "when rcpt_to is not provided" do
+        let(:params) { default_params.except(:rcpt_to) }
+
+        it "returns an error" do
+          parsed_body = JSON.parse(response.body)
+          expect(parsed_body["status"]).to eq "parameter-error"
+          expect(parsed_body["data"]["message"]).to match(/`rcpt_to` parameter is required but is missing/i)
+        end
+      end
+
+      context "when mail_from is not provided" do
+        let(:params) { default_params.except(:mail_from) }
+
+        it "returns an error" do
+          parsed_body = JSON.parse(response.body)
+          expect(parsed_body["status"]).to eq "parameter-error"
+          expect(parsed_body["data"]["message"]).to match(/`mail_from` parameter is required but is missing/i)
+        end
+      end
+
+      context "when data is not provided" do
+        let(:params) { default_params.except(:data) }
+
+        it "returns an error" do
+          parsed_body = JSON.parse(response.body)
+          expect(parsed_body["status"]).to eq "parameter-error"
+          expect(parsed_body["data"]["message"]).to match(/`data` parameter is required but is missing/i)
+        end
+      end
+
+      context "when no recipients are provided" do
+        let(:params) { default_params.merge(rcpt_to: []) }
+
+        it "returns success but with no messages" do
+          parsed_body = JSON.parse(response.body)
+          expect(parsed_body["status"]).to eq "success"
+          expect(parsed_body["data"]["messages"]).to eq({})
+          expect(parsed_body["data"]["message_id"]).to be nil
+        end
+      end
+
+      context "when a valid email is provided" do
+        it "returns details of the messages created" do
+          parsed_body = JSON.parse(response.body)
+          expect(parsed_body["data"]["message_id"]).to be_a String
+          expect(parsed_body["data"]["messages"]).to be_a Hash
+          expect(parsed_body["data"]["messages"]).to match({
+            "test1@example.com" => { "id" => kind_of(Integer), "token" => /\A[a-zA-Z0-9]{16}\z/ },
+            "test2@example.com" => { "id" => kind_of(Integer), "token" => /\A[a-zA-Z0-9]{16}\z/ }
+          })
+        end
+
+        it "creates appropriate message objects" do
+          parsed_body = JSON.parse(response.body)
+          ["test1@example.com", "test2@example.com"].each do |rcpt_to|
+            message_id = parsed_body["data"]["messages"][rcpt_to]["id"]
+            message = server.message(message_id)
+            expect(message).to have_attributes(
+              server: server,
+              rcpt_to: rcpt_to,
+              mail_from: "test@#{domain.name}",
+              subject: "test",
+              message_id: kind_of(String),
+              timestamp: kind_of(Time),
+              domain_id: domain.id,
+              credential_id: credential.id,
+              bounce: false,
+              headers: hash_including("to" => ["test1@example.com"]),
+              plain_body: "plain text",
+              html_body: "<p>html</p>",
+              attachments: [],
+              received_with_ssl: true,
+              scope: "outgoing",
+              raw_message: data.to_s
+            )
+          end
+        end
+
+        context "when params are provided as a param" do
+          let(:content_type) { nil }
+          let(:params) { { params: default_params.to_json } }
+
+          it "returns details of the messages created" do
+            parsed_body = JSON.parse(response.body)
+            expect(parsed_body["data"]["message_id"]).to be_a String
+            expect(parsed_body["data"]["messages"]).to be_a Hash
+            expect(parsed_body["data"]["messages"]).to match({
+              "test1@example.com" => { "id" => kind_of(Integer), "token" => /\A[a-zA-Z0-9]{16}\z/ },
+              "test2@example.com" => { "id" => kind_of(Integer), "token" => /\A[a-zA-Z0-9]{16}\z/ }
+            })
+          end
+        end
+      end
+    end
+  end
+end

spec/lib/message_dequeuer/initial_message_processor_spec.rb

@@ -62,13 +62,28 @@ module MessageDequeuer
         @queued_message3 = create(:queued_message, message: @message3)
       end
 
-      it "calls the single message process for the initial message and all batchable messages" do
-        [queued_message, @queued_message2, @queued_message3].each do |msg|
-          expect(SingleMessageProcessor).to receive(:process).with(msg,
-                                                                   logger: logger,
-                                                                   state: processor.state)
+      context "when postal.batch_queued_messages is enabled" do
+        it "calls the single message process for the initial message and all batchable messages" do
+          [queued_message, @queued_message2, @queued_message3].each do |msg|
+            expect(SingleMessageProcessor).to receive(:process).with(msg,
+                                                                     logger: logger,
+                                                                     state: processor.state)
+          end
+          processor.process
+        end
+      end
+
+      context "when postal.batch_queued_messages is disabled" do
+        before do
+          allow(Postal::Config.postal).to receive(:batch_queued_messages?) { false }
+        end
+
+        it "does not call the single message process more than once" do
+          expect(SingleMessageProcessor).to receive(:process).once.with(queued_message,
+                                                                        logger: logger,
+                                                                        state: processor.state)
+          processor.process
         end
-        processor.process
       end
     end
 

spec/lib/postal/signer_spec.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+module Postal
+
+  RSpec.describe Signer do
+    STATIC_PRIVATE_KEY = OpenSSL::PKey::RSA.new(2048) # rubocop:disable Lint/ConstantDefinitionInBlock
+
+    subject(:signer) { described_class.new(STATIC_PRIVATE_KEY) }
+
+    describe "#private_key" do
+      it "returns the private key" do
+        expect(signer.private_key).to eq(STATIC_PRIVATE_KEY)
+      end
+    end
+
+    describe "#public_key" do
+      it "returns the public key" do
+        expect(signer.public_key.to_s).to eq(STATIC_PRIVATE_KEY.public_key.to_s)
+      end
+    end
+
+    describe "#sign" do
+      it "returns a valid signature" do
+        data = "hello world!"
+        signature = signer.sign(data)
+        expect(signature).to be_a(String)
+        verification = STATIC_PRIVATE_KEY.public_key.verify(OpenSSL::Digest.new("SHA256"),
+                                                            signature,
+                                                            data)
+        expect(verification).to be true
+      end
+    end
+
+    describe "#sign64" do
+      it "returns a valid Base64-encoded signature" do
+        data = "hello world!"
+        signature = signer.sign64(data)
+        expect(signature).to be_a(String)
+        verification = STATIC_PRIVATE_KEY.public_key.verify(OpenSSL::Digest.new("SHA256"),
+                                                            Base64.strict_decode64(signature),
+                                                            data)
+        expect(verification).to be true
+      end
+    end
+
+    describe "#jwk" do
+      it "returns a valid JWK" do
+        jwk = signer.jwk
+        expect(jwk).to be_a(JWT::JWK::RSA)
+      end
+    end
+
+    describe "#sha1_sign" do
+      it "returns a valid signature" do
+        data = "hello world!"
+        signature = signer.sha1_sign(data)
+        expect(signature).to be_a(String)
+        verification = STATIC_PRIVATE_KEY.public_key.verify(OpenSSL::Digest.new("SHA1"),
+                                                            signature,
+                                                            data)
+        expect(verification).to be true
+      end
+    end
+
+    describe "#sha1_sign64" do
+      it "returns a valid Base64-encoded signature" do
+        data = "hello world!"
+        signature = signer.sha1_sign64(data)
+        expect(signature).to be_a(String)
+        verification = STATIC_PRIVATE_KEY.public_key.verify(OpenSSL::Digest.new("SHA1"),
+                                                            Base64.strict_decode64(signature),
+                                                            data)
+        expect(verification).to be true
+      end
+    end
+  end
+
+end

spec/lib/postal_spec.rb

@@ -5,8 +5,14 @@ require "rails_helper"
 RSpec.describe Postal do
   describe "#signer" do
     it "returns a signer with the installation's signing key" do
-      expect(Postal.signer).to be_a(Signer)
+      expect(Postal.signer).to be_a(Postal::Signer)
       expect(Postal.signer.private_key.to_pem).to eq OpenSSL::PKey::RSA.new(File.read(Postal::Config.postal.signing_key_path)).to_pem
     end
   end
+
+  describe "#change_database_connection_pool_size" do
+    it "changes the connection pool size" do
+      expect { Postal.change_database_connection_pool_size(8) }.to change { ActiveRecord::Base.connection_pool.size }.from(5).to(8)
+    end
+  end
 end

spec/lib/signer_spec.rb

@@ -1,76 +0,0 @@
-# frozen_string_literal: true
-
-require "rails_helper"
-
-RSpec.describe Signer do
-  STATIC_PRIVATE_KEY = OpenSSL::PKey::RSA.new(2048) # rubocop:disable Lint/ConstantDefinitionInBlock
-
-  subject(:signer) { described_class.new(STATIC_PRIVATE_KEY) }
-
-  describe "#private_key" do
-    it "returns the private key" do
-      expect(signer.private_key).to eq(STATIC_PRIVATE_KEY)
-    end
-  end
-
-  describe "#public_key" do
-    it "returns the public key" do
-      expect(signer.public_key.to_s).to eq(STATIC_PRIVATE_KEY.public_key.to_s)
-    end
-  end
-
-  describe "#sign" do
-    it "returns a valid signature" do
-      data = "hello world!"
-      signature = signer.sign(data)
-      expect(signature).to be_a(String)
-      verification = STATIC_PRIVATE_KEY.public_key.verify(OpenSSL::Digest.new("SHA256"),
-                                                          signature,
-                                                          data)
-      expect(verification).to be true
-    end
-  end
-
-  describe "#sign64" do
-    it "returns a valid Base64-encoded signature" do
-      data = "hello world!"
-      signature = signer.sign64(data)
-      expect(signature).to be_a(String)
-      verification = STATIC_PRIVATE_KEY.public_key.verify(OpenSSL::Digest.new("SHA256"),
-                                                          Base64.strict_decode64(signature),
-                                                          data)
-      expect(verification).to be true
-    end
-  end
-
-  describe "#jwk" do
-    it "returns a valid JWK" do
-      jwk = signer.jwk
-      expect(jwk).to be_a(JWT::JWK::RSA)
-    end
-  end
-
-  describe "#sha1_sign" do
-    it "returns a valid signature" do
-      data = "hello world!"
-      signature = signer.sha1_sign(data)
-      expect(signature).to be_a(String)
-      verification = STATIC_PRIVATE_KEY.public_key.verify(OpenSSL::Digest.new("SHA1"),
-                                                          signature,
-                                                          data)
-      expect(verification).to be true
-    end
-  end
-
-  describe "#sha1_sign64" do
-    it "returns a valid Base64-encoded signature" do
-      data = "hello world!"
-      signature = signer.sha1_sign64(data)
-      expect(signature).to be_a(String)
-      verification = STATIC_PRIVATE_KEY.public_key.verify(OpenSSL::Digest.new("SHA1"),
-                                                          Base64.strict_decode64(signature),
-                                                          data)
-      expect(verification).to be true
-    end
-  end
-end

spec/lib/smtp_server/client/proxy_spec.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+module SMTPServer
+
+  describe Client do
+    let(:ip_address) { nil }
+    subject(:client) { described_class.new(ip_address) }
+
+    describe "PROXY" do
+      context "when the proxy header is sent correctly" do
+        it "sets the IP address" do
+          expect(client.handle("PROXY TCP4 1.1.1.1 2.2.2.2 1111 2222")).to eq "220 #{Postal::Config.postal.smtp_hostname} ESMTP Postal/#{client.trace_id}"
+          expect(client.ip_address).to eq "1.1.1.1"
+        end
+      end
+
+      context "when the proxy header is not valid" do
+        it "returns an error" do
+          expect(client.handle("PROXY TCP4")).to eq "502 Proxy Error"
+          expect(client.finished?).to be true
+        end
+      end
+    end
+  end
+
+end

spec/models/queued_message_spec.rb

@@ -1,5 +1,30 @@
 # frozen_string_literal: true
 
+# == Schema Information
+#
+# Table name: queued_messages
+#
+#  id            :integer          not null, primary key
+#  attempts      :integer          default(0)
+#  batch_key     :string(255)
+#  domain        :string(255)
+#  locked_at     :datetime
+#  locked_by     :string(255)
+#  manual        :boolean          default(FALSE)
+#  retry_after   :datetime
+#  created_at    :datetime
+#  updated_at    :datetime
+#  ip_address_id :integer
+#  message_id    :integer
+#  route_id      :integer
+#  server_id     :integer
+#
+# Indexes
+#
+#  index_queued_messages_on_domain      (domain)
+#  index_queued_messages_on_message_id  (message_id)
+#  index_queued_messages_on_server_id   (server_id)
+#
 require "rails_helper"
 
 RSpec.describe QueuedMessage do

spec/rails_helper.rb

@@ -37,9 +37,16 @@ RSpec.configure do |config|
   config.include FactoryBot::Syntax::Methods
   config.include GeneralHelpers
 
+  # Before all request specs, set the hostname to the web hostname for
+  # Postal otherwise it'll be www.example.com which will fail host
+  # authorization checks.
+  config.before(:each, type: :request) do
+    host! Postal::Config.postal.web_hostname
+  end
+
+  # Test that the factories are working as they should and then clean up before getting started on
+  # the rest of the suite.
   config.before(:suite) do
-    # Test that the factories are working as they should and then clean up before getting started on
-    # the rest of the suite.
     DatabaseCleaner.start
     FactoryBot.lint
   ensure

spec/senders/smtp_sender_spec.rb

@@ -524,4 +524,41 @@ RSpec.describe SMTPSender do
       expect(sender.endpoints).to all have_received(:finish_smtp_session).at_least(:once)
     end
   end
+
+  describe ".smtp_relays" do
+    before do
+      if described_class.instance_variable_defined?("@smtp_relays")
+        described_class.remove_instance_variable("@smtp_relays")
+      end
+    end
+
+    it "returns nil if smtp relays is nil" do
+      allow(Postal::Config.postal).to receive(:smtp_relays).and_return(nil)
+      expect(described_class.smtp_relays).to be nil
+    end
+
+    it "returns nil if there are no smtp relays" do
+      allow(Postal::Config.postal).to receive(:smtp_relays).and_return([])
+      expect(described_class.smtp_relays).to be nil
+    end
+
+    it "does not return relays where the host is nil" do
+      allow(Postal::Config.postal).to receive(:smtp_relays).and_return([
+                                                                         Hashie::Mash.new(host: nil, port: 25, ssl_mode: "Auto"),
+                                                                         Hashie::Mash.new(host: "test.example.com", port: 25, ssl_mode: "Auto"),
+                                                                       ])
+      expect(described_class.smtp_relays).to match [kind_of(SMTPClient::Server)]
+    end
+
+    it "returns relays with options" do
+      allow(Postal::Config.postal).to receive(:smtp_relays).and_return([
+                                                                         Hashie::Mash.new(host: "test.example.com", port: 25, ssl_mode: "Auto"),
+                                                                         Hashie::Mash.new(host: "test2.example.com", port: 2525, ssl_mode: "TLS"),
+                                                                       ])
+      expect(described_class.smtp_relays).to match [
+        have_attributes(hostname: "test.example.com", port: 25, ssl_mode: "Auto"),
+        have_attributes(hostname: "test2.example.com", port: 2525, ssl_mode: "TLS"),
+      ]
+    end
+  end
 end