chore(release): 2.1.0

closes #1652

CHANGELOG.md

@@ -2,6 +2,18 @@
 
 This file contains all the latest changes and updates to Postal.
 
+## 2.1.0
+
+### Features
+
+- support for configuring the default spam threshold values for new servers ([724325](https://github.com/postalserver/postal/commit/724325a1b97d61ef1e134240e4f70aaad39dbf98))
+- support for using rspamd for spam filtering ([a1277b](https://github.com/postalserver/postal/commit/a1277baba56ea6d6b4da4bba87b00cd3dbf0305e))
+
+### Bug Fixes
+
+- **dkim:** fixes timing race condition when signing emails ([232b60](https://github.com/postalserver/postal/commit/232b605f5bb8ab61156e1fb9860705fed017ed41))
+- **docker:** fixes issue caused by changes to underlying ruby:2.6 image ([6570ff](https://github.com/postalserver/postal/commit/6570ff1f7797ff9a307dd96ed4ff37be14bf79ab))
+
 ## 2.0.0
 
 ### Features

Dockerfile

@@ -1,11 +1,11 @@
-FROM ruby:2.6 AS base
+FROM ruby:2.6-buster AS base
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN apt-get update \
   && apt-get install -y --no-install-recommends \
-  software-properties-common \
-  && apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8 \
-  && add-apt-repository 'deb [arch=amd64,i386,ppc64el] http://mirrors.coreix.net/mariadb/repo/10.1/ubuntu xenial main' \
+  software-properties-common dirmngr apt-transport-https \
+  && apt-key adv --fetch-keys 'https://mariadb.org/mariadb_release_signing_key.asc' \
+  && add-apt-repository 'deb [arch=amd64,arm64,ppc64el] https://mirrors.xtom.nl/mariadb/repo/10.6/debian buster main' \
   && (curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add -) \
   && (echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list) \
   && (curl -sL https://deb.nodesource.com/setup_12.x | bash -) \

Gemfile.lock

@@ -132,7 +132,7 @@ GEM
     marcel (1.0.1)
     method_source (1.0.0)
     mini_mime (1.1.0)
-    mini_portile2 (2.5.3)
+    mini_portile2 (2.6.1)
     minitest (5.14.4)
     mongo (2.6.2)
       bson (>= 4.3.0, < 5.0.0)
@@ -146,13 +146,13 @@ GEM
     nilify_blanks (1.3.0)
       activerecord (>= 3.0.0)
       activesupport (>= 3.0.0)
-    nio4r (2.5.7)
-    nokogiri (1.11.7)
-      mini_portile2 (~> 2.5.0)
+    nio4r (2.5.8)
+    nokogiri (1.12.5)
+      mini_portile2 (~> 2.6.1)
       racc (~> 1.4)
-    puma (4.3.8)
+    puma (4.3.10)
       nio4r (~> 2.0)
-    racc (1.5.2)
+    racc (1.6.0)
     rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)

app/models/server.rb

@@ -69,8 +69,8 @@ class Server < ApplicationRecord
   default_value :raw_message_retention_days, -> { 30 }
   default_value :raw_message_retention_size, -> { 2048 }
   default_value :message_retention_days, -> { 60 }
-  default_value :spam_threshold, -> { 5.0 }
-  default_value :spam_failure_threshold, -> { 20.0 }
+  default_value :spam_threshold, -> { Postal.config.general.default_spam_threshold }
+  default_value :spam_failure_threshold, -> { Postal.config.general.default_spam_failure_threshold }
 
   validates :name, :presence => true, :uniqueness => {:scope => :organization_id}
   validates :mode, :inclusion => {:in => MODES}

app/views/track_domains/_form.html.haml

@@ -17,8 +17,8 @@
       .fieldSet__input
         = f.select :ssl_enabled, [["Yes - use SSL for tracking whenever possible", true], ["No - never use SSL for tracking", false]], {}, :class => 'input input--select'
         %p.fieldSet__text
-          If enabled, we'll try to remove the replies/signatures from the plain body and send them separately to the rest of the body.
-          This is useful if you just want to see the latest message in a thread.
+          If enabled, we'll use https for the tracking domain when replacing links and images. Please note that a SSL certificate 
+          should be installed on the tracking domain if enabled. 
 
     .fieldSet__field
       = f.label :track_loads, :class => 'fieldSet__label'

config/postal.defaults.yml

@@ -15,6 +15,8 @@ general:
   maximum_hold_expiry_days: 7
   suppression_list_removal_delay: 30
   use_local_ns_for_domains: false
+  default_spam_threshold: 5.0
+  default_spam_failure_threshold: 20.0
 
 web_server:
   bind_address: 127.0.0.1
@@ -100,6 +102,14 @@ rails:
   environment: production
   secret_key:
 
+rspamd:
+  enabled: false
+  host: 127.0.0.1
+  port: 11334
+  ssl: false
+  password: null
+  flags: null
+
 spamd:
   enabled: false
   host: 127.0.0.1

lib/postal.rb

@@ -15,6 +15,8 @@ module Postal
     autoload :Job
     autoload :MessageDB
     autoload :MessageInspection
+    autoload :MessageInspector
+    autoload :MessageInspectors
     autoload :MessageParser
     autoload :MessageRequeuer
     autoload :MXLookup
@@ -37,6 +39,7 @@ module Postal
     super
     Postal::MessageDB.eager_load!
     Postal::SMTPServer.eager_load!
+    Postal::MessageInspectors.eager_load!
   end
 
 end

lib/postal/dkim_header.rb

@@ -96,7 +96,7 @@ module Postal
     end
 
     def dkim_properties
-      Array.new.tap do |header|
+      @dkim_properties ||= Array.new.tap do |header|
         header << "a=rsa-sha256; c=relaxed/relaxed;"
         header << "d=#{@domain_name};"
         header << "s=#{@dkim_identifier}; t=#{Time.now.utc.to_i};"

lib/postal/message_db/message.rb

@@ -498,14 +498,16 @@ module Postal
       # Inspect this message
       #
       def inspect_message
-        if result = MessageInspection.new(self.raw_message, self.scope&.to_sym)
-          # Update the messages table with the results of our inspection
-          update(:inspected => 1, :spam_score => result.filtered_spam_score,  :threat => result.threat?, :threat_details => result.threat_message)
-          # Add any spam details into the spam checks database
-          self.database.insert_multi(:spam_checks, [:message_id, :code, :score, :description], result.filtered_spam_checks.map { |d| [self.id, d.code, d.score, d.description]})
-          # Return the result
-          result
-        end
+        result = MessageInspection.scan(self, self.scope&.to_sym)
+
+        # Update the messages table with the results of our inspection
+        update(:inspected => 1, :spam_score => result.spam_score,  :threat => result.threat?, :threat_details => result.threat_message)
+
+        # Add any spam details into the spam checks database
+        self.database.insert_multi(:spam_checks, [:message_id, :code, :score, :description], result.spam_checks.map { |d| [self.id, d.code, d.score, d.description] })
+
+        # Return the result
+        result
       end
 
       #

lib/postal/message_inspection.rb

@@ -1,140 +1,41 @@
-require 'timeout'
-require 'socket'
-require 'json'
-
 module Postal
   class MessageInspection
 
-    SPAM_EXCLUSIONS = {
-      :outgoing => ['NO_RECEIVED', 'NO_RELAYS', 'ALL_TRUSTED', 'FREEMAIL_FORGED_REPLYTO', 'RDNS_DYNAMIC', 'CK_HELO_GENERIC', /^SPF\_/, /^HELO\_/, /DKIM_/, /^RCVD_IN_/],
-      :incoming => []
-    }
+    attr_reader :message
+    attr_reader :scope
+    attr_reader :spam_checks
+    attr_accessor :threat
+    attr_accessor :threat_message
 
-    def initialize(message, scope = :incoming)
+    def initialize(message, scope)
       @message = message
       @scope = scope
-      @threat = false
-      @spam_score = 0.0
       @spam_checks = []
-
-      if Postal.config.spamd.enabled?
-        scan_for_spam
-      end
-
-      if Postal.config.clamav.enabled?
-        scan_for_threats
-      end
+      @threat = false
     end
 
     def spam_score
-      @spam_score
-    end
+      return 0 if @spam_checks.empty?
 
-    def spam_checks
-      @spam_checks
-    end
-
-    def filtered_spam_checks
-      @filtered_spam_checks ||= @spam_checks.reject do |check|
-        SPAM_EXCLUSIONS[@scope].any? do |item|
-          item == check.code || (item.is_a?(Regexp) && item =~ check.code)
-        end
-      end
-    end
-
-    def filtered_spam_score
-      filtered_spam_checks.inject(0.0) do |total, check|
-        total += check.score || 0.0
-      end.round(2)
+      @spam_checks.sum(&:score)
     end
 
     def threat?
-      @threat
+      @threat == true
     end
 
-    def threat_message
-      @threat_message
+    def scan
+      MessageInspector.inspectors.each do |inspector|
+        inspector.inspect_message(self)
+      end
     end
 
-    private
-
-    def scan_for_spam
-      data = nil
-      Timeout.timeout(15) do
-        tcp_socket = TCPSocket.new(Postal.config.spamd.host, Postal.config.spamd.port)
-        tcp_socket.write("REPORT SPAMC/1.2\r\n")
-        tcp_socket.write("Content-length: #{@message.bytesize}\r\n")
-        tcp_socket.write("\r\n")
-        tcp_socket.write(@message)
-        tcp_socket.close_write
-        data = tcp_socket.read
+    class << self
+      def scan(message, scope)
+        inspection = new(message, scope)
+        inspection.scan
+        inspection
       end
-
-      spam_checks = []
-      total = 0.0
-      rules = data ? data.split(/^---(.*)\r?\n/).last.split(/\r?\n/) : []
-      while line = rules.shift
-        if line =~ /\A([\- ]?[\d\.]+)\s+(\w+)\s+(.*)/
-          total += $1.to_f
-          spam_checks << SPAMCheck.new($2, $1.to_f, $3)
-        else
-          spam_checks.last.description << " " + line.strip
-        end
-      end
-
-      @spam_score = total.round(1)
-      @spam_checks = spam_checks
-
-    rescue Timeout::Error
-      @spam_checks = [SPAMCheck.new("TIMEOUT", 0, "Timed out when scanning for spam")]
-    rescue => e
-      logger.error "Error talking to spamd: #{e.class} (#{e.message})"
-      logger.error e.backtrace[0,5]
-      @spam_checks = [SPAMCheck.new("ERROR", 0, "Error when scanning for spam")]
-    ensure
-      tcp_socket.close rescue nil
-    end
-
-    def scan_for_threats
-      @threat = false
-
-      data = nil
-      Timeout.timeout(10) do
-        tcp_socket = TCPSocket.new(Postal.config.clamav.host, Postal.config.clamav.port)
-        tcp_socket.write("zINSTREAM\0")
-        tcp_socket.write([@message.bytesize].pack("N"))
-        tcp_socket.write(@message)
-        tcp_socket.write([0].pack("N"))
-        tcp_socket.close_write
-        data = tcp_socket.read
-      end
-
-      if data && data =~ /\Astream\:\s+(.*?)[\s\0]+?/
-        if $1.upcase == 'OK'
-          @threat = false
-          @threat_message = "No threats found"
-        else
-          @threat = true
-          @threat_message = $1
-        end
-      else
-        @threat = false
-        @threat_message = "Could not scan message"
-      end
-    rescue Timeout::Error
-      @threat = false
-      @threat_message = "Timed out scanning for threats"
-    rescue => e
-      logger.error "Error talking to clamav: #{e.class} (#{e.message})"
-      logger.error e.backtrace[0,5]
-      @threat = false
-      @threat_message = "Error when scanning for threats"
-    ensure
-      tcp_socket.close rescue nil
-    end
-
-    def logger
-      Postal.logger_for(:message_inspection)
     end
 
   end

lib/postal/message_inspector.rb

@@ -0,0 +1,40 @@
+module Postal
+  class MessageInspector
+
+    def initialize(config)
+      @config = config
+    end
+
+    # Inspect a message and update the inspection with the results
+    # as appropriate.
+    def inspect_message(message, scope, inspection)
+    end
+
+    private
+
+    def logger
+      Postal.logger_for(:message_inspection)
+    end
+
+    class << self
+      # Return an array of all inspectors that are available for this
+      # installation.
+      def inspectors
+        Array.new.tap do |inspectors|
+
+          if Postal.config.rspamd&.enabled
+            inspectors << MessageInspectors::Rspamd.new(Postal.config.rspamd)
+          elsif Postal.config.spamd&.enabled
+            inspectors << MessageInspectors::SpamAssassin.new(Postal.config.spamd)
+          end
+
+          if Postal.config.clamav&.enabled
+            inspectors << MessageInspectors::Clamav.new(Postal.config.clamav)
+          end
+
+        end
+      end
+    end
+
+  end
+end

lib/postal/message_inspectors.rb

@@ -0,0 +1,10 @@
+module Postal
+  module MessageInspectors
+    extend ActiveSupport::Autoload
+    eager_autoload do
+      autoload :Clamav
+      autoload :Rspamd
+      autoload :SpamAssassin
+    end
+  end
+end

lib/postal/message_inspectors/clamav.rb

@@ -0,0 +1,45 @@
+module Postal
+  module MessageInspectors
+    class Clamav < MessageInspector
+
+      def inspect_message(inspection)
+        raw_message = inspection.message.raw_message
+
+        data = nil
+        Timeout.timeout(10) do
+          tcp_socket = TCPSocket.new(@config.host, @config.port)
+          tcp_socket.write("zINSTREAM\0")
+          tcp_socket.write([raw_message.bytesize].pack("N"))
+          tcp_socket.write(raw_message)
+          tcp_socket.write([0].pack("N"))
+          tcp_socket.close_write
+          data = tcp_socket.read
+        end
+
+        if data && data =~ /\Astream\:\s+(.*?)[\s\0]+?/
+          if $1.upcase == 'OK'
+            inspection.threat = false
+            inspection.threat_message = "No threats found"
+          else
+            inspection.threat = true
+            inspection.threat_message = $1
+          end
+        else
+          inspection.threat = false
+          inspection.threat_message = "Could not scan message"
+        end
+      rescue Timeout::Error
+        inspection.threat = false
+        inspection.threat_message = "Timed out scanning for threats"
+      rescue => e
+        logger.error "Error talking to clamav: #{e.class} (#{e.message})"
+        logger.error e.backtrace[0,5]
+        inspection.threat = false
+        inspection.threat_message = "Error when scanning for threats"
+      ensure
+        tcp_socket.close rescue nil
+      end
+
+    end
+  end
+end

lib/postal/message_inspectors/rspamd.rb

@@ -0,0 +1,74 @@
+require 'net/http'
+
+module Postal
+  module MessageInspectors
+    class Rspamd < MessageInspector
+
+      class Error < StandardError
+      end
+
+      def inspect_message(inspection)
+        response = request(inspection.message, inspection.scope)
+        response = JSON.parse(response.body)
+        return unless response['symbols'].is_a?(Hash)
+
+        response['symbols'].values.each do |symbol|
+          next if symbol['description'].blank?
+
+          inspection.spam_checks << SpamCheck.new(symbol['name'], symbol['score'], symbol['description'])
+        end
+      rescue Error => e
+        inspection.spam_checks << SpamCheck.new("ERROR", 0, e.message)
+      end
+
+      private
+
+      def request(message, scope)
+        http = Net::HTTP.new(@config.host, @config.port)
+        http.use_ssl = true if @config.ssl
+        http.read_timeout = 10
+        http.open_timeout = 10
+
+        raw_message = message.raw_message
+
+        request = Net::HTTP::Post.new('/checkv2')
+        request.body = raw_message
+        request['Content-Length'] = raw_message.bytesize.to_s
+        request['Password'] = @config.password if @config.password
+        request['Flags'] = @config.flags if @config.flags
+        request['User-Agent'] = 'Postal'
+        request['Deliver-To'] = message.rcpt_to
+        request['From'] = message.mail_from
+        request['Rcpt'] = message.rcpt_to
+        request['Queue-Id'] = message.token
+
+        if scope == :outgoing
+          request['User'] = ''
+          # We don't actually know the IP but an empty input here will
+          # still trigger rspamd to treat this as an outbound email
+          # and disable certain checks.
+          # https://rspamd.com/doc/tutorials/scanning_outbound.html
+          request['Ip'] = ''
+        end
+
+        response = nil
+        begin
+          response = http.request(request)
+        rescue Exception => e
+          logger.error "Error talking to rspamd: #{e.class} (#{e.message})"
+          logger.error e.backtrace[0,5]
+
+          raise Error, "Error when scanning with rspamd (#{e.class})"
+        end
+
+        unless response.is_a?(Net::HTTPOK)
+          logger.info "Got #{response.code} status from rspamd, wanted 200"
+          raise Error, "Error when scanning with rspamd (got #{response.code})"
+        end
+
+        response
+      end
+
+    end
+  end
+end

lib/postal/message_inspectors/spam_assassin.rb

@@ -0,0 +1,54 @@
+module Postal
+  module MessageInspectors
+    class SpamAssassin < MessageInspector
+
+      EXCLUSIONS = {
+        :outgoing => ['NO_RECEIVED', 'NO_RELAYS', 'ALL_TRUSTED', 'FREEMAIL_FORGED_REPLYTO', 'RDNS_DYNAMIC', 'CK_HELO_GENERIC', /^SPF\_/, /^HELO\_/, /DKIM_/, /^RCVD_IN_/],
+        :incoming => []
+      }
+
+      def inspect_message(inspection)
+        data = nil
+        raw_message = inspection.message.raw_message
+        Timeout.timeout(15) do
+          tcp_socket = TCPSocket.new(@config.host, @config.port)
+          tcp_socket.write("REPORT SPAMC/1.2\r\n")
+          tcp_socket.write("Content-length: #{raw_message.bytesize}\r\n")
+          tcp_socket.write("\r\n")
+          tcp_socket.write(raw_message)
+          tcp_socket.close_write
+          data = tcp_socket.read
+        end
+
+        spam_checks = []
+        total = 0.0
+        rules = data ? data.split(/^---(.*)\r?\n/).last.split(/\r?\n/) : []
+        while line = rules.shift
+          if line =~ /\A([\- ]?[\d\.]+)\s+(\w+)\s+(.*)/
+            total += $1.to_f
+            spam_checks << SpamCheck.new($2, $1.to_f, $3)
+          else
+            spam_checks.last.description << " " + line.strip
+          end
+        end
+
+        checks = spam_checks.reject { |s| EXCLUSIONS[inspection.scope].include?(s.code) }
+        checks.each do |check|
+          inspection.spam_checks << check
+        end
+
+      rescue Timeout::Error
+        inspection.spam_checks << SpamCheck.new("TIMEOUT", 0, "Timed out when scanning for spam")
+
+      rescue => e
+        logger.error "Error talking to spamd: #{e.class} (#{e.message})"
+        logger.error e.backtrace[0,5]
+        inspection.spam_checks << SpamCheck.new("ERROR", 0, "Error when scanning for spam")
+
+      ensure
+        tcp_socket.close rescue nil
+      end
+
+    end
+  end
+end

lib/postal/spam_check.rb

@@ -1,5 +1,5 @@
 module Postal
-  class SPAMCheck
+  class SpamCheck
 
     attr_reader :code, :score, :description