From 044058d0f159f444c4c1d8d7765307bb64529ca4 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Thu, 1 Feb 2024 17:41:45 +0000 Subject: [PATCH 01/56] feat: more consistent logging --- Gemfile | 1 + Gemfile.lock | 6 +++ app/models/webhook_request.rb | 63 +++++++++++++++--------------- config/application.rb | 6 ++- config/cron.rb | 2 +- config/environments/production.rb | 6 --- config/initializers/logging.rb | 51 ++++++++++++++++++++++++ config/initializers/smtp.rb | 1 + config/postal.defaults.yml | 6 +-- docker/ci-config/postal.test.yml | 3 -- lib/postal/app_logger.rb | 64 ------------------------------- lib/postal/config.rb | 38 +++++++++--------- lib/postal/http_sender.rb | 2 +- lib/postal/job.rb | 2 +- lib/postal/message_db/database.rb | 10 ++--- lib/postal/message_inspector.rb | 2 +- lib/postal/message_requeuer.rb | 2 +- lib/postal/smtp_sender.rb | 2 +- lib/postal/smtp_server/client.rb | 20 +++++----- lib/postal/smtp_server/server.rb | 8 ++-- lib/postal/worker.rb | 36 ++++++++--------- 21 files changed, 160 insertions(+), 171 deletions(-) create mode 100644 config/initializers/logging.rb delete mode 100644 lib/postal/app_logger.rb diff --git a/Gemfile b/Gemfile index 31b5bef..c072b2e 100644 --- a/Gemfile +++ b/Gemfile @@ -20,6 +20,7 @@ gem "hashie" gem "highline", require: false gem "jwt" gem "kaminari" +gem "klogger-logger" gem "mail" gem "moonrope" gem "mysql2" diff --git a/Gemfile.lock b/Gemfile.lock index 83cf9a0..2344799 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -147,6 +147,10 @@ GEM activerecord kaminari-core (= 1.2.2) kaminari-core (1.2.2) + klogger-logger (1.3.2) + concurrent-ruby (>= 1.0, < 2.0) + json + rouge (>= 3.30, < 5.0) loofah (2.22.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -229,6 +233,7 @@ GEM regexp_parser (2.7.0) resolv (0.2.2) rexml (3.2.5) + rouge (4.2.0) rspec (3.12.0) rspec-core (~> 3.12.0) rspec-expectations (~> 3.12.0) @@ -345,6 +350,7 @@ DEPENDENCIES jquery-rails jwt kaminari + klogger-logger mail moonrope mysql2 diff --git a/app/models/webhook_request.rb b/app/models/webhook_request.rb index 97d4b6f..ada4e34 100644 --- a/app/models/webhook_request.rb +++ b/app/models/webhook_request.rb @@ -53,41 +53,42 @@ class WebhookRequest < ApplicationRecord end def deliver - logger = Postal.logger_for(:webhooks) payload = { event: event, timestamp: created_at.to_f, payload: self.payload, uuid: uuid }.to_json - logger.info "[#{id}] Sending webhook request to `#{url}`" - result = Postal::HTTP.post(url, sign: true, json: payload, timeout: 5) - self.attempts += 1 - self.retry_after = RETRIES[self.attempts]&.from_now - server.message_db.webhooks.record( - event: event, - url: url, - webhook_id: webhook_id, - attempt: self.attempts, - timestamp: Time.now.to_f, - payload: self.payload.to_json, - uuid: uuid, - status_code: result[:code], - body: result[:body], - will_retry: (retry_after ? 0 : 1) - ) + Postal.logger.tagged(event: event, url: url, component: "webhooks") do + Postal.logger.info "Sending webhook request" + result = Postal::HTTP.post(url, sign: true, json: payload, timeout: 5) + self.attempts += 1 + self.retry_after = RETRIES[self.attempts]&.from_now + server.message_db.webhooks.record( + event: event, + url: url, + webhook_id: webhook_id, + attempt: self.attempts, + timestamp: Time.now.to_f, + payload: self.payload.to_json, + uuid: uuid, + status_code: result[:code], + body: result[:body], + will_retry: (retry_after ? 0 : 1) + ) - if result[:code] >= 200 && result[:code] < 300 - logger.info "[#{id}] -> Received #{result[:code]} status code. That's OK." - destroy - webhook&.update_column(:last_used_at, Time.now) - true - else - logger.error "[#{id}] -> Received #{result[:code]} status code. That's not OK." - self.error = "Couldn't send to URL. Code received was #{result[:code]}" - if retry_after - logger.info "[#{id}] -> Will retry #{retry_after} (this was attempt #{self.attempts})" - save - else - logger.info "[#{id}] -> Have tried #{self.attempts} times. Giving up." + if result[:code] >= 200 && result[:code] < 300 + Postal.logger.info "Received #{result[:code]} status code. That's OK." destroy + webhook&.update_column(:last_used_at, Time.now) + true + else + Postal.logger.error "Received #{result[:code]} status code. That's not OK." + self.error = "Couldn't send to URL. Code received was #{result[:code]}" + if retry_after + Postal.logger.info "Will retry #{retry_after} (this was attempt #{self.attempts})" + save + else + Postal.logger.info "Have tried #{self.attempts} times. Giving up." + destroy + end + false end - false end end diff --git a/config/application.rb b/config/application.rb index d0fb313..12e8c27 100644 --- a/config/application.rb +++ b/config/application.rb @@ -38,9 +38,11 @@ module Postal require "postal/tracking_middleware" config.middleware.insert_before ActionDispatch::HostAuthorization, Postal::TrackingMiddleware - config.logger = Postal.logger_for(:rails) - config.hosts << Postal.config.web.host + if Postal.config.logging.rails_log == false + config.logger = Logger.new("/dev/null") + end + end end diff --git a/config/cron.rb b/config/cron.rb index ab18723..fed7a60 100644 --- a/config/cron.rb +++ b/config/cron.rb @@ -4,7 +4,7 @@ module Clockwork configure do |config| config[:tz] = "UTC" - config[:logger] = Postal.logger_for(:cron) + config[:logger] = Postal.logger end every 1.minute, "every-1-minutes" do diff --git a/config/environments/production.rb b/config/environments/production.rb index cca5f81..386e938 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -72,12 +72,6 @@ Rails.application.configure do # require 'syslog/logger' # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name') - if ENV["RAILS_LOG_TO_STDOUT"].present? - logger = ActiveSupport::Logger.new($stdout) - logger.formatter = config.log_formatter - config.logger = ActiveSupport::TaggedLogging.new(logger) - end - # Do not dump schema after migrations. config.active_record.dump_schema_after_migration = false end diff --git a/config/initializers/logging.rb b/config/initializers/logging.rb new file mode 100644 index 0000000..5108da4 --- /dev/null +++ b/config/initializers/logging.rb @@ -0,0 +1,51 @@ +begin + def add_exception_to_payload(payload, event) + return unless exception = event.payload[:exception_object] + + payload[:exception_class] = exception.class.name + payload[:exception_message] = exception.message + payload[:exception_backtrace] = exception.backtrace[0, 4].join("\n") + end + + ActiveSupport::Notifications.subscribe "process_action.action_controller" do |*args| + event = ActiveSupport::Notifications::Event.new(*args) + + payload = { + event: "request", + transaction: event.transaction_id, + controller: event.payload[:controller], + action: event.payload[:action], + format: event.payload[:format], + method: event.payload[:method], + path: event.payload[:path], + request_id: event.payload[:request].request_id, + ip_address: event.payload[:request].ip, + status: event.payload[:status], + view_runtime: event.payload[:view_runtime], + db_runtime: event.payload[:db_runtime] + } + + add_exception_to_payload(payload, event) + + string = "#{payload[:method]} #{payload[:path]} (#{payload[:status]})" + + if payload[:exception_class] + Postal.logger.error(string, **payload) + else + Postal.logger.info(string, **payload) + end + end + + ActiveSupport::Notifications.subscribe "deliver.action_mailer" do |*args| + event = ActiveSupport::Notifications::Event.new(*args) + + Postal.logger.info({ + event: "send_email", + transaction: event.transaction_id, + message_id: event.payload[:message_id], + subject: event.payload[:subject], + from: event.payload[:from], + to: event.payload[:to].is_a?(Array) ? event.payload[:to].join(", ") : event.payload[:to].to_s + }) + end +end diff --git a/config/initializers/smtp.rb b/config/initializers/smtp.rb index a9a1de2..dcc06a1 100644 --- a/config/initializers/smtp.rb +++ b/config/initializers/smtp.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require "postal/config" + if Postal.config&.smtp ActionMailer::Base.delivery_method = :smtp ActionMailer::Base.smtp_settings = { address: Postal.config.smtp.host, user_name: Postal.config.smtp.username, password: Postal.config.smtp.password, port: Postal.config.smtp.port || 25 } diff --git a/config/postal.defaults.yml b/config/postal.defaults.yml index dc65e34..2c9fa33 100644 --- a/config/postal.defaults.yml +++ b/config/postal.defaults.yml @@ -51,13 +51,11 @@ rabbitmq: tls_ca_certificates: <%= ENV.fetch('RABBITMQ_TLS_CA_CERTIFICATES', '/etc/ssl/certs/ca-certificates.crt'.split(',').inspect) %> logging: - stdout: <%= ENV.fetch('LOGGING_STDOUT', 'false') %> - root: <%= ENV.fetch('LOGGING_ROOT', '') %> - max_log_file_size: <%= ENV.fetch('LOGGING_MAX_LOG_FILES', '20') %> - max_log_files: <%= ENV.fetch('LOGGING_MAX_LOG_FILES', '10') %> + rails_log: <%= ENV.fetch('LOGGING_RAILS_LOG', 'false') %> graylog: host: <%= ENV.fetch('GRAYLOG_HOST', '') %> port: <%= ENV.fetch('GRAYLOG_PORT', '12201') %> + facility: <%= ENV.fetch('GRAYLOG_FACILITY', 'postal') %> workers: threads: <%= ENV.fetch('WORKER_THREADS', '4') %> diff --git a/docker/ci-config/postal.test.yml b/docker/ci-config/postal.test.yml index 982fcec..69922c9 100644 --- a/docker/ci-config/postal.test.yml +++ b/docker/ci-config/postal.test.yml @@ -8,9 +8,6 @@ web_server: smtp_server: port: 2525 -logging: - stdout: false - main_db: host: mariadb username: root diff --git a/lib/postal/app_logger.rb b/lib/postal/app_logger.rb deleted file mode 100644 index 13fd044..0000000 --- a/lib/postal/app_logger.rb +++ /dev/null @@ -1,64 +0,0 @@ -# frozen_string_literal: true - -require "logger" - -module Postal - - class AppLogger < Logger - - def initialize(log_name, *args) - @log_name = log_name - super(*args) - self.formatter = LogFormatter.new - end - - def add(severity, message = nil, progname = nil) - super - if severity >= @level && n = self.class.graylog_notifier - begin - if message.nil? - message = block_given? ? yield : progname - end - message = message.to_s.force_encoding("UTF-8").scrub - message_without_ansi = begin - message.gsub(/\e\[([\d;]+)?m/, "") - rescue StandardError - message - end - n.notify!(short_message: message_without_ansi, log_name: @log_name, facility: "postal", application_name: "postal", process_name: ENV.fetch("PROC_NAME", nil), pid: Process.pid) - rescue StandardError - # Can't log this to GELF. Soz. - end - end - true - end - - def self.graylog? - !!Postal.config.logging.graylog&.host - end - - def self.graylog_notifier - @graylog_notifier ||= graylog? ? GELF::Notifier.new(Postal.config.logging.graylog.host, Postal.config.logging.graylog.port) : nil - end - - end - - class LogFormatter - - TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%3N" - COLORS = [32, 34, 35, 31, 32, 33].freeze - - def call(severity, datetime, progname, msg) - time = datetime.strftime(TIME_FORMAT) - if number = ENV["PROC_NAME"] - id = number.split(".").last.to_i - proc_text = "\e[#{COLORS[id % COLORS.size]}m[#{ENV['PROC_NAME']}:#{Process.pid}]\e[0m" - else - proc_text = "[#{Process.pid}]" - end - "#{proc_text} [#{time}] #{severity} -- : #{msg}\n" - end - - end - -end diff --git a/lib/postal/config.rb b/lib/postal/config.rb index 6dd148d..1f1f42d 100644 --- a/lib/postal/config.rb +++ b/lib/postal/config.rb @@ -49,14 +49,6 @@ module Postal end end - def self.log_root - if config.logging.root - @log_root ||= Pathname.new(config.logging.root) - else - @log_root ||= app_root.join("log") - end - end - def self.config_file_path if env == "default" @config_file_path ||= File.join(config_root, "postal.yml") @@ -93,16 +85,11 @@ module Postal end end - def self.logger_for(name) - @loggers ||= {} - @loggers[name.to_sym] ||= begin - require "postal/app_logger" - if config.logging.stdout || ENV["LOG_TO_STDOUT"] - Postal::AppLogger.new(name, $stdout) - else - FileUtils.mkdir_p(log_root) - Postal::AppLogger.new(name, log_root.join("#{name}.log"), config.logging.max_log_files, config.logging.max_log_file_size.megabytes) - end + def self.logger + @logger ||= begin + k = Klogger.new(nil, destination: Rails.env.test? ? "/dev/null" : $stdout, highlight: Rails.env.development?) + k.add_destination(graylog_logging_destination) if config.logging&.graylog&.host.present? + k end end @@ -187,4 +174,19 @@ module Postal config.general.use_ip_pools? end + def self.graylog_logging_destination + @graylog_destination ||= begin + notifier = GELF::Notifier.new(config.logging.graylog.host, config.logging.graylog.port, "WAN") + proc do |_logger, payload, group_ids| + short_message = payload.delete(:message) || "[message missing]" + notifier.notify!(short_message: short_message, **{ + facility: config.logging.graylog.facility, + _environment: Rails.env.to_s, + _version: Postal::VERSION.to_s, + _group_ids: group_ids.join(" ") + }.merge(payload.transform_keys { |k| "_#{k}".to_sym }.transform_values(&:to_s))) + end + end + end + end diff --git a/lib/postal/http_sender.rb b/lib/postal/http_sender.rb index 0de927f..e07a637 100644 --- a/lib/postal/http_sender.rb +++ b/lib/postal/http_sender.rb @@ -61,7 +61,7 @@ module Postal private def log(text) - Postal.logger_for(:http_sender).info("[#{@log_id}] #{text}") + Postal.logger.info text, id: @log_id, component: "http-sender" end def parameters(message, options = {}) diff --git a/lib/postal/job.rb b/lib/postal/job.rb index 32759cb..1311d1f 100644 --- a/lib/postal/job.rb +++ b/lib/postal/job.rb @@ -30,7 +30,7 @@ module Postal end def log(text) - Worker.logger.info "[#{@id}] #{text}" + Worker.logger.info(text) end def self.queue(queue, params = {}) diff --git a/lib/postal/message_db/database.rb b/lib/postal/message_db/database.rb index bdf0222..061e1e4 100644 --- a/lib/postal/message_db/database.rb +++ b/lib/postal/message_db/database.rb @@ -325,12 +325,12 @@ module Postal result = connection.query(query, cast_booleans: true) time = Time.now.to_f - start_time logger.debug " \e[4;34mMessageDB Query (#{time.round(2)}s) \e[0m \e[33m#{query}\e[0m" - if time > 0.5 && query =~ /\A(SELECT|UPDATE|DELETE) / + if time.positive? && query =~ /\A(SELECT|UPDATE|DELETE) / id = Nifty::Utils::RandomString.generate(length: 6).upcase explain_result = ResultForExplainPrinter.new(connection.query("EXPLAIN #{query}")) - slow_query_logger.info "[#{id}] EXPLAIN #{query}" + logger.info " [#{id}] EXPLAIN #{query}" ActiveRecord::ConnectionAdapters::MySQL::ExplainPrettyPrinter.new.pp(explain_result, time).split("\n").each do |line| - slow_query_logger.info "[#{id}] " + line + logger.info " [#{id}] " + line end end result @@ -340,10 +340,6 @@ module Postal defined?(Rails) ? Rails.logger : Logger.new($stdout) end - def slow_query_logger - Postal.logger_for(:slow_message_db_queries) - end - def with_mysql(&block) self.class.connection_pool.use(&block) end diff --git a/lib/postal/message_inspector.rb b/lib/postal/message_inspector.rb index 5eeadc8..2b01a49 100644 --- a/lib/postal/message_inspector.rb +++ b/lib/postal/message_inspector.rb @@ -15,7 +15,7 @@ module Postal private def logger - Postal.logger_for(:message_inspection) + Postal.logger end class << self diff --git a/lib/postal/message_requeuer.rb b/lib/postal/message_requeuer.rb index a42732f..31e5510 100644 --- a/lib/postal/message_requeuer.rb +++ b/lib/postal/message_requeuer.rb @@ -20,7 +20,7 @@ module Postal private def log(text) - Postal.logger_for(:message_requeuer).info text + Postal.logger.info text, component: "message-requeuer" end def check_exit diff --git a/lib/postal/smtp_sender.rb b/lib/postal/smtp_sender.rb index e158390..764b057 100644 --- a/lib/postal/smtp_sender.rb +++ b/lib/postal/smtp_sender.rb @@ -234,7 +234,7 @@ module Postal end def log(text) - Postal.logger_for(:smtp_sender).info "[#{@log_id}] #{text}" + Postal.logger.info text, id: @log_id, component: "smtp-sender" end def destination_host_description diff --git a/lib/postal/smtp_server/client.rb b/lib/postal/smtp_server/client.rb index 790e7dc..6aedcfc 100644 --- a/lib/postal/smtp_server/client.rb +++ b/lib/postal/smtp_server/client.rb @@ -48,16 +48,18 @@ module Postal end def handle(data) - if @state == :preauth - return proxy(data) - end + Postal.logger.tagged(id: id) do + if @state == :preauth + return proxy(data) + end - log "\e[32m<= #{sanitize_input_for_log(data.strip)}\e[0m" - if @proc - @proc.call(data) + log "\e[32m<= #{sanitize_input_for_log(data.strip)}\e[0m" + if @proc + @proc.call(data) - else - handle_command(data) + else + handle_command(data) + end end end @@ -93,7 +95,7 @@ module Postal def log(text) return false unless @logging_enabled - Postal.logger_for(:smtp_server).debug "[#{id}] #{text}" + Postal.logger.debug(text, id: id) end private diff --git a/lib/postal/smtp_server/server.rb b/lib/postal/smtp_server/server.rb index bd7cd74..3a110a0 100644 --- a/lib/postal/smtp_server/server.rb +++ b/lib/postal/smtp_server/server.rb @@ -14,8 +14,10 @@ module Postal end def run - listen - run_event_loop + logger.tagged(component: "smtp-server") do + listen + run_event_loop + end end private @@ -264,7 +266,7 @@ module Postal end def logger - Postal.logger_for(:smtp_server) + Postal.logger end end diff --git a/lib/postal/worker.rb b/lib/postal/worker.rb index 7d5f858..969d81e 100644 --- a/lib/postal/worker.rb +++ b/lib/postal/worker.rb @@ -49,33 +49,25 @@ module Postal private - def receive_job(delivery_info, properties, body) - message = begin - JSON.parse(body) - rescue StandardError - nil - end + def receive_job(delivery_info, properties, message) if message && message["class_name"] @running_jobs << message["id"] set_process_name start_time = Time.now Thread.current[:job_id] = message["id"] - logger.info "[#{message['id']}] Started processing \e[34m#{message['class_name']}\e[0m job" + logger.info "Processing job" begin klass = message["class_name"].constantize.new(message["id"], message["params"]) klass.perform GC.start rescue StandardError => e klass.on_error(e) if defined?(klass) - logger.warn "[#{message['id']}] \e[31m#{e.class}: #{e.message}\e[0m" - e.backtrace.each do |line| - logger.warn "[#{message['id']}] " + line - end + logger.exception(e) if defined?(Sentry) Sentry.capture_exception(e, extra: { job_id: message["id"] }) end ensure - logger.info "[#{message['id']}] Finished processing \e[34m#{message['class_name']}\e[0m job in #{Time.now - start_time}s" + logger.info "Finished job", time: (Time.now - start_time).to_i end end ensure @@ -92,13 +84,21 @@ module Postal def join_queue(queue) if @active_queues[queue] - logger.info "Attempted to join queue #{queue} but already joined." + logger.error "attempted to join queue but already joined", queue: queue else consumer = self.class.job_queue(queue).subscribe(manual_ack: true) do |delivery_info, properties, body| - receive_job(delivery_info, properties, body) + message = begin + JSON.parse(body) + rescue StandardError + nil + end + + logger.tagged(job_id: message["id"], queue: queue, job_class: message["class_name"]) do + receive_job(delivery_info, properties, message) + end end @active_queues[queue] = consumer - logger.info "Joined \e[32m#{queue}\e[0m queue" + logger.info "joined queue", queue: queue end end @@ -106,9 +106,9 @@ module Postal if consumer = @active_queues[queue] consumer.cancel @active_queues.delete(queue) - logger.info "Left \e[32m#{queue}\e[0m queue" + logger.info "left queue", queue: queue else - logger.info "Not joined #{queue} so cannot leave" + logger.error "requested to leave queue, but not joined", queue: queue end end @@ -198,7 +198,7 @@ module Postal class << self def logger - Postal.logger_for(:worker) + Postal.logger end def job_channel From dc8e895bfedd08e97f4da40783df4ce1d30d9108 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Wed, 14 Feb 2024 13:46:04 +0000 Subject: [PATCH 02/56] feat: new background work process This removes all previous dependencies on RabbitMQ and the need to run separate cron and requeueing processes. --- .rubocop.yml | 1 + Gemfile | 3 +- Gemfile.lock | 27 +- Procfile.dev | 4 +- app/controllers/messages_controller.rb | 2 +- app/jobs/action_deletion_job.rb | 16 - app/jobs/action_deletions_job.rb | 17 - app/jobs/prune_suppression_lists_job.rb | 12 - app/jobs/prune_webhook_requests_job.rb | 12 - app/jobs/requeue_webhooks_job.rb | 9 - app/jobs/send_notifications_job.rb | 9 - app/jobs/send_webhook_job.rb | 29 -- app/jobs/sleep_job.rb | 9 - app/jobs/tidy_raw_messages_job.rb | 8 - app/jobs/unqueue_message_job.rb | 468 ----------------- app/jobs/webhook_delivery_job.rb | 17 - app/models/concerns/has_locking.rb | 47 ++ app/models/concerns/has_soft_destroy.rb | 1 - app/models/queued_message.rb | 63 +-- app/models/scheduled_task.rb | 16 + app/models/server.rb | 4 +- app/models/webhook_request.rb | 69 +-- app/models/worker_role.rb | 54 ++ .../action_deletions_scheduled_task.rb | 17 + .../application_scheduled_task.rb | 46 ++ .../check_all_dns_scheduled_task.rb} | 8 +- ...cleanup_authie_sessions_scheduled_task.rb} | 4 +- .../expire_held_messages_scheduled_task.rb} | 4 +- ...ocess_message_retention_scheduled_task.rb} | 12 +- .../prune_suppression_lists_scheduled_task.rb | 16 + .../prune_webhook_requests_scheduled_task.rb | 16 + .../send_notifications_scheduled_task.rb | 13 + app/services/unqueue_message_service.rb | 487 ++++++++++++++++++ app/services/webhook_delivery_service.rb | 19 + bin/postal | 12 +- config/cron.rb | 30 -- config/initializers/logging.rb | 2 + config/postal.defaults.yml | 13 - .../20240213165450_create_worker_roles.rb | 14 + .../20240213171830_create_scheduled_tasks.rb | 13 + ...253_add_lock_fields_to_webhook_requests.rb | 12 + db/schema.rb | 18 +- docker-compose.yml | 6 - docker/ci-config/postal.test.yml | 6 - lib/postal/config.rb | 10 +- lib/postal/job.rb | 44 -- lib/postal/message_db/message.rb | 13 +- lib/postal/message_requeuer.rb | 34 -- lib/postal/rabbit_mq.rb | 38 -- lib/postal/tracking_middleware.rb | 24 +- lib/postal/worker.rb | 220 -------- lib/tasks/postal.rake | 21 - lib/worker/jobs/base_job.rb | 29 ++ .../jobs/process_queued_messages_job.rb | 73 +++ .../jobs/process_webhook_requests_job.rb | 48 ++ lib/worker/process.rb | 242 +++++++++ script/send_html_email.rb | 21 +- script/smtp_server.rb | 4 + script/worker.rb | 2 +- spec/app/models/worker_role_spec.rb | 58 +++ spec/factories/ip_address_factory.rb | 23 + spec/factories/ip_pool_factory.rb | 23 + spec/factories/queued_message_factory.rb | 40 ++ spec/factories/webhook_factory.rb | 10 + spec/factories/webhook_request_factory.rb | 41 ++ spec/factories/worker_role_factory.rb | 7 + .../jobs/process_queued_messages_job.rb | 114 ++++ .../jobs/process_webhook_requests_job.rb | 56 ++ spec/rails_helper.rb | 1 + 69 files changed, 1675 insertions(+), 1186 deletions(-) delete mode 100644 app/jobs/action_deletion_job.rb delete mode 100644 app/jobs/action_deletions_job.rb delete mode 100644 app/jobs/prune_suppression_lists_job.rb delete mode 100644 app/jobs/prune_webhook_requests_job.rb delete mode 100644 app/jobs/requeue_webhooks_job.rb delete mode 100644 app/jobs/send_notifications_job.rb delete mode 100644 app/jobs/send_webhook_job.rb delete mode 100644 app/jobs/sleep_job.rb delete mode 100644 app/jobs/tidy_raw_messages_job.rb delete mode 100644 app/jobs/unqueue_message_job.rb delete mode 100644 app/jobs/webhook_delivery_job.rb create mode 100644 app/models/concerns/has_locking.rb create mode 100644 app/models/scheduled_task.rb create mode 100644 app/models/worker_role.rb create mode 100644 app/scheduled_tasks/action_deletions_scheduled_task.rb create mode 100644 app/scheduled_tasks/application_scheduled_task.rb rename app/{jobs/check_all_dns_job.rb => scheduled_tasks/check_all_dns_scheduled_task.rb} (62%) rename app/{jobs/cleanup_authie_sessions_job.rb => scheduled_tasks/cleanup_authie_sessions_scheduled_task.rb} (55%) rename app/{jobs/expire_held_messages_job.rb => scheduled_tasks/expire_held_messages_scheduled_task.rb} (77%) rename app/{jobs/process_message_retention_job.rb => scheduled_tasks/process_message_retention_scheduled_task.rb} (55%) create mode 100644 app/scheduled_tasks/prune_suppression_lists_scheduled_task.rb create mode 100644 app/scheduled_tasks/prune_webhook_requests_scheduled_task.rb create mode 100644 app/scheduled_tasks/send_notifications_scheduled_task.rb create mode 100644 app/services/unqueue_message_service.rb create mode 100644 app/services/webhook_delivery_service.rb delete mode 100644 config/cron.rb create mode 100644 db/migrate/20240213165450_create_worker_roles.rb create mode 100644 db/migrate/20240213171830_create_scheduled_tasks.rb create mode 100644 db/migrate/20240214132253_add_lock_fields_to_webhook_requests.rb delete mode 100644 lib/postal/job.rb delete mode 100644 lib/postal/message_requeuer.rb delete mode 100644 lib/postal/rabbit_mq.rb delete mode 100644 lib/postal/worker.rb create mode 100644 lib/worker/jobs/base_job.rb create mode 100644 lib/worker/jobs/process_queued_messages_job.rb create mode 100644 lib/worker/jobs/process_webhook_requests_job.rb create mode 100644 lib/worker/process.rb create mode 100644 script/smtp_server.rb create mode 100644 spec/app/models/worker_role_spec.rb create mode 100644 spec/factories/ip_address_factory.rb create mode 100644 spec/factories/ip_pool_factory.rb create mode 100644 spec/factories/queued_message_factory.rb create mode 100644 spec/factories/webhook_factory.rb create mode 100644 spec/factories/webhook_request_factory.rb create mode 100644 spec/factories/worker_role_factory.rb create mode 100644 spec/lib/worker/jobs/process_queued_messages_job.rb create mode 100644 spec/lib/worker/jobs/process_webhook_requests_job.rb diff --git a/.rubocop.yml b/.rubocop.yml index 25bb950..17ab0f2 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -6,6 +6,7 @@ AllCops: - "db/schema.rb" # Fixes missing gem exception when running Rubocop on GitHub Actions. - "vendor/bundle/**/*" + - lib/tasks/auto_annotate_models.rake # Always use double quotes Style/StringLiterals: diff --git a/Gemfile b/Gemfile index c072b2e..9772788 100644 --- a/Gemfile +++ b/Gemfile @@ -5,10 +5,8 @@ gem "authie" gem "autoprefixer-rails" gem "basic_ssl" gem "bcrypt" -gem "bunny" gem "changey" gem "chronic" -gem "clockwork" gem "dotenv-rails" gem "dynamic_form" gem "encrypto_signo" @@ -55,4 +53,5 @@ group :development do gem "rubocop" gem "rubocop-rails" gem "timecop" + gem "webmock" end diff --git a/Gemfile.lock b/Gemfile.lock index 2344799..2471753 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -60,7 +60,8 @@ GEM minitest (>= 5.1) tzinfo (~> 2.0) zeitwerk (~> 2.3) - amq-protocol (2.3.2) + addressable (2.8.6) + public_suffix (>= 2.0.2, < 6.0) annotate (3.2.0) activerecord (>= 3.2, < 8.0) rake (>= 10.4, < 14.0) @@ -71,17 +72,12 @@ GEM execjs (~> 2) basic_ssl (1.0.3) bcrypt (3.1.18) + bigdecimal (3.1.6) builder (3.2.4) - bunny (2.20.3) - amq-protocol (~> 2.3, >= 2.3.1) - sorted_set (~> 1, >= 1.0.2) byebug (11.1.3) changey (1.1.0) activerecord (>= 4.2, < 7) chronic (0.10.2) - clockwork (3.0.2) - activesupport - tzinfo coffee-rails (5.0.0) coffee-script (>= 2.2.0) railties (>= 5.2.0) @@ -90,6 +86,9 @@ GEM execjs coffee-script-source (1.12.2) concurrent-ruby (1.2.3) + crack (1.0.0) + bigdecimal + rexml crass (1.0.6) database_cleaner (2.0.2) database_cleaner-active_record (>= 2, < 3) @@ -125,6 +124,7 @@ GEM temple (>= 0.8.2) thor tilt + hashdiff (1.1.0) hashie (5.0.0) highline (2.1.0) i18n (1.14.1) @@ -193,6 +193,7 @@ GEM parallel (1.22.1) parser (3.2.1.1) ast (~> 2.4.1) + public_suffix (5.0.4) puma (6.4.2) nio4r (~> 2.0) racc (1.7.3) @@ -229,7 +230,6 @@ GEM thor (~> 1.0) rainbow (3.1.1) rake (13.1.0) - rbtree (0.4.6) regexp_parser (2.7.0) resolv (0.2.2) rexml (3.2.5) @@ -289,10 +289,6 @@ GEM sentry-ruby (~> 5.8.0) sentry-ruby (5.8.0) concurrent-ruby (~> 1.0, >= 1.0.2) - set (1.0.3) - sorted_set (1.0.3) - rbtree - set (~> 1.0) sprockets (4.2.0) concurrent-ruby (~> 1.0) rack (>= 2.2.4, < 4) @@ -313,6 +309,10 @@ GEM uglifier (4.2.0) execjs (>= 0.3.0, < 3) unicode-display_width (2.4.2) + webmock (3.20.0) + addressable (>= 2.8.0) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) websocket-driver (0.7.6) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) @@ -330,11 +330,9 @@ DEPENDENCIES autoprefixer-rails basic_ssl bcrypt - bunny byebug changey chronic - clockwork coffee-rails (~> 5.0) database_cleaner dotenv-rails @@ -371,6 +369,7 @@ DEPENDENCIES timecop turbolinks (~> 5) uglifier (>= 1.3.0) + webmock BUNDLED WITH 2.4.9 diff --git a/Procfile.dev b/Procfile.dev index 9f0e14a..6fd1dd7 100644 --- a/Procfile.dev +++ b/Procfile.dev @@ -1,5 +1,3 @@ web: bundle exec puma -C config/puma.rb worker: bundle exec ruby script/worker.rb -cron: bundle exec rake postal:cron -smtp: bundle exec rake postal:smtp_server -requeuer: bundle exec rake postal:requeuer +smtp: bundle exec ruby script/smtp_server.rb diff --git a/app/controllers/messages_controller.rb b/app/controllers/messages_controller.rb index 6a6c2f1..c028cdb 100644 --- a/app/controllers/messages_controller.rb +++ b/app/controllers/messages_controller.rb @@ -116,7 +116,7 @@ class MessagesController < ApplicationController def retry if @message.raw_message? if @message.queued_message - @message.queued_message.queue! + @message.queued_message.retry_now flash[:notice] = "This message will be retried shortly." elsif @message.held? @message.add_to_message_queue(manual: true) diff --git a/app/jobs/action_deletion_job.rb b/app/jobs/action_deletion_job.rb deleted file mode 100644 index e598a85..0000000 --- a/app/jobs/action_deletion_job.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -class ActionDeletionJob < Postal::Job - - def perform - object = params["type"].constantize.deleted.find_by_id(params["id"]) - if object - log "Deleting #{params['type']}##{params['id']}" - object.destroy - log "Deleted #{params['type']}##{params['id']}" - else - log "Couldn't find deleted object #{params['type']}##{params['id']}" - end - end - -end diff --git a/app/jobs/action_deletions_job.rb b/app/jobs/action_deletions_job.rb deleted file mode 100644 index f68b1d9..0000000 --- a/app/jobs/action_deletions_job.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -class ActionDeletionsJob < Postal::Job - - def perform - Organization.deleted.each do |org| - log "Permanently removing organization #{org.id} (#{org.permalink})" - org.destroy - end - - Server.deleted.each do |server| - log "Permanently removing server #{server.id} (#{server.full_permalink})" - server.destroy - end - end - -end diff --git a/app/jobs/prune_suppression_lists_job.rb b/app/jobs/prune_suppression_lists_job.rb deleted file mode 100644 index ece9db7..0000000 --- a/app/jobs/prune_suppression_lists_job.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -class PruneSuppressionListsJob < Postal::Job - - def perform - Server.all.each do |s| - log "Pruning suppression lists for server #{s.id}" - s.message_db.suppression_list.prune - end - end - -end diff --git a/app/jobs/prune_webhook_requests_job.rb b/app/jobs/prune_webhook_requests_job.rb deleted file mode 100644 index 632bc4a..0000000 --- a/app/jobs/prune_webhook_requests_job.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -class PruneWebhookRequestsJob < Postal::Job - - def perform - Server.all.each do |s| - log "Pruning webhook requests for server #{s.id}" - s.message_db.webhooks.prune - end - end - -end diff --git a/app/jobs/requeue_webhooks_job.rb b/app/jobs/requeue_webhooks_job.rb deleted file mode 100644 index 736f0b6..0000000 --- a/app/jobs/requeue_webhooks_job.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -class RequeueWebhooksJob < Postal::Job - - def perform - WebhookRequest.requeue_all - end - -end diff --git a/app/jobs/send_notifications_job.rb b/app/jobs/send_notifications_job.rb deleted file mode 100644 index 9d519c6..0000000 --- a/app/jobs/send_notifications_job.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -class SendNotificationsJob < Postal::Job - - def perform - Server.send_send_limit_notifications - end - -end diff --git a/app/jobs/send_webhook_job.rb b/app/jobs/send_webhook_job.rb deleted file mode 100644 index 8228563..0000000 --- a/app/jobs/send_webhook_job.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -class SendWebhookJob < Postal::Job - - def perform - if server = Server.find(params["server_id"]) - new_items = {} - params["payload"]&.each do |key, value| - next unless key.to_s =~ /\A_(\w+)/ - - begin - new_items[::Regexp.last_match(1)] = server.message_db.message(value.to_i).webhook_hash - rescue Postal::MessageDB::Message::NotFound - # No message found, don't do any replacement - end - end - - new_items.each do |key, value| - params["payload"].delete("_#{key}") - params["payload"][key] = value - end - - WebhookRequest.trigger(server, params["event"], params["payload"]) - else - log "Couldn't find server with ID #{params['server_id']}" - end - end - -end diff --git a/app/jobs/sleep_job.rb b/app/jobs/sleep_job.rb deleted file mode 100644 index 9604b76..0000000 --- a/app/jobs/sleep_job.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -class SleepJob < Postal::Job - - def perform - sleep 5 - end - -end diff --git a/app/jobs/tidy_raw_messages_job.rb b/app/jobs/tidy_raw_messages_job.rb deleted file mode 100644 index 151188f..0000000 --- a/app/jobs/tidy_raw_messages_job.rb +++ /dev/null @@ -1,8 +0,0 @@ -# frozen_string_literal: true - -class TidyRawMessagesJob < Postal::Job - - def perform - end - -end diff --git a/app/jobs/unqueue_message_job.rb b/app/jobs/unqueue_message_job.rb deleted file mode 100644 index 809ecfb..0000000 --- a/app/jobs/unqueue_message_job.rb +++ /dev/null @@ -1,468 +0,0 @@ -# frozen_string_literal: true - -class UnqueueMessageJob < Postal::Job - - # rubocop:disable Layout/LineLength - def perform - if original_message = QueuedMessage.find_by_id(params["id"]) - if original_message.acquire_lock - - log "Lock acquired for queued message #{original_message.id}" - - begin - original_message.message - rescue Postal::MessageDB::Message::NotFound - log "Unqueue #{original_message.id} because backend message has been removed." - original_message.destroy - return - end - - unless original_message.retriable? - log "Skipping because retry after isn't reached" - original_message.unlock - return - end - - begin - other_messages = original_message.batchable_messages(100) - log "Found #{other_messages.size} associated messages to process at the same time (batch key: #{original_message.batch_key})" - rescue StandardError - original_message.unlock - raise - end - - ([original_message] + other_messages).each do |queued_message| - log_prefix = "[#{queued_message.server_id}::#{queued_message.message_id} #{queued_message.id}]" - begin - log "#{log_prefix} Got queued message with exclusive lock" - - begin - queued_message.message - rescue Postal::MessageDB::Message::NotFound - log "#{log_prefix} Unqueueing #{queued_message.id} because backend message has been removed" - queued_message.destroy - next - end - - # - # If the server is suspended, hold all messages - # - if queued_message.server.suspended? - log "#{log_prefix} Server is suspended. Holding message." - queued_message.message.create_delivery("Held", details: "Mail server has been suspended. No e-mails can be processed at present. Contact support for assistance.") - queued_message.destroy - next - end - - # We might not be able to send this any more, check the attempts - if queued_message.attempts >= Postal.config.general.maximum_delivery_attempts - details = "Maximum number of delivery attempts (#{queued_message.attempts}) has been reached." - if queued_message.message.scope == "incoming" - # Send bounces to incoming e-mails when they are hard failed - if bounce_id = queued_message.send_bounce - details += " Bounce sent to sender (see message )" - end - elsif queued_message.message.scope == "outgoing" - # Add the recipient to the suppression list - if queued_message.server.message_db.suppression_list.add(:recipient, queued_message.message.rcpt_to, reason: "too many soft fails") - log "Added #{queued_message.message.rcpt_to} to suppression list because maximum attempts has been reached" - details += " Added #{queued_message.message.rcpt_to} to suppression list because delivery has failed #{queued_message.attempts} times." - end - end - queued_message.message.create_delivery("HardFail", details: details) - queued_message.destroy - log "#{log_prefix} Message has reached maximum number of attempts. Hard failing." - next - end - - # If the raw message has been removed (removed by retention) - unless queued_message.message.raw_message? - log "#{log_prefix} Raw message has been removed. Not sending." - queued_message.message.create_delivery("HardFail", details: "Raw message has been removed. Cannot send message.") - queued_message.destroy - next - end - - # - #  Handle Incoming Messages - # - if queued_message.message.scope == "incoming" - # - # If this is a bounce, we need to handle it as such - # - if queued_message.message.bounce - log "#{log_prefix} Message is a bounce" - original_messages = queued_message.message.original_messages - unless original_messages.empty? - queued_message.message.original_messages.each do |orig_msg| - queued_message.message.update(bounce_for_id: orig_msg.id, domain_id: orig_msg.domain_id) - queued_message.message.create_delivery("Processed", details: "This has been detected as a bounce message for .") - orig_msg.bounce!(queued_message.message) - log "#{log_prefix} Bounce linked with message #{orig_msg.id}" - end - queued_message.destroy - next - end - - # This message was sent to the return path but hasn't been matched - #  to an original message. If we have a route for this, route it - #  otherwise we'll drop at this point. - if queued_message.message.route_id.nil? - log "#{log_prefix} No source messages found. Hard failing." - queued_message.message.create_delivery("HardFail", details: "This message was a bounce but we couldn't link it with any outgoing message and there was no route for it.") - queued_message.destroy - next - end - end - - # - # Update live stats - # - queued_message.message.database.live_stats.increment(queued_message.message.scope) - - # - # Inspect incoming messages - # - unless queued_message.message.inspected - log "#{log_prefix} Inspecting message" - queued_message.message.inspect_message - if queued_message.message.inspected - is_spam = queued_message.message.spam_score > queued_message.server.spam_threshold - queued_message.message.update(spam: true) if is_spam - queued_message.message.append_headers( - "X-Postal-Spam: #{queued_message.message.spam ? 'yes' : 'no'}", - "X-Postal-Spam-Threshold: #{queued_message.server.spam_threshold}", - "X-Postal-Spam-Score: #{queued_message.message.spam_score}", - "X-Postal-Threat: #{queued_message.message.threat ? 'yes' : 'no'}" - ) - log "#{log_prefix} Message inspected successfully. Headers added." - end - end - - # - # If this message has a SPAM score higher than is permitted - # - if queued_message.message.spam_score >= queued_message.server.spam_failure_threshold - log "#{log_prefix} Message has a spam score higher than the server's maxmimum. Hard failing." - queued_message.message.create_delivery("HardFail", details: "Message's spam score is higher than the failure threshold for this server. Threshold is currently #{queued_message.server.spam_failure_threshold}.") - queued_message.destroy - next - end - - # If the server is in development mode, hold it - if queued_message.server.mode == "Development" && !queued_message.manual? - log "Server is in development mode so holding." - queued_message.message.create_delivery("Held", details: "Server is in development mode.") - queued_message.destroy - log "#{log_prefix} Server is in development mode. Holding." - next - end - - # - # Find out what sort of message we're supposed to be sending and dispatch this request over to - # the sender. - # - if route = queued_message.message.route - - # If the route says we're holding quananteed mail and this is spam, we'll hold this - if route.spam_mode == "Quarantine" && queued_message.message.spam && !queued_message.manual? - queued_message.message.create_delivery("Held", details: "Message placed into quarantine.") - queued_message.destroy - log "#{log_prefix} Route says to quarantine spam message. Holding." - next - end - - # If the route says we're holding quananteed mail and this is spam, we'll hold this - if route.spam_mode == "Fail" && queued_message.message.spam && !queued_message.manual? - queued_message.message.create_delivery("HardFail", details: "Message is spam and the route specifies it should be failed.") - queued_message.destroy - log "#{log_prefix} Route says to fail spam message. Hard failing." - next - end - - # - # Messages that should be blindly accepted are blindly accepted - # - if route.mode == "Accept" - queued_message.message.create_delivery("Processed", details: "Message has been accepted but not sent to any endpoints.") - queued_message.destroy - log "#{log_prefix} Route says to accept without endpoint. Marking as processed." - next - end - - # - # Messages that should be accepted and held should be held - # - if route.mode == "Hold" - log "#{log_prefix} Route says to hold message." - if queued_message.manual? - log "#{log_prefix} Message was queued manually. Marking as processed." - queued_message.message.create_delivery("Processed", details: "Message has been processed.") - else - log "#{log_prefix} Message was not queued manually. Holding." - queued_message.message.create_delivery("Held", details: "Message has been accepted but not sent to any endpoints.") - end - queued_message.destroy - next - end - - # - # Messages that should be bounced should be bounced (or rejected if they got this far) - # - if route.mode == "Bounce" || route.mode == "Reject" - if id = queued_message.send_bounce - queued_message.message.create_delivery("HardFail", details: "Message has been bounced because the route asks for this. See message ") - log "#{log_prefix} Route says to bounce. Hard failing and sent bounce (#{id})." - end - queued_message.destroy - next - end - - if @fixed_result - result = @fixed_result - else - case queued_message.message.endpoint - when SMTPEndpoint - sender = cached_sender(Postal::SMTPSender, queued_message.message.recipient_domain, nil, servers: [queued_message.message.endpoint]) - when HTTPEndpoint - sender = cached_sender(Postal::HTTPSender, queued_message.message.endpoint) - when AddressEndpoint - sender = cached_sender(Postal::SMTPSender, queued_message.message.endpoint.domain, nil, force_rcpt_to: queued_message.message.endpoint.address) - else - log "#{log_prefix} Invalid endpoint for route (#{queued_message.message.endpoint_type})" - queued_message.message.create_delivery("HardFail", details: "Invalid endpoint for route.") - queued_message.destroy - next - end - result = sender.send_message(queued_message.message) - if result.connect_error - @fixed_result = result - end - end - - # Log the result - log_details = result.details - if result.type == "HardFail" && result.suppress_bounce - # The delivery hard failed, but requested that no bounce be sent - log "#{log_prefix} Suppressing bounce message after hard fail" - elsif result.type == "HardFail" && queued_message.message.send_bounces? - # If the message is a hard fail, send a bounce message for this message. - log "#{log_prefix} Sending a bounce because message hard failed" - if bounce_id = queued_message.send_bounce - log_details += ". " unless log_details =~ /\.\z/ - log_details += " Sent bounce message to sender (see message )" - end - end - - queued_message.message.create_delivery(result.type, details: log_details, output: result.output&.strip, sent_with_ssl: result.secure, log_id: result.log_id, time: result.time) - - if result.retry - log "#{log_prefix} Message requeued for trying later." - queued_message.retry_later(result.retry.is_a?(Integer) ? result.retry : nil) - queued_message.allocate_ip_address - queued_message.update_column(:ip_address_id, queued_message.ip_address&.id) - else - log "#{log_prefix} Message processing completed." - queued_message.message.endpoint.mark_as_used - queued_message.destroy - end - else - log "#{log_prefix} No route and/or endpoint available for processing. Hard failing." - queued_message.message.create_delivery("HardFail", details: "Message does not have a route and/or endpoint available for delivery.") - queued_message.destroy - next - end - end - - # - # Handle Outgoing Messages - # - if queued_message.message.scope == "outgoing" - if queued_message.message.domain.nil? - log "#{log_prefix} Message has no domain. Hard failing." - queued_message.message.create_delivery("HardFail", details: "Message's domain no longer exist") - queued_message.destroy - next - end - - # - # If there's no to address, we can't do much. Fail it. - # - if queued_message.message.rcpt_to.blank? - log "#{log_prefix} Message has no to address. Hard failing." - queued_message.message.create_delivery("HardFail", details: "Message doesn't have an RCPT to") - queued_message.destroy - next - end - - # Extract a tag and add it to the message if one doesn't exist - if queued_message.message.tag.nil? && tag = queued_message.message.headers["x-postal-tag"] - log "#{log_prefix} Added tag #{tag.last}" - queued_message.message.update(tag: tag.last) - end - - # - # If the credentials for this message is marked as holding and this isn't manual, hold it - # - if !queued_message.manual? && queued_message.message.credential && queued_message.message.credential.hold? - log "#{log_prefix} Credential wants us to hold messages. Holding." - queued_message.message.create_delivery("Held", details: "Credential is configured to hold all messages authenticated by it.") - queued_message.destroy - next - end - - # - # If the recipient is on the suppression list and this isn't a manual queueing block sending - # - if !queued_message.manual? && sl = queued_message.server.message_db.suppression_list.get(:recipient, queued_message.message.rcpt_to) - log "#{log_prefix} Recipient is on the suppression list. Holding." - queued_message.message.create_delivery("Held", details: "Recipient (#{queued_message.message.rcpt_to}) is on the suppression list (reason: #{sl['reason']})") - queued_message.destroy - next - end - - # Parse the content of the message as appropriate - if queued_message.message.should_parse? - log "#{log_prefix} Parsing message content as it hasn't been parsed before" - queued_message.message.parse_content - end - - # Inspect outgoing messages when there's a threshold set for the server - if !queued_message.message.inspected && queued_message.server.outbound_spam_threshold - log "#{log_prefix} Inspecting message" - queued_message.message.inspect_message - if queued_message.message.inspected - if queued_message.message.spam_score >= queued_message.server.outbound_spam_threshold - queued_message.message.update(spam: true) - end - log "#{log_prefix} Message inspected successfully" - end - end - - if queued_message.message.spam - queued_message.message.create_delivery("HardFail", details: "Message is likely spam. Threshold is #{queued_message.server.outbound_spam_threshold} and the message scored #{queued_message.message.spam_score}.") - queued_message.destroy - log "#{log_prefix} Message is spam (#{queued_message.message.spam_score}). Hard failing." - next - end - - # Add outgoing headers - unless queued_message.message.has_outgoing_headers? - queued_message.message.add_outgoing_headers - end - - # Check send limits - if queued_message.server.send_limit_exceeded? - # If we're over the limit, we're going to be holding this message - queued_message.server.update_columns(send_limit_exceeded_at: Time.now, send_limit_approaching_at: nil) - queued_message.message.create_delivery("Held", details: "Message held because send limit (#{queued_message.server.send_limit}) has been reached.") - queued_message.destroy - log "#{log_prefix} Server send limit has been exceeded. Holding." - next - elsif queued_message.server.send_limit_approaching? - # If we're approaching the limit, just say we are but continue to process the message - queued_message.server.update_columns(send_limit_approaching_at: Time.now, send_limit_exceeded_at: nil) - else - queued_message.server.update_columns(send_limit_approaching_at: nil, send_limit_exceeded_at: nil) - end - - # Update the live stats for this message. - queued_message.message.database.live_stats.increment(queued_message.message.scope) - - # If the server is in development mode, hold it - if queued_message.server.mode == "Development" && !queued_message.manual? - log "Server is in development mode so holding." - queued_message.message.create_delivery("Held", details: "Server is in development mode.") - queued_message.destroy - log "#{log_prefix} Server is in development mode. Holding." - next - end - - # Send the outgoing message to the SMTP sender - - if @fixed_result - result = @fixed_result - else - sender = cached_sender(Postal::SMTPSender, queued_message.message.recipient_domain, queued_message.ip_address) - result = sender.send_message(queued_message.message) - if result.connect_error - @fixed_result = result - end - end - - # - # If the message has been hard failed, check to see how many other recent hard fails we've had for the address - # and if there are more than 2, suppress the address for 30 days. - # - if result.type == "HardFail" - recent_hard_fails = queued_message.server.message_db.select(:messages, where: { rcpt_to: queued_message.message.rcpt_to, status: "HardFail", timestamp: { greater_than: 24.hours.ago.to_f } }, count: true) - if recent_hard_fails >= 1 && queued_message.server.message_db.suppression_list.add(:recipient, queued_message.message.rcpt_to, reason: "too many hard fails") - log "#{log_prefix} Added #{queued_message.message.rcpt_to} to suppression list because #{recent_hard_fails} hard fails in 24 hours" - result.details += "." if result.details =~ /\.\z/ - result.details += " Recipient added to suppression list (too many hard fails)." - end - end - - # - # If a message is sent successfully, remove the users from the suppression list - # - if result.type == "Sent" && queued_message.server.message_db.suppression_list.remove(:recipient, queued_message.message.rcpt_to) - log "#{log_prefix} Removed #{queued_message.message.rcpt_to} from suppression list because success" - result.details += "." if result.details =~ /\.\z/ - result.details += " Recipient removed from suppression list." - end - - # Log the result - queued_message.message.create_delivery(result.type, details: result.details, output: result.output, sent_with_ssl: result.secure, log_id: result.log_id, time: result.time) - if result.retry - log "#{log_prefix} Message requeued for trying later." - queued_message.retry_later(result.retry.is_a?(Integer) ? result.retry : nil) - else - log "#{log_prefix} Processing complete" - queued_message.destroy - end - end - rescue StandardError => e - log "#{log_prefix} Internal error: #{e.class}: #{e.message}" - e.backtrace.each { |line| log("#{log_prefix} #{line}") } - queued_message.retry_later - log "#{log_prefix} Queued message was unlocked" - if defined?(Sentry) - Sentry.capture_exception(e, extra: { job_id: self.id, server_id: queued_message.server_id, message_id: queued_message.message_id }) - end - queued_message.message&.create_delivery("Error", - details: "An internal error occurred while sending " \ - "this message. This message will be retried " \ - "automatically.", - output: "#{e.class}: #{e.message}", log_id: "J-#{self.id}") - end - end - - else - log "Couldn't get lock for message #{params['id']}. I won't do this." - end - else - log "No queued message with ID #{params['id']} was available for processing." - end - ensure - begin - @sender&.finish - rescue StandardError - nil - end - end - # rubocop:enable Layout/LineLength - - private - - # rubocop:disable Naming/MemoizedInstanceVariableName - def cached_sender(klass, *args) - @sender ||= begin - sender = klass.new(*args) - sender.start - sender - end - end - # rubocop:enable Naming/MemoizedInstanceVariableName - -end diff --git a/app/jobs/webhook_delivery_job.rb b/app/jobs/webhook_delivery_job.rb deleted file mode 100644 index a7369f5..0000000 --- a/app/jobs/webhook_delivery_job.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -class WebhookDeliveryJob < Postal::Job - - def perform - if webhook_request = WebhookRequest.find_by_id(params["id"]) - if webhook_request.deliver - log "Succesfully delivered" - else - log "Delivery failed" - end - else - log "No webhook request found with ID '#{params['id']}'" - end - end - -end diff --git a/app/models/concerns/has_locking.rb b/app/models/concerns/has_locking.rb new file mode 100644 index 0000000..6223416 --- /dev/null +++ b/app/models/concerns/has_locking.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +# This concern provides functionality for locking items along with additional functionality to handle +# the concept of retrying items after a certain period of time. The following database columns are +# required on the model +# +# * locked_by - A string column to store the name of the process that has locked the item +# * locked_at - A datetime column to store the time the item was locked +# * retry_after - A datetime column to store the time after which the item should be retried +# * attempts - An integer column to store the number of attempts that have been made to process the item +# +# 'ready' means that it's ready to be processed. +module HasLocking + + extend ActiveSupport::Concern + + included do + scope :unlocked, -> { where(locked_at: nil) } + scope :ready, -> { where("retry_after IS NULL OR retry_after < ?", Time.now) } + end + + def ready? + retry_after.nil? || retry_after < Time.now + end + + def unlock + self.locked_by = nil + self.locked_at = nil + update_columns(locked_by: nil, locked_at: nil) + end + + def locked? + locked_at.present? + end + + def retry_later(time = nil) + retry_time = time || calculate_retry_time(attempts, 5.minutes) + self.locked_by = nil + self.locked_at = nil + update_columns(locked_by: nil, locked_at: nil, retry_after: Time.now + retry_time, attempts: attempts + 1) + end + + def calculate_retry_time(attempts, initial_period) + (1.3**attempts) * initial_period + end + +end diff --git a/app/models/concerns/has_soft_destroy.rb b/app/models/concerns/has_soft_destroy.rb index 55cfcaf..cb07790 100644 --- a/app/models/concerns/has_soft_destroy.rb +++ b/app/models/concerns/has_soft_destroy.rb @@ -14,7 +14,6 @@ module HasSoftDestroy run_callbacks :soft_destroy do self.deleted_at = Time.now save! - ActionDeletionJob.queue(:main, type: self.class.name, id: id) end end diff --git a/app/models/queued_message.rb b/app/models/queued_message.rb index 5fc413b..bbe7468 100644 --- a/app/models/queued_message.rb +++ b/app/models/queued_message.rb @@ -29,33 +29,18 @@ class QueuedMessage < ApplicationRecord include HasMessage + include HasLocking belongs_to :server belongs_to :ip_address, optional: true belongs_to :user, optional: true before_create :allocate_ip_address - after_commit :queue, on: :create - scope :unlocked, -> { where(locked_at: nil) } - scope :retriable, -> { where("retry_after IS NULL OR retry_after < ?", Time.now) } - scope :requeueable, -> { where("retry_after IS NULL OR retry_after < ?", 30.seconds.ago) } + scope :ready_with_delayed_retry, -> { where("retry_after IS NULL OR retry_after < ?", 30.seconds.ago) } - def retriable? - retry_after.nil? || retry_after < Time.now - end - - def queue - UnqueueMessageJob.queue(queue_name, id: id) - end - - def queue! - update_column(:retry_after, nil) - queue - end - - def queue_name - ip_address ? :"outgoing-#{ip_address.id}" : :main + def retry_now + update(retry_after: nil) end def send_bounce @@ -70,40 +55,6 @@ class QueuedMessage < ApplicationRecord self.ip_address = pool.ip_addresses.select_by_priority end - def acquire_lock - time = Time.now - locker = Postal.locker_name - rows = self.class.where(id: id, locked_by: nil, locked_at: nil).update_all(locked_by: locker, locked_at: time) - if rows == 1 - self.locked_by = locker - self.locked_at = time - true - else - false - end - end - - def retry_later(time = nil) - retry_time = time || self.class.calculate_retry_time(attempts, 5.minutes) - self.locked_by = nil - self.locked_at = nil - update_columns(locked_by: nil, locked_at: nil, retry_after: Time.now + retry_time, attempts: attempts + 1) - end - - def unlock - self.locked_by = nil - self.locked_at = nil - update_columns(locked_by: nil, locked_at: nil) - end - - def self.calculate_retry_time(attempts, initial_period) - (1.3**attempts) * initial_period - end - - def locked? - locked_at.present? - end - def batchable_messages(limit = 10) unless locked? raise Postal::Error, "Must lock current message before locking any friends" @@ -114,13 +65,9 @@ class QueuedMessage < ApplicationRecord else time = Time.now locker = Postal.locker_name - self.class.retriable.where(batch_key: batch_key, ip_address_id: ip_address_id, locked_by: nil, locked_at: nil).limit(limit).update_all(locked_by: locker, locked_at: time) + self.class.ready.where(batch_key: batch_key, ip_address_id: ip_address_id, locked_by: nil, locked_at: nil).limit(limit).update_all(locked_by: locker, locked_at: time) QueuedMessage.where(batch_key: batch_key, ip_address_id: ip_address_id, locked_by: locker, locked_at: time).where.not(id: id) end end - def self.requeue_all - unlocked.requeueable.each(&:queue) - end - end diff --git a/app/models/scheduled_task.rb b/app/models/scheduled_task.rb new file mode 100644 index 0000000..0989815 --- /dev/null +++ b/app/models/scheduled_task.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: scheduled_tasks +# +# id :bigint not null, primary key +# name :string(255) +# next_run_after :datetime +# +# Indexes +# +# index_scheduled_tasks_on_name (name) UNIQUE +# +class ScheduledTask < ApplicationRecord +end diff --git a/app/models/server.rb b/app/models/server.rb index fd23c4e..942f40c 100644 --- a/app/models/server.rb +++ b/app/models/server.rb @@ -206,7 +206,7 @@ class Server < ApplicationRecord end def queue_size - @queue_size ||= queued_messages.retriable.count + @queue_size ||= queued_messages.ready.count end def stats @@ -222,7 +222,7 @@ class Server < ApplicationRecord # Return the domain which can be used to authenticate emails sent from the given e-mail address. # - #  @param address [String] an e-mail address + # @param address [String] an e-mail address # @return [Domain, nil] the domain to use for authentication def authenticated_domain_for_address(address) return nil if address.blank? diff --git a/app/models/webhook_request.rb b/app/models/webhook_request.rb index ada4e34..c475155 100644 --- a/app/models/webhook_request.rb +++ b/app/models/webhook_request.rb @@ -5,21 +5,28 @@ # Table name: webhook_requests # # id :integer not null, primary key +# attempts :integer default(0) +# error :text(65535) +# event :string(255) +# locked_at :datetime +# locked_by :string(255) +# payload :text(65535) +# retry_after :datetime +# url :string(255) +# uuid :string(255) +# created_at :datetime # server_id :integer # webhook_id :integer -# url :string(255) -# event :string(255) -# uuid :string(255) -# payload :text(65535) -# attempts :integer default(0) -# retry_after :datetime -# error :text(65535) -# created_at :datetime +# +# Indexes +# +# index_webhook_requests_on_locked_by (locked_by) # class WebhookRequest < ApplicationRecord include HasUUID + include HasLocking RETRIES = { 1 => 2.minutes, 2 => 3.minutes, 3 => 6.minutes, 4 => 10.minutes, 5 => 15.minutes }.freeze @@ -31,30 +38,9 @@ class WebhookRequest < ApplicationRecord serialize :payload, Hash - after_commit :queue, on: :create - - def self.trigger(server, event, payload = {}) - unless server.is_a?(Server) - server = Server.find(server.to_i) - end - - webhooks = server.webhooks.enabled.includes(:webhook_events).references(:webhook_events).where("webhooks.all_events = ? OR webhook_events.event = ?", true, event) - webhooks.each do |webhook| - server.webhook_requests.create!(event: event, payload: payload, webhook: webhook, url: webhook.url) - end - end - - def self.requeue_all - where("retry_after < ?", Time.now).find_each(&:queue) - end - - def queue - WebhookDeliveryJob.queue(:main, id: id) - end - def deliver payload = { event: event, timestamp: created_at.to_f, payload: self.payload, uuid: uuid }.to_json - Postal.logger.tagged(event: event, url: url, component: "webhooks") do + Postal.logger.tagged(event: event, url: url) do Postal.logger.info "Sending webhook request" result = Postal::HTTP.post(url, sign: true, json: payload, timeout: 5) self.attempts += 1 @@ -74,7 +60,7 @@ class WebhookRequest < ApplicationRecord if result[:code] >= 200 && result[:code] < 300 Postal.logger.info "Received #{result[:code]} status code. That's OK." - destroy + destroy! webhook&.update_column(:last_used_at, Time.now) true else @@ -82,14 +68,31 @@ class WebhookRequest < ApplicationRecord self.error = "Couldn't send to URL. Code received was #{result[:code]}" if retry_after Postal.logger.info "Will retry #{retry_after} (this was attempt #{self.attempts})" - save + self.locked_by = nil + self.locked_at = nil + save! else Postal.logger.info "Have tried #{self.attempts} times. Giving up." - destroy + destroy! end false end end end + class << self + + def trigger(server, event, payload = {}) + unless server.is_a?(Server) + server = Server.find(server.to_i) + end + + webhooks = server.webhooks.enabled.includes(:webhook_events).references(:webhook_events).where("webhooks.all_events = ? OR webhook_events.event = ?", true, event) + webhooks.each do |webhook| + server.webhook_requests.create!(event: event, payload: payload, webhook: webhook, url: webhook.url) + end + end + + end + end diff --git a/app/models/worker_role.rb b/app/models/worker_role.rb new file mode 100644 index 0000000..22e83ea --- /dev/null +++ b/app/models/worker_role.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: worker_roles +# +# id :bigint not null, primary key +# acquired_at :datetime +# role :string(255) +# worker :string(255) +# +# Indexes +# +# index_worker_roles_on_role (role) UNIQUE +# +class WorkerRole < ApplicationRecord + + class << self + + # Acquire or renew a lock for the given role. + # + # @param role [String] The name of the role to acquire + # @return [Symbol, false] True if the lock was acquired or renewed, false otherwise + def acquire(role) + # update our existing lock if we already have one + updates = where(role: role, worker: Postal.locker_name).update_all(acquired_at: Time.current) + return :renewed if updates.positive? + + # attempt to steal a role from another worker + updates = where(role: role).where("acquired_at is null OR acquired_at < ?", 5.minutes.ago) + .update_all(acquired_at: Time.current, worker: Postal.locker_name) + return :stolen if updates.positive? + + # attempt to create a new role for this worker + begin + create!(role: role, worker: Postal.locker_name, acquired_at: Time.current) + :created + rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid + false + end + end + + # Release a lock for the given role for the current process. + # + # @param role [String] The name of the role to release + # @return [Boolean] True if the lock was released, false otherwise + def release(role) + updates = where(role: role, worker: Postal.locker_name).delete_all + updates.positive? + end + + end + +end diff --git a/app/scheduled_tasks/action_deletions_scheduled_task.rb b/app/scheduled_tasks/action_deletions_scheduled_task.rb new file mode 100644 index 0000000..e5c7d5a --- /dev/null +++ b/app/scheduled_tasks/action_deletions_scheduled_task.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class ActionDeletionsScheduledTask < ApplicationScheduledTask + + def call + Organization.deleted.each do |org| + logger.info "permanently removing organization #{org.id} (#{org.permalink})" + org.destroy + end + + Server.deleted.each do |server| + logger.info "permanently removing server #{server.id} (#{server.full_permalink})" + server.destroy + end + end + +end diff --git a/app/scheduled_tasks/application_scheduled_task.rb b/app/scheduled_tasks/application_scheduled_task.rb new file mode 100644 index 0000000..0aebe09 --- /dev/null +++ b/app/scheduled_tasks/application_scheduled_task.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +class ApplicationScheduledTask + + def initialize(logger:) + @logger = logger + end + + def call + # override me + end + + attr_reader :logger + + class << self + + def next_run_after + quarter_past_each_hour + end + + private + + def quarter_past_each_hour + time = Time.current + time = time.change(min: 15, sec: 0) + time += 1.hour if time < Time.current + time + end + + def quarter_to_each_hour + time = Time.current + time = time.change(min: 45, sec: 0) + time += 1.hour if time < Time.current + time + end + + def three_am + time = Time.current + time = time.change(hour: 3, min: 0, sec: 0) + time += 1.day if time < Time.current + time + end + + end + +end diff --git a/app/jobs/check_all_dns_job.rb b/app/scheduled_tasks/check_all_dns_scheduled_task.rb similarity index 62% rename from app/jobs/check_all_dns_job.rb rename to app/scheduled_tasks/check_all_dns_scheduled_task.rb index 0393afa..140d4a8 100644 --- a/app/jobs/check_all_dns_job.rb +++ b/app/scheduled_tasks/check_all_dns_scheduled_task.rb @@ -1,15 +1,15 @@ # frozen_string_literal: true -class CheckAllDNSJob < Postal::Job +class CheckAllDNSScheduledTask < ApplicationScheduledTask - def perform + def call Domain.where.not(dns_checked_at: nil).where("dns_checked_at <= ?", 1.hour.ago).each do |domain| - log "Checking DNS for domain: #{domain.name}" + logger.info "checking DNS for domain: #{domain.name}" domain.check_dns(:auto) end TrackDomain.where("dns_checked_at IS NULL OR dns_checked_at <= ?", 1.hour.ago).includes(:domain).each do |domain| - log "Checking DNS for track domain: #{domain.full_name}" + logger.info "checking DNS for track domain: #{domain.full_name}" domain.check_dns end end diff --git a/app/jobs/cleanup_authie_sessions_job.rb b/app/scheduled_tasks/cleanup_authie_sessions_scheduled_task.rb similarity index 55% rename from app/jobs/cleanup_authie_sessions_job.rb rename to app/scheduled_tasks/cleanup_authie_sessions_scheduled_task.rb index 36a93d0..333fec5 100644 --- a/app/jobs/cleanup_authie_sessions_job.rb +++ b/app/scheduled_tasks/cleanup_authie_sessions_scheduled_task.rb @@ -2,9 +2,9 @@ require "authie/session" -class CleanupAuthieSessionsJob < Postal::Job +class CleanupAuthieSessionsScheduledTask < ApplicationScheduledTask - def perform + def call Authie::Session.cleanup end diff --git a/app/jobs/expire_held_messages_job.rb b/app/scheduled_tasks/expire_held_messages_scheduled_task.rb similarity index 77% rename from app/jobs/expire_held_messages_job.rb rename to app/scheduled_tasks/expire_held_messages_scheduled_task.rb index e54d052..7abfa5b 100644 --- a/app/jobs/expire_held_messages_job.rb +++ b/app/scheduled_tasks/expire_held_messages_scheduled_task.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -class ExpireHeldMessagesJob < Postal::Job +class ExpireHeldMessagesScheduledTask < ApplicationScheduledTask - def perform + def call Server.all.each do |server| messages = server.message_db.messages(where: { status: "Held", diff --git a/app/jobs/process_message_retention_job.rb b/app/scheduled_tasks/process_message_retention_scheduled_task.rb similarity index 55% rename from app/jobs/process_message_retention_job.rb rename to app/scheduled_tasks/process_message_retention_scheduled_task.rb index 35c3bf3..671ec40 100644 --- a/app/jobs/process_message_retention_job.rb +++ b/app/scheduled_tasks/process_message_retention_scheduled_task.rb @@ -1,25 +1,29 @@ # frozen_string_literal: true -class ProcessMessageRetentionJob < Postal::Job +class ProcessMessageRetentionScheduledTask < ApplicationScheduledTask def perform Server.all.each do |server| if server.raw_message_retention_days # If the server has a maximum number of retained raw messages, remove any that are older than this - log "Tidying raw messages (by days) for #{server.permalink} (ID: #{server.id}). Keeping #{server.raw_message_retention_days} days." + logger.info "Tidying raw messages (by days) for #{server.permalink} (ID: #{server.id}). Keeping #{server.raw_message_retention_days} days." server.message_db.provisioner.remove_raw_tables_older_than(server.raw_message_retention_days) end if server.raw_message_retention_size - log "Tidying raw messages (by size) for #{server.permalink} (ID: #{server.id}). Keeping #{server.raw_message_retention_size} MB of data." + logger.info "Tidying raw messages (by size) for #{server.permalink} (ID: #{server.id}). Keeping #{server.raw_message_retention_size} MB of data." server.message_db.provisioner.remove_raw_tables_until_less_than_size(server.raw_message_retention_size * 1024 * 1024) end if server.message_retention_days - log "Tidying messages for #{server.permalink} (ID: #{server.id}). Keeping #{server.message_retention_days} days." + logger.info "Tidying messages for #{server.permalink} (ID: #{server.id}). Keeping #{server.message_retention_days} days." server.message_db.provisioner.remove_messages(server.message_retention_days) end end end + def self.next_run_after + three_am + end + end diff --git a/app/scheduled_tasks/prune_suppression_lists_scheduled_task.rb b/app/scheduled_tasks/prune_suppression_lists_scheduled_task.rb new file mode 100644 index 0000000..2f73904 --- /dev/null +++ b/app/scheduled_tasks/prune_suppression_lists_scheduled_task.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class PruneSuppressionListsScheduledTask < ApplicationScheduledTask + + def call + Server.all.each do |s| + logger.info "Pruning suppression lists for server #{s.id}" + s.message_db.suppression_list.prune + end + end + + def self.next_run_after + three_am + end + +end diff --git a/app/scheduled_tasks/prune_webhook_requests_scheduled_task.rb b/app/scheduled_tasks/prune_webhook_requests_scheduled_task.rb new file mode 100644 index 0000000..95147d7 --- /dev/null +++ b/app/scheduled_tasks/prune_webhook_requests_scheduled_task.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class PruneWebhookRequestsScheduledTask < ApplicationScheduledTask + + def call + Server.all.each do |s| + logger.info "Pruning webhook requests for server #{s.id}" + s.message_db.webhooks.prune + end + end + + def self.next_run_after + quarter_to_each_hour + end + +end diff --git a/app/scheduled_tasks/send_notifications_scheduled_task.rb b/app/scheduled_tasks/send_notifications_scheduled_task.rb new file mode 100644 index 0000000..53f3f6d --- /dev/null +++ b/app/scheduled_tasks/send_notifications_scheduled_task.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class SendNotificationsScheduledTask < ApplicationScheduledTask + + def call + Server.send_send_limit_notifications + end + + def self.next_run_after + 1.minute.from_now + end + +end diff --git a/app/services/unqueue_message_service.rb b/app/services/unqueue_message_service.rb new file mode 100644 index 0000000..5433771 --- /dev/null +++ b/app/services/unqueue_message_service.rb @@ -0,0 +1,487 @@ +# frozen_string_literal: true + +class UnqueueMessageService + + def initialize(queued_message:, logger:) + @queued_message = queued_message + @logger = logger + end + + def call + @logger.tagged(original_queued_message: @queued_message.id) do + log "starting message unqueue" + process_original_message + log "finished message unqueue" + end + end + + private + + def process_original_message + begin + @queued_message.message + rescue Postal::MessageDB::Message::NotFound + log "unqueue because backend message has been removed." + @queued_message.destroy + return + end + + unless @queued_message.ready? + log "skipping because message isn't ready for processing" + return + end + + begin + other_messages = @queued_message.batchable_messages(100) + log "found #{other_messages.size} associated messages to process at the same time", batch_key: @queued_message.batch_key + rescue StandardError + @queued_message.unlock + raise + end + + ([@queued_message] + other_messages).each do |queued_message| + @logger.tagged(queued_message: queued_message.id) do + process_message(queued_message) + end + end + ensure + begin + @sender&.finish + rescue StandardError + nil + end + end + + # rubocop:disable Naming/MemoizedInstanceVariableName + def cached_sender(klass, *args) + @sender ||= begin + sender = klass.new(*args) + sender.start + sender + end + end + # rubocop:enable Naming/MemoizedInstanceVariableName + + def log(message, **tags) + @logger.info(message, **tags) + end + + def process_message(queued_message) + begin + queued_message.message + rescue Postal::MessageDB::Message::NotFound + log "unqueueing because backend message has been removed" + queued_message.destroy + return + end + + log "processing message" + + # + # If the server is suspended, hold all messages + # + if queued_message.server.suspended? + log "server is suspended, holding message" + queued_message.message.create_delivery("Held", details: "Mail server has been suspended. No e-mails can be processed at present. Contact support for assistance.") + queued_message.destroy + return + end + + # We might not be able to send this any more, check the attempts + if queued_message.attempts >= Postal.config.general.maximum_delivery_attempts + details = "Maximum number of delivery attempts (#{queued_message.attempts}) has been reached." + if queued_message.message.scope == "incoming" + # Send bounces to incoming e-mails when they are hard failed + if bounce_id = queued_message.send_bounce + details += " Bounce sent to sender (see message )" + end + elsif queued_message.message.scope == "outgoing" + # Add the recipient to the suppression list + if queued_message.server.message_db.suppression_list.add(:recipient, queued_message.message.rcpt_to, reason: "too many soft fails") + log "added #{queued_message.message.rcpt_to} to suppression list because maximum attempts has been reached" + details += " Added #{queued_message.message.rcpt_to} to suppression list because delivery has failed #{queued_message.attempts} times." + end + end + queued_message.message.create_delivery("HardFail", details: details) + queued_message.destroy + log "message has reached maximum number of attempts, hard failing" + return + end + + # If the raw message has been removed (removed by retention) + unless queued_message.message.raw_message? + log "raw message has been removed, not sending" + queued_message.message.create_delivery("HardFail", details: "Raw message has been removed. Cannot send message.") + queued_message.destroy + return + end + + # + # Handle Incoming Messages + # + if queued_message.message.scope == "incoming" + log "message is incoming" + + # + # If this is a bounce, we need to handle it as such + # + if queued_message.message.bounce + log "message is a bounce" + original_messages = queued_message.message.original_messages + unless original_messages.empty? + queued_message.message.original_messages.each do |orig_msg| + queued_message.message.update(bounce_for_id: orig_msg.id, domain_id: orig_msg.domain_id) + queued_message.message.create_delivery("Processed", details: "This has been detected as a bounce message for .") + orig_msg.bounce!(queued_message.message) + log "bounce linked with message #{orig_msg.id}" + end + queued_message.destroy + return + end + + # This message was sent to the return path but hasn't been matched + # to an original message. If we have a route for this, route it + # otherwise we'll drop at this point. + if queued_message.message.route_id.nil? + log "no source messages found, hard failing" + queued_message.message.create_delivery("HardFail", details: "This message was a bounce but we couldn't link it with any outgoing message and there was no route for it.") + queued_message.destroy + return + end + end + + # + # Update live stats + # + queued_message.message.database.live_stats.increment(queued_message.message.scope) + + # + # Inspect incoming messages + # + unless queued_message.message.inspected + log "inspecting message" + queued_message.message.inspect_message + if queued_message.message.inspected + is_spam = queued_message.message.spam_score > queued_message.server.spam_threshold + queued_message.message.update(spam: true) if is_spam + queued_message.message.append_headers( + "X-Postal-Spam: #{queued_message.message.spam ? 'yes' : 'no'}", + "X-Postal-Spam-Threshold: #{queued_message.server.spam_threshold}", + "X-Postal-Spam-Score: #{queued_message.message.spam_score}", + "X-Postal-Threat: #{queued_message.message.threat ? 'yes' : 'no'}" + ) + log "message inspected, headers added", spam: queued_message.message.spam?, spam_score: queued_message.message.spam_score, threat: queued_message.message.threat? + end + end + + # + # If this message has a SPAM score higher than is permitted + # + if queued_message.message.spam_score >= queued_message.server.spam_failure_threshold + log "message has a spam score higher than the server's maxmimum, hard failing", server_threshold: queued_message.server.spam_failure_threshold + queued_message.message.create_delivery("HardFail", + details: "Message's spam score is higher than the failure threshold for this server. " \ + "Threshold is currently #{queued_message.server.spam_failure_threshold}.") + queued_message.destroy + return + end + + # If the server is in development mode, hold it + if queued_message.server.mode == "Development" && !queued_message.manual? + log "server is in development mode, holding" + queued_message.message.create_delivery("Held", details: "Server is in development mode.") + queued_message.destroy + return + end + + # + # Find out what sort of message we're supposed to be sending and dispatch this request over to + # the sender. + # + if route = queued_message.message.route + + # If the route says we're holding quananteed mail and this is spam, we'll hold this + if route.spam_mode == "Quarantine" && queued_message.message.spam && !queued_message.manual? + log "message is spam and route says to quarantine spam message, holding" + queued_message.message.create_delivery("Held", details: "Message placed into quarantine.") + queued_message.destroy + return + end + + # If the route says we're holding quananteed mail and this is spam, we'll hold this + if route.spam_mode == "Fail" && queued_message.message.spam && !queued_message.manual? + log "message is spam and route says to fail spam message, hard failing" + queued_message.message.create_delivery("HardFail", details: "Message is spam and the route specifies it should be failed.") + queued_message.destroy + return + end + + # + # Messages that should be blindly accepted are blindly accepted + # + if route.mode == "Accept" + log "route says to accept without endpoint, marking as processed" + queued_message.message.create_delivery("Processed", details: "Message has been accepted but not sent to any endpoints.") + queued_message.destroy + return + end + + # + # Messages that should be accepted and held should be held + # + if route.mode == "Hold" + if queued_message.manual? + log "route says to hold and message was queued manually, marking as processed" + queued_message.message.create_delivery("Processed", details: "Message has been processed.") + else + log "route says to hold, marking as held" + queued_message.message.create_delivery("Held", details: "Message has been accepted but not sent to any endpoints.") + end + queued_message.destroy + return + end + + # + # Messages that should be bounced should be bounced (or rejected if they got this far) + # + if route.mode == "Bounce" || route.mode == "Reject" + log "route says to bounce, hard failing and sending bounce" + if id = queued_message.send_bounce + log "bounce sent with id #{id}" + queued_message.message.create_delivery("HardFail", details: "Message has been bounced because the route asks for this. See message ") + end + queued_message.destroy + return + end + + if @fixed_result + result = @fixed_result + else + case queued_message.message.endpoint + when SMTPEndpoint + sender = cached_sender(Postal::SMTPSender, queued_message.message.recipient_domain, nil, servers: [queued_message.message.endpoint]) + when HTTPEndpoint + sender = cached_sender(Postal::HTTPSender, queued_message.message.endpoint) + when AddressEndpoint + sender = cached_sender(Postal::SMTPSender, queued_message.message.endpoint.domain, nil, force_rcpt_to: queued_message.message.endpoint.address) + else + log "invalid endpoint for route (#{queued_message.message.endpoint_type})" + queued_message.message.create_delivery("HardFail", details: "Invalid endpoint for route.") + queued_message.destroy + return + end + result = sender.send_message(queued_message.message) + if result.connect_error + @fixed_result = result + end + end + + # Log the result + log_details = result.details + if result.type == "HardFail" && result.suppress_bounce + # The delivery hard failed, but requested that no bounce be sent + log "suppressing bounce message after hard fail" + elsif result.type == "HardFail" && queued_message.message.send_bounces? + # If the message is a hard fail, send a bounce message for this message. + log "sending a bounce because message hard failed" + if bounce_id = queued_message.send_bounce + log_details += ". " unless log_details =~ /\.\z/ + log_details += " Sent bounce message to sender (see message )" + end + end + + queued_message.message.create_delivery(result.type, details: log_details, output: result.output&.strip, sent_with_ssl: result.secure, log_id: result.log_id, time: result.time) + + if result.retry + queued_message.retry_later(result.retry.is_a?(Integer) ? result.retry : nil) + log "message requeued for trying later, at #{queued_message.retry_after}" + queued_message.allocate_ip_address + queued_message.update_column(:ip_address_id, queued_message.ip_address&.id) + else + log "message processing completed" + queued_message.message.endpoint.mark_as_used + queued_message.destroy + end + else + log "no route and/or endpoint available for processing, hard failing" + queued_message.message.create_delivery("HardFail", details: "Message does not have a route and/or endpoint available for delivery.") + queued_message.destroy + return + end + end + + # + # Handle Outgoing Messages + # + return unless queued_message.message.scope == "outgoing" + + log "message is outgoing" + + if queued_message.message.domain.nil? + log "message has no domain, hard failing" + queued_message.message.create_delivery("HardFail", details: "Message's domain no longer exist") + queued_message.destroy + return + end + + # + # If there's no to address, we can't do much. Fail it. + # + if queued_message.message.rcpt_to.blank? + log "message has no 'to' address, hard failing" + queued_message.message.create_delivery("HardFail", details: "Message doesn't have an RCPT to") + queued_message.destroy + return + end + + # Extract a tag and add it to the message if one doesn't exist + if queued_message.message.tag.nil? && tag = queued_message.message.headers["x-postal-tag"] + log "added tag: #{tag.last}" + queued_message.message.update(tag: tag.last) + end + + # + # If the credentials for this message is marked as holding and this isn't manual, hold it + # + if !queued_message.manual? && queued_message.message.credential && queued_message.message.credential.hold? + log "credential wants us to hold messages, holding" + queued_message.message.create_delivery("Held", details: "Credential is configured to hold all messages authenticated by it.") + queued_message.destroy + return + end + + # + # If the recipient is on the suppression list and this isn't a manual queueing block sending + # + if !queued_message.manual? && sl = queued_message.server.message_db.suppression_list.get(:recipient, queued_message.message.rcpt_to) + log "recipient is on the suppression list, holding" + queued_message.message.create_delivery("Held", details: "Recipient (#{queued_message.message.rcpt_to}) is on the suppression list (reason: #{sl['reason']})") + queued_message.destroy + return + end + + # Parse the content of the message as appropriate + if queued_message.message.should_parse? + log "parsing message content as it hasn't been parsed before" + queued_message.message.parse_content + end + + # Inspect outgoing messages when there's a threshold set for the server + if !queued_message.message.inspected && queued_message.server.outbound_spam_threshold + log "inspecting message" + queued_message.message.inspect_message + if queued_message.message.inspected + if queued_message.message.spam_score >= queued_message.server.outbound_spam_threshold + queued_message.message.update(spam: true) + end + log "message inspected successfully", spam: queued_message.message.spam?, spam_score: queued_message.message.spam_score + end + end + + if queued_message.message.spam + log "message is spam (#{queued_message.message.spam_score}), hard failing", server_threshold: queued_message.server.outbound_spam_threshold + queued_message.message.create_delivery("HardFail", + details: "Message is likely spam. Threshold is #{queued_message.server.outbound_spam_threshold} and " \ + "the message scored #{queued_message.message.spam_score}.") + queued_message.destroy + return + end + + # Add outgoing headers + unless queued_message.message.has_outgoing_headers? + queued_message.message.add_outgoing_headers + end + + # Check send limits + if queued_message.server.send_limit_exceeded? + # If we're over the limit, we're going to be holding this message + log "server send limit has been exceeded, holding", send_limit: queued_message.server.send_limit + queued_message.server.update_columns(send_limit_exceeded_at: Time.now, send_limit_approaching_at: nil) + queued_message.message.create_delivery("Held", details: "Message held because send limit (#{queued_message.server.send_limit}) has been reached.") + queued_message.destroy + return + elsif queued_message.server.send_limit_approaching? + # If we're approaching the limit, just say we are but continue to process the message + queued_message.server.update_columns(send_limit_approaching_at: Time.now, send_limit_exceeded_at: nil) + else + queued_message.server.update_columns(send_limit_approaching_at: nil, send_limit_exceeded_at: nil) + end + + # Update the live stats for this message. + queued_message.message.database.live_stats.increment(queued_message.message.scope) + + # If the server is in development mode, hold it + if queued_message.server.mode == "Development" && !queued_message.manual? + log "server is in development mode, holding" + queued_message.message.create_delivery("Held", details: "Server is in development mode.") + queued_message.destroy + return + end + + # Send the outgoing message to the SMTP sender + + if @fixed_result + result = @fixed_result + else + sender = cached_sender(Postal::SMTPSender, queued_message.message.recipient_domain, queued_message.ip_address) + result = sender.send_message(queued_message.message) + if result.connect_error + @fixed_result = result + end + end + + # + # If the message has been hard failed, check to see how many other recent hard fails we've had for the address + # and if there are more than 2, suppress the address for 30 days. + # + if result.type == "HardFail" + recent_hard_fails = queued_message.server.message_db.select(:messages, + where: { + rcpt_to: queued_message.message.rcpt_to, + status: "HardFail", + timestamp: { greater_than: 24.hours.ago.to_f } + }, + count: true) + if recent_hard_fails >= 1 && queued_message.server.message_db.suppression_list.add(:recipient, queued_message.message.rcpt_to, reason: "too many hard fails") + log "Added #{queued_message.message.rcpt_to} to suppression list because #{recent_hard_fails} hard fails in 24 hours" + result.details += "." if result.details =~ /\.\z/ + result.details += " Recipient added to suppression list (too many hard fails)." + end + end + + # + # If a message is sent successfully, remove the users from the suppression list + # + if result.type == "Sent" && queued_message.server.message_db.suppression_list.remove(:recipient, queued_message.message.rcpt_to) + log "removed #{queued_message.message.rcpt_to} from suppression list" + result.details += "." if result.details =~ /\.\z/ + result.details += " Recipient removed from suppression list." + end + + # Log the result + queued_message.message.create_delivery(result.type, details: result.details, output: result.output, sent_with_ssl: result.secure, log_id: result.log_id, time: result.time) + if result.retry + queued_message.retry_later(result.retry.is_a?(Integer) ? result.retry : nil) + log "message requeued for trying later", retry_after: queued_message.retry_after + else + log "message processing complete" + queued_message.destroy + end + rescue StandardError => e + log "internal error: #{e.class}: #{e.message}" + e.backtrace.each { |line| log(line) } + + queued_message.retry_later + log "message requeued for trying later, at #{queued_message.retry_after}" + + if defined?(Sentry) + Sentry.capture_exception(e, extra: { server_id: queued_message.server_id, queued_message_id: queued_message.message_id }) + end + queued_message.message&.create_delivery("Error", + details: "An internal error occurred while sending " \ + "this message. This message will be retried " \ + "automatically.", + output: "#{e.class}: #{e.message}", log_id: "J-#{id}") + end + +end diff --git a/app/services/webhook_delivery_service.rb b/app/services/webhook_delivery_service.rb new file mode 100644 index 0000000..617054e --- /dev/null +++ b/app/services/webhook_delivery_service.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class WebhookDeliveryService + + def initialize(webhook_delivery:) + @webhook_delivery = webhook_delivery + end + + # TODO: move the logic from WebhookDelivery#deliver in to this service. + # + def call + if @webhook_delivery.deliver + log "Succesfully delivered" + else + log "Delivery failed" + end + end + +end diff --git a/bin/postal b/bin/postal index b113e2d..34a6d22 100755 --- a/bin/postal +++ b/bin/postal @@ -16,21 +16,13 @@ case "$1" in ;; smtp-server) - run "bundle exec rake postal:smtp_server" + run "bundle exec ruby script/smtp_server.rb" ;; worker) run "bundle exec ruby script/worker.rb" ;; - cron) - run "bundle exec rake postal:cron" - ;; - - requeuer) - run "bundle exec rake postal:requeuer" - ;; - initialize) echo 'Initializing database' run "bundle exec rake db:create db:schema:load db:seed" @@ -69,8 +61,6 @@ case "$1" in echo -e " * \033[35mweb-server\033[0m - run the web server" echo -e " * \033[35msmtp-server\033[0m - run the SMTP server" echo -e " * \033[35mworker\033[0m - run a worker" - echo -e " * \033[35mcron\033[0m - run the cron process" - echo -e " * \033[35mrequeuer\033[0m - run the message requeuer" echo echo "Setup/upgrade tools:" echo diff --git a/config/cron.rb b/config/cron.rb deleted file mode 100644 index fed7a60..0000000 --- a/config/cron.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Clockwork - - configure do |config| - config[:tz] = "UTC" - config[:logger] = Postal.logger - end - - every 1.minute, "every-1-minutes" do - RequeueWebhooksJob.queue(:main) - SendNotificationsJob.queue(:main) - end - - every 1.hour, "every-hour", at: ["**:15"] do - CheckAllDNSJob.queue(:main) - ExpireHeldMessagesJob.queue(:main) - CleanupAuthieSessionsJob.queue(:main) - end - - every 1.hour, "every-hour", at: ["**:45"] do - PruneWebhookRequestsJob.queue(:main) - end - - every 1.day, "every-day", at: ["03:00"] do - ProcessMessageRetentionJob.queue(:main) - PruneSuppressionListsJob.queue(:main) - end - -end diff --git a/config/initializers/logging.rb b/config/initializers/logging.rb index 5108da4..22e16ab 100644 --- a/config/initializers/logging.rb +++ b/config/initializers/logging.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + begin def add_exception_to_payload(payload, event) return unless exception = event.payload[:exception_object] diff --git a/config/postal.defaults.yml b/config/postal.defaults.yml index 2c9fa33..6574a0d 100644 --- a/config/postal.defaults.yml +++ b/config/postal.defaults.yml @@ -40,16 +40,6 @@ message_db: password: <%= ENV.fetch('MESSAGE_DB_PASSWORD', '') %> prefix: <%= ENV.fetch('MESSAGE_DB_PREFIX', 'postal') %> -rabbitmq: - host: <%= ENV.fetch('RABBITMQ_HOST', '127.0.0.1') %> - port: <%= ENV.fetch('RABBITMQ_PORT', '5672') %> - username: <%= ENV.fetch('RABBITMQ_USERNAME', 'postal') %> - password: <%= ENV.fetch('RABBITMQ_PASSWORD', '') %> - vhost: <%= ENV.fetch('RABBITMQ_VHOST', '/postal') %> - tls: <%= ENV.fetch('RABBITMQ_TLS', 'false') %> - verify_peer: <%= ENV.fetch('RABBITMQ_VERIFY_PEER', 'true') %> - tls_ca_certificates: <%= ENV.fetch('RABBITMQ_TLS_CA_CERTIFICATES', '/etc/ssl/certs/ca-certificates.crt'.split(',').inspect) %> - logging: rails_log: <%= ENV.fetch('LOGGING_RAILS_LOG', 'false') %> graylog: @@ -57,9 +47,6 @@ logging: port: <%= ENV.fetch('GRAYLOG_PORT', '12201') %> facility: <%= ENV.fetch('GRAYLOG_FACILITY', 'postal') %> -workers: - threads: <%= ENV.fetch('WORKER_THREADS', '4') %> - smtp_server: port: <%= ENV.fetch('SMTP_SERVER_PORT', '25') %> bind_address: "<%= ENV.fetch('SMTP_SERVER_BIND_ADDRESS', '::') %>" diff --git a/db/migrate/20240213165450_create_worker_roles.rb b/db/migrate/20240213165450_create_worker_roles.rb new file mode 100644 index 0000000..4b62843 --- /dev/null +++ b/db/migrate/20240213165450_create_worker_roles.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class CreateWorkerRoles < ActiveRecord::Migration[6.1] + + def change + create_table :worker_roles do |t| + t.string :role + t.string :worker + t.datetime :acquired_at + t.index :role, unique: true + end + end + +end diff --git a/db/migrate/20240213171830_create_scheduled_tasks.rb b/db/migrate/20240213171830_create_scheduled_tasks.rb new file mode 100644 index 0000000..fb18b5e --- /dev/null +++ b/db/migrate/20240213171830_create_scheduled_tasks.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class CreateScheduledTasks < ActiveRecord::Migration[6.1] + + def change + create_table :scheduled_tasks do |t| + t.string :name + t.datetime :next_run_after + t.index :name, unique: true + end + end + +end diff --git a/db/migrate/20240214132253_add_lock_fields_to_webhook_requests.rb b/db/migrate/20240214132253_add_lock_fields_to_webhook_requests.rb new file mode 100644 index 0000000..edf5662 --- /dev/null +++ b/db/migrate/20240214132253_add_lock_fields_to_webhook_requests.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class AddLockFieldsToWebhookRequests < ActiveRecord::Migration[6.1] + + def change + add_column :webhook_requests, :locked_by, :string + add_column :webhook_requests, :locked_at, :datetime + + add_index :webhook_requests, :locked_by + end + +end diff --git a/db/schema.rb b/db/schema.rb index a1b9e76..063f8fa 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2024_02_06_173036) do +ActiveRecord::Schema.define(version: 2024_02_14_132253) do create_table "additional_route_endpoints", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.integer "route_id" @@ -213,6 +213,12 @@ ActiveRecord::Schema.define(version: 2024_02_06_173036) do t.index ["token"], name: "index_routes_on_token", length: 6 end + create_table "scheduled_tasks", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.string "name" + t.datetime "next_run_after" + t.index ["name"], name: "index_scheduled_tasks_on_name", unique: true + end + create_table "servers", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| t.integer "organization_id" t.string "uuid" @@ -343,6 +349,9 @@ ActiveRecord::Schema.define(version: 2024_02_06_173036) do t.datetime "retry_after", precision: 6 t.text "error" t.datetime "created_at", precision: 6 + t.string "locked_by" + t.datetime "locked_at" + t.index ["locked_by"], name: "index_webhook_requests_on_locked_by" end create_table "webhooks", id: :integer, charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| @@ -359,4 +368,11 @@ ActiveRecord::Schema.define(version: 2024_02_06_173036) do t.index ["server_id"], name: "index_webhooks_on_server_id" end + create_table "worker_roles", charset: "utf8mb4", collation: "utf8mb4_general_ci", force: :cascade do |t| + t.string "role" + t.string "worker" + t.datetime "acquired_at" + t.index ["role"], name: "index_worker_roles_on_role", unique: true + end + end diff --git a/docker-compose.yml b/docker-compose.yml index f3d8670..e3cb991 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,6 @@ services: image: ${POSTAL_IMAGE} depends_on: - mariadb - - rabbitmq entrypoint: ["/docker-entrypoint.sh"] volumes: - "./docker/ci-config:/config" @@ -14,7 +13,6 @@ services: WAIT_FOR_TIMEOUT: 90 WAIT_FOR_TARGETS: |- mariadb:3306 - rabbitmq:5672 mariadb: image: mariadb @@ -23,7 +21,3 @@ services: MARIADB_DATABASE: postal MARIADB_ALLOW_EMPTY_PASSWORD: 'yes' MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: 'yes' - - rabbitmq: - image: rabbitmq:3 - restart: always diff --git a/docker/ci-config/postal.test.yml b/docker/ci-config/postal.test.yml index 69922c9..65dfef5 100644 --- a/docker/ci-config/postal.test.yml +++ b/docker/ci-config/postal.test.yml @@ -20,12 +20,6 @@ message_db: password: prefix: postal -rabbitmq: - host: rabbitmq - username: guest - password: guest - vhost: null - dns: mx_records: - mx.postal.example.com diff --git a/lib/postal/config.rb b/lib/postal/config.rb index 1f1f42d..6907d9e 100644 --- a/lib/postal/config.rb +++ b/lib/postal/config.rb @@ -85,6 +85,9 @@ module Postal end end + # Return a generic logger for use generally throughout Postal. + # + # @return [Klogger::Logger] A logger instance def self.logger @logger ||= begin k = Klogger.new(nil, destination: Rails.env.test? ? "/dev/null" : $stdout, highlight: Rails.env.development?) @@ -106,9 +109,14 @@ module Postal def self.locker_name string = process_name.dup string += " job:#{Thread.current[:job_id]}" if Thread.current[:job_id] + string += " thread:#{Thread.current.native_thread_id}" string end + def self.locker_name_with_suffix(suffix) + "#{locker_name} #{suffix}" + end + def self.smtp_from_name config.smtp&.from_name || "Postal" end @@ -175,7 +183,7 @@ module Postal end def self.graylog_logging_destination - @graylog_destination ||= begin + @graylog_logging_destination ||= begin notifier = GELF::Notifier.new(config.logging.graylog.host, config.logging.graylog.port, "WAN") proc do |_logger, payload, group_ids| short_message = payload.delete(:message) || "[message missing]" diff --git a/lib/postal/job.rb b/lib/postal/job.rb deleted file mode 100644 index 1311d1f..0000000 --- a/lib/postal/job.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -require "nifty/utils/random_string" - -module Postal - class Job - - def initialize(id, params = {}) - @id = id - @params = params - on_initialize - end - - attr_reader :id - - def params - @params || {} - end - - def on_initialize - # Called whenever the class is initialized. Can be overriden. - end - - def on_error(exception) - # Called if there's an exception while processing the perform block. - # Receives the exception. - end - - def perform - end - - def log(text) - Worker.logger.info(text) - end - - def self.queue(queue, params = {}) - job_id = Nifty::Utils::RandomString.generate(length: 10).upcase - job_payload = { "params" => params, "class_name" => name, "id" => job_id, "queue" => queue } - Postal::Worker.job_queue(queue).publish(job_payload.to_json, persistent: false) - job_id - end - - end -end diff --git a/lib/postal/message_db/message.rb b/lib/postal/message_db/message.rb index db70c3b..46df1c9 100644 --- a/lib/postal/message_db/message.rb +++ b/lib/postal/message_db/message.rb @@ -445,7 +445,11 @@ module Postal # def bounce!(bounce_message) create_delivery("Bounced", details: "We've received a bounce message for this e-mail. See for details.") - SendWebhookJob.queue(:main, server_id: database.server_id, event: "MessageBounced", payload: { _original_message: id, _bounce: bounce_message.id }) + + WebhookRequest.trigger(server, "MessageBounced", { + original_message: webhook_hash, + bounce: bounce_message.webhook_hash + }) end # @@ -461,7 +465,12 @@ module Postal def create_load(request) update("loaded" => Time.now.to_f) if loaded.nil? database.insert(:loads, { message_id: id, ip_address: request.ip, user_agent: request.user_agent, timestamp: Time.now.to_f }) - SendWebhookJob.queue(:main, server_id: database.server_id, event: "MessageLoaded", payload: { _message: id, ip_address: request.ip, user_agent: request.user_agent }) + + WebhookRequest.trigger(server, "MessageLoaded", { + message: webhook_hash, + ip_address: request.ip, + user_agent: request.user_agent + }) end # diff --git a/lib/postal/message_requeuer.rb b/lib/postal/message_requeuer.rb deleted file mode 100644 index 31e5510..0000000 --- a/lib/postal/message_requeuer.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Postal - class MessageRequeuer - - def run - Signal.trap("INT") { @running ? @exit = true : Process.exit(0) } - Signal.trap("TERM") { @running ? @exit = true : Process.exit(0) } - - log "Running message requeuer..." - loop do - @running = true - QueuedMessage.requeue_all - @running = false - check_exit - sleep 5 - end - end - - private - - def log(text) - Postal.logger.info text, component: "message-requeuer" - end - - def check_exit - return unless @exit - - log "Exiting" - Process.exit(0) - end - - end -end diff --git a/lib/postal/rabbit_mq.rb b/lib/postal/rabbit_mq.rb deleted file mode 100644 index 2703cd5..0000000 --- a/lib/postal/rabbit_mq.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -require "postal/config" -require "bunny" - -module Postal - module RabbitMQ - - def self.create_connection - bunny_host = ["localhost"] - - if Postal.config.rabbitmq&.host.is_a?(Array) - bunny_host = Postal.config.rabbitmq&.host - elsif Postal.config.rabbitmq&.host.is_a?(String) - bunny_host = [Postal.config.rabbitmq&.host] - end - - conn = Bunny.new( - hosts: bunny_host, - port: Postal.config.rabbitmq&.port || 5672, - tls: Postal.config.rabbitmq&.tls || false, - verify_peer: Postal.config.rabbitmq&.verify_peer || true, - tls_ca_certificates: Postal.config.rabbitmq&.tls_ca_certificates || ["/etc/ssl/certs/ca-certificates.crt"], - username: Postal.config.rabbitmq&.username || "guest", - password: Postal.config.rabbitmq&.password || "guest", - vhost: Postal.config.rabbitmq&.vhost || nil - ) - conn.start - conn - end - - def self.create_channel - conn = create_connection - conn.create_channel(nil, Postal.config.workers.threads) - end - - end -end diff --git a/lib/postal/tracking_middleware.rb b/lib/postal/tracking_middleware.rb index 82a25a0..1593af9 100644 --- a/lib/postal/tracking_middleware.rb +++ b/lib/postal/tracking_middleware.rb @@ -94,16 +94,20 @@ module Postal user_agent: request.user_agent, timestamp: time }) - SendWebhookJob.queue(:main, - server_id: message_db.server_id, - event: "MessageLinkClicked", - payload: { - _message: link["message_id"], - url: link["url"], - token: link["token"], - ip_address: request.ip, - user_agent: request.user_agent - }) + + begin + message_webhook_hash = message_db.message(link["message_id"]).webhook_hash + WebhookRequest.trigger(message_db.server, "MessageLinkClicked", { + message: message_webhook_hash, + url: link["url"], + token: link["token"], + ip_address: request.ip, + user_agent: request.user_agent + }) + rescue Postal::MessageDB::Message::NotFound + # If we can't find the message that this link is associated with, we'll just ignore it + # and not trigger any webhooks. + end end [307, { "Location" => link["url"] }, ["Redirected to: #{link['url']}"]] diff --git a/lib/postal/worker.rb b/lib/postal/worker.rb deleted file mode 100644 index 969d81e..0000000 --- a/lib/postal/worker.rb +++ /dev/null @@ -1,220 +0,0 @@ -# frozen_string_literal: true - -module Postal - class Worker - - def initialize(queues) - @initial_queues = queues - @active_queues = {} - @process_name = $0 - @running_jobs = [] - end - - def work - logger.info "Worker running with #{Postal.config.workers.threads} threads" - - Signal.trap("INT") do - @exit = true - set_process_name - end - Signal.trap("TERM") do - @exit = true - set_process_name - end - - self.class.job_channel.prefetch(Postal.config.workers.threads) - @initial_queues.each { |queue| join_queue(queue) } - - exit_checks = 0 - loop do - if @exit && @running_jobs.empty? - logger.info "Exiting immediately because no jobs running" - exit 0 - elsif @exit - if exit_checks >= 60 - logger.info "Job did not finish in a timely manner. Exiting" - exit 0 - end - if exit_checks.zero? - logger.info "Exit requested but job is running. Waiting for job to finish." - end - sleep 60 - exit_checks += 1 - else - manage_ip_queues - sleep 1 - end - end - end - - private - - def receive_job(delivery_info, properties, message) - if message && message["class_name"] - @running_jobs << message["id"] - set_process_name - start_time = Time.now - Thread.current[:job_id] = message["id"] - logger.info "Processing job" - begin - klass = message["class_name"].constantize.new(message["id"], message["params"]) - klass.perform - GC.start - rescue StandardError => e - klass.on_error(e) if defined?(klass) - logger.exception(e) - if defined?(Sentry) - Sentry.capture_exception(e, extra: { job_id: message["id"] }) - end - ensure - logger.info "Finished job", time: (Time.now - start_time).to_i - end - end - ensure - Thread.current[:job_id] = nil - self.class.job_channel.ack(delivery_info.delivery_tag) - @running_jobs.delete(message["id"]) if message["id"] - set_process_name - - if @exit && @running_jobs.empty? - logger.info "Exiting because all jobs have finished." - exit 0 - end - end - - def join_queue(queue) - if @active_queues[queue] - logger.error "attempted to join queue but already joined", queue: queue - else - consumer = self.class.job_queue(queue).subscribe(manual_ack: true) do |delivery_info, properties, body| - message = begin - JSON.parse(body) - rescue StandardError - nil - end - - logger.tagged(job_id: message["id"], queue: queue, job_class: message["class_name"]) do - receive_job(delivery_info, properties, message) - end - end - @active_queues[queue] = consumer - logger.info "joined queue", queue: queue - end - end - - def leave_queue(queue) - if consumer = @active_queues[queue] - consumer.cancel - @active_queues.delete(queue) - logger.info "left queue", queue: queue - else - logger.error "requested to leave queue, but not joined", queue: queue - end - end - - def manage_ip_queues - @ip_queues ||= [] - @ip_to_id_mapping ||= {} - @unassigned_ips ||= [] - @pairs ||= {} - @counter ||= 0 - - if @counter >= 15 - @ip_to_id_mapping = {} - @unassigned_ips = [] - @counter = 0 - else - @counter += 1 - end - - # Get all IP addresses on the system - current_ip_addresses = Socket.ip_address_list.map(&:ip_address) - - # Map them to an actual ID in the database if we can and cache that - needed_ip_ids = [] - current_ip_addresses.each do |ip| - need = nil - if id = @ip_to_id_mapping[ip] - # We know this IPs ID, we'll just use that. - need = id - elsif @unassigned_ips.include?(ip) - # We know this IP isn't valid. We don't need to do anything - elsif !self.class.local_ip?(ip) && ip_address = IPAddress.where("ipv4 = ? OR ipv6 = ?", ip, ip).first - # We need to look this up - @pairs[ip_address.ipv4] = ip_address.ipv6 - @ip_to_id_mapping[ip] = ip_address.id - need = ip_address.id - else - @unassigned_ips << ip - end - - next unless need - - pair = @pairs[ip] || @pairs.key(ip) - if pair.nil? || current_ip_addresses.include?(pair) - needed_ip_ids << @ip_to_id_mapping[ip] - else - logger.info "Host has '#{ip}' but its pair (#{pair}) isn't here. Cannot add now." - end - end - - # Make an array of needed queue names - # Work out what we need to actually do here - missing_queues = needed_ip_ids - @ip_queues - unwanted_queues = @ip_queues - needed_ip_ids - # Leave the queues we don't want any more - unwanted_queues.each do |id| - leave_queue("outgoing-#{id}") - @ip_queues.delete(id) - ip_addresses_to_clear = [] - @ip_to_id_mapping.each do |iip, iid| - if id == iid - ip_addresses_to_clear << iip - end - end - ip_addresses_to_clear.each { |ip| @ip_to_id_mapping.delete(ip) } - end - # Join any missing queues - missing_queues.uniq.each do |id| - join_queue("outgoing-#{id}") - @ip_queues << id - end - end - - def set_process_name - prefix = @process_name.to_s - prefix += " [exiting]" if @exit - if @running_jobs.empty? - $0 = "#{prefix} (idle)" - else - $0 = "#{prefix} (running #{@running_jobs.join(', ')})" - end - end - - def logger - self.class.logger - end - - class << self - - def logger - Postal.logger - end - - def job_channel - @job_channel ||= Postal::RabbitMQ.create_channel - end - - def job_queue(name) - @job_queues ||= {} - @job_queues[name] ||= job_channel.queue("deliver-jobs-#{name}", durable: true, arguments: { "x-message-ttl" => 60_000 }) - end - - def local_ip?(ip) - !!(ip =~ /\A(127\.|fe80:|::)/) - end - - end - - end -end diff --git a/lib/tasks/postal.rake b/lib/tasks/postal.rake index 81770f6..b0b363c 100644 --- a/lib/tasks/postal.rake +++ b/lib/tasks/postal.rake @@ -1,27 +1,6 @@ # frozen_string_literal: true namespace :postal do - desc "Start the cron worker" - task cron: :environment do - require "clockwork" - require Rails.root.join("config", "cron") - trap("TERM") do - puts "Exiting..." - Process.exit(0) - end - Clockwork.run - end - - desc "Start SMTP Server" - task smtp_server: :environment do - Postal::SMTPServer::Server.new(debug: true).run - end - - desc "Start the message requeuer" - task requeuer: :environment do - Postal::MessageRequeuer.new.run - end - desc "Run all migrations on message databases" task migrate_message_databases: :environment do Server.all.each do |server| diff --git a/lib/worker/jobs/base_job.rb b/lib/worker/jobs/base_job.rb new file mode 100644 index 0000000..f324b51 --- /dev/null +++ b/lib/worker/jobs/base_job.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Worker + module Jobs + class BaseJob + + def initialize(logger:) + @logger = logger + end + + def call + # Override me. + end + + def work_completed? + @work_completed == true + end + + private + + def work_completed! + @work_completed = true + end + + attr_reader :logger + + end + end +end diff --git a/lib/worker/jobs/process_queued_messages_job.rb b/lib/worker/jobs/process_queued_messages_job.rb new file mode 100644 index 0000000..60c8ba3 --- /dev/null +++ b/lib/worker/jobs/process_queued_messages_job.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Worker + module Jobs + class ProcessQueuedMessagesJob < BaseJob + + def call + @lock_time = Time.current + @locker = Postal.locker_name_with_suffix(SecureRandom.hex(8)) + + find_ip_addresses + lock_message_for_processing + obtain_locked_messages + process_messages + @messages_to_process + end + + private + + # Returns an array of IP address IDs that are present on the host that is + # running this job. + # + # @return [Array] + def find_ip_addresses + ip_addresses = { 4 => [], 6 => [] } + Socket.ip_address_list.each do |address| + next if local_ip?(address.ip_address) + + ip_addresses[address.ipv4? ? 4 : 6] << address.ip_address + end + @ip_addresses = IPAddress.where(ipv4: ip_addresses[4]).or(IPAddress.where(ipv6: ip_addresses[6])).pluck(:id) + end + + # Is the given IP address a local address? + # + # @param [String] ip + # @return [Boolean] + def local_ip?(ip) + !!(ip =~ /\A(127\.|fe80:|::)/) + end + + # Obtain a queued message from the database for processing + # + # @return [void] + def lock_message_for_processing + QueuedMessage.where(ip_address_id: [nil, @ip_addresses]) + .where(locked_by: nil, locked_at: nil) + .ready_with_delayed_retry + .limit(1) + .update_all(locked_by: @locker, locked_at: @lock_time) + end + + # Get a full list of all messages which we can process (i.e. those which have just + # been locked by us for processing) + # + # @return [void] + def obtain_locked_messages + @messages_to_process = QueuedMessage.where(locked_by: @locker, locked_at: @lock_time) + end + + # Process the messages we obtained from the database + # + # @return [void] + def process_messages + @messages_to_process.each do |message| + work_completed! + UnqueueMessageService.new(queued_message: message, logger: logger).call + end + end + + end + end +end diff --git a/lib/worker/jobs/process_webhook_requests_job.rb b/lib/worker/jobs/process_webhook_requests_job.rb new file mode 100644 index 0000000..c33de5f --- /dev/null +++ b/lib/worker/jobs/process_webhook_requests_job.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Worker + module Jobs + class ProcessWebhookRequestsJob < BaseJob + + def call + @lock_time = Time.current + @locker = Postal.locker_name_with_suffix(SecureRandom.hex(8)) + + lock_request_for_processing + obtain_locked_requests + process_requests + end + + private + + # Obtain a webhook request from the database for processing + # + # @return [void] + def lock_request_for_processing + WebhookRequest.unlocked + .ready + .limit(1) + .update_all(locked_by: @locker, locked_at: @lock_time) + end + + # Get a full list of all webhooks which we can process (i.e. those which have just + # been locked by us for processing) + # + # @return [void] + def obtain_locked_requests + @requests_to_process = WebhookRequest.where(locked_by: @locker, locked_at: @lock_time) + end + + # Process the webhook requests we obtained from the database + # + # @return [void] + def process_requests + @requests_to_process.each do |request| + work_completed! + request.deliver + end + end + + end + end +end diff --git a/lib/worker/process.rb b/lib/worker/process.rb new file mode 100644 index 0000000..34eaec2 --- /dev/null +++ b/lib/worker/process.rb @@ -0,0 +1,242 @@ +# frozen_string_literal: true + +module Worker + # The Postal Worker process is responsible for handling all background tasks. This includes processing of all + # messages, webhooks and other administrative tasks. There are two main types of background work which is completed, + # jobs and scheduled tasks. + # + # The 'Jobs' here allow for the continuous monitoring of a database table (or queue) and processing of any new items + # which may appear in that. The polling takes place every 5 seconds by default and the work is able to run multiple + # threads to look for and process this work. + # + # Scheduled Tasks allow for code to be executed on a ROUGH schedule. This is used for administrative tasks. A single + # thread will run within each worker process and attempt to acquire the 'tasks' role. If successful it will run all + # tasks which are due to be run. The tasks are then scheduled to run again at a future time. Workers which are not + # successful in acquiring the role will not run any tasks but will still attempt to acquire a lock in case the current + # acquiree disappears. + # + # The worker process will run until it receives a TERM or INT signal. It will then attempt to gracefully shut down + # after it has completed any outstanding jobs which are already inflight. + class Process + + # An array of job classes that should be processed each time the worker ticks. + # + # @return [Array] + JOBS = [ + Jobs::ProcessQueuedMessagesJob, + Jobs::ProcessWebhookRequestsJob + ].freeze + + # An array of tasks that should be processed + # + # @return [Array] + TASKS = [ + ActionDeletionsScheduledTask, + CheckAllDNSScheduledTask, + CleanupAuthieSessionsScheduledTask, + ExpireHeldMessagesScheduledTask, + ProcessMessageRetentionScheduledTask, + PruneSuppressionListsScheduledTask, + PruneWebhookRequestsScheduledTask, + SendNotificationsScheduledTask + ].freeze + + # @param [Integer] thread_count The number of worker threads to run in this process + def initialize(thread_count: 2, work_sleep_time: 5, task_sleep_time: 60) + @thread_count = thread_count + @exit_pipe_read, @exit_pipe_write = IO.pipe + @work_sleep_time = work_sleep_time + @task_sleep_time = task_sleep_time + @threads = [] + end + + def run + logger.tagged(component: "worker") do + setup_traps + start_work_threads + start_tasks_thread + wait_for_threads + end + end + + private + + # Install signal traps to allow for graceful shutdown + # + # @return [void] + def setup_traps + trap("INT") { receive_signal("INT") } + trap("TERM") { receive_signal("TERM") } + end + + # Receive a signal and set the shutdown flag + # + # @param [String] signal The signal that was received z + # @return [void] + def receive_signal(signal) + puts "Received #{signal} signal. Stopping when able." + @shutdown = true + @exit_pipe_write.close + end + + # Wait for the period of time and return true or false if shutdown has been requested. If the shutdown is + # requested during the wait, it will return immediately otherwise it will return false when it has finished + # waiting for the period of time. + # + # @param [Integer] wait_time The time to wait for + # @return [Boolean] + def shutdown_after_wait?(wait_time) + @exit_pipe_read.wait_readable(wait_time) ? true : false + end + + # Wait for all threads to complete + # + # @return [void] + def wait_for_threads + @threads.each(&:join) + end + + # Start the worker threads + # + # @return [void] + def start_work_threads + logger.info "starting #{@thread_count} work threads" + @thread_count.times do |index| + start_work_thread(index) + end + end + + # Start a worker thread + # + # @return [void] + def start_work_thread(index) + @threads << Thread.new do + logger.tagged(component: "worker", thread: "work#{index}") do + logger.info "started work thread #{index}" + loop do + work_completed = work + + if shutdown_after_wait?(work_completed ? 0 : @work_sleep_time) + break + end + end + + logger.info "stopping work thread #{index}" + end + end + end + + # Actually perform the work for this tick. This will call each job which has been registered. + # + # @return [Boolean] Whether any work was completed in this job or not + def work + completed_work = 0 + ActiveRecord::Base.connection_pool.with_connection do + JOBS.each do |job_class| + capture_errors do + job = job_class.new(logger: logger) + job.call + + completed_work += 1 if job.work_completed? + end + end + end + completed_work.positive? + end + + # Start the tasks thread + # + # @return [void] + def start_tasks_thread + logger.info "starting tasks thread" + @threads << Thread.new do + logger.tagged(component: "worker", thread: "tasks") do + loop do + run_tasks + + if shutdown_after_wait?(@task_sleep_time) + break + end + end + + logger.info "stopping tasks thread" + ActiveRecord::Base.connection_pool.with_connection do + if WorkerRole.release(:tasks) + logger.info "releasesd tasks role" + end + end + end + end + end + + # Run the tasks. This will attempt to acquire the tasks role and if successful it will all the registered + # tasks if they are due to be run. + # + # @return [void] + def run_tasks + role_acquisition_status = ActiveRecord::Base.connection_pool.with_connection do + WorkerRole.acquire(:tasks) + end + + case role_acquisition_status + when :stolen + logger.info "acquired task role by stealing it from a lazy worker" + when :created + logger.info "acquired task role by creating it" + when :renewed + logger.debug "acquired task role by renewing it" + else + logger.debug "could not acquire task role, not doing anything" + return false + end + + ActiveRecord::Base.connection_pool.with_connection do + TASKS.each { |task| run_task(task) } + end + end + + # Run a single task + # + # @param [Class] task The task to run + # @return [void] + def run_task(task) + logger.tagged task: task do + scheduled_task = ScheduledTask.find_by(name: task.to_s) + if scheduled_task.nil? + logger.info "no existing task object, creating it now" + scheduled_task = ScheduledTask.create!(name: task.to_s, next_run_after: task.next_run_after) + end + + next unless scheduled_task.next_run_after < Time.current + + logger.info "running task" + + capture_errors { task.new(logger: logger).call } + + next_run_after = task.next_run_after + logger.info "scheduling task to next run at #{next_run_after}" + scheduled_task.update!(next_run_after: next_run_after) + end + end + + # Return the logger + # + # @return [Klogger::Logger] + def logger + Postal.logger + end + + # Capture exceptions and handle this as appropriate. + # + # @yield The block of code to run + # @return [void] + def capture_errors + yield + rescue StandardError => e + logger.error "#{e.class} (#{e.message})" + e.backtrace.each { |line| logger.error line } + Sentry.capture_exception(e) if defined?(Sentry) + end + + end +end diff --git a/script/send_html_email.rb b/script/send_html_email.rb index 96ead18..ba81aa0 100644 --- a/script/send_html_email.rb +++ b/script/send_html_email.rb @@ -43,21 +43,10 @@ end c = OpenSSL::SSL::SSLContext.new c.verify_mode = OpenSSL::SSL::VERIFY_NONE -<<<<<<< Updated upstream -Net::SMTP.start("127.0.0.1", 2525) do |smtp| - smtp.send_message mail.to_s, mail.from.first, mail.to.first -end -======= -1000.times.map do - Thread.new do - smtp = Net::SMTP.new("77.72.7.155", 25) - # smtp.enable_starttls(c) - smtp.disable_starttls - smtp.start("localhost") - smtp.send_message mail.to_s, mail.from.first, mail.to.first - smtp.finish - end -end.each(&:join) ->>>>>>> Stashed changes +smtp = Net::SMTP.new("127.0.0.1", 2525) +smtp.enable_starttls(c) +smtp.start("localhost") +smtp.send_message mail.to_s, mail.from.first, mail.to.first +smtp.finish puts "Sent" diff --git a/script/smtp_server.rb b/script/smtp_server.rb new file mode 100644 index 0000000..f9ea52f --- /dev/null +++ b/script/smtp_server.rb @@ -0,0 +1,4 @@ +# frozen_string_literal: true + +require_relative "../config/environment" +Postal::SMTPServer::Server.new(debug: true).run diff --git a/script/worker.rb b/script/worker.rb index 121f5ce..42b7781 100755 --- a/script/worker.rb +++ b/script/worker.rb @@ -2,4 +2,4 @@ # frozen_string_literal: true require_relative "../config/environment" -Postal::Worker.new([:main]).work +Worker::Process.new.run diff --git a/spec/app/models/worker_role_spec.rb b/spec/app/models/worker_role_spec.rb new file mode 100644 index 0000000..4776960 --- /dev/null +++ b/spec/app/models/worker_role_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe WorkerRole do + let(:locker_name) { "test" } + + before do + allow(Postal).to receive(:locker_name).and_return(locker_name) + end + + describe ".acquire" do + context "when there are no existing roles" do + it "returns :created" do + expect(WorkerRole.acquire("test")).to eq(:created) + end + end + + context "when the current process holds a lock for a role" do + it "returns :renewed" do + create(:worker_role, role: "test", worker: "test", acquired_at: 1.minute.ago) + expect(WorkerRole.acquire("test")).to eq(:renewed) + end + end + + context "when the role has become stale" do + it "returns :stolen" do + create(:worker_role, role: "test", worker: "another", acquired_at: 10.minute.ago) + expect(WorkerRole.acquire("test")).to eq(:stolen) + end + end + + context "when the role is already locked by another worker" do + it "returns false" do + create(:worker_role, role: "test", worker: "another", acquired_at: 1.minute.ago) + expect(WorkerRole.acquire("test")).to eq(false) + end + end + end + + describe ".release" do + context "when the role is locked by the current worker" do + it "deletes the role and returns true" do + role = create(:worker_role, role: "test", worker: "test") + expect(WorkerRole.release("test")).to eq(true) + expect(WorkerRole.find_by(id: role.id)).to be_nil + end + end + + context "when the role is locked by another worker" do + it "does not delete the role and returns false" do + role = create(:worker_role, role: "test", worker: "another") + expect(WorkerRole.release("test")).to eq(false) + expect(WorkerRole.find_by(id: role.id)).to be_present + end + end + end +end diff --git a/spec/factories/ip_address_factory.rb b/spec/factories/ip_address_factory.rb new file mode 100644 index 0000000..b1a479e --- /dev/null +++ b/spec/factories/ip_address_factory.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: ip_addresses +# +# id :integer not null, primary key +# hostname :string(255) +# ipv4 :string(255) +# ipv6 :string(255) +# priority :integer +# created_at :datetime +# updated_at :datetime +# ip_pool_id :integer +# +FactoryBot.define do + factory :ip_address do + ip_pool + ipv4 { "10.0.0.1" } + ipv6 { "2001:0db8:85a3:0000:0000:8a2e:0370:7334" } + hostname { "ip.example.com" } + end +end diff --git a/spec/factories/ip_pool_factory.rb b/spec/factories/ip_pool_factory.rb new file mode 100644 index 0000000..2870204 --- /dev/null +++ b/spec/factories/ip_pool_factory.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: ip_pools +# +# id :integer not null, primary key +# default :boolean default(FALSE) +# name :string(255) +# uuid :string(255) +# created_at :datetime +# updated_at :datetime +# +# Indexes +# +# index_ip_pools_on_uuid (uuid) +# +FactoryBot.define do + factory :ip_pool do + name { "Default Pool" } + default { true } + end +end diff --git a/spec/factories/queued_message_factory.rb b/spec/factories/queued_message_factory.rb new file mode 100644 index 0000000..b15d456 --- /dev/null +++ b/spec/factories/queued_message_factory.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: queued_messages +# +# id :integer not null, primary key +# attempts :integer default(0) +# batch_key :string(255) +# domain :string(255) +# locked_at :datetime +# locked_by :string(255) +# manual :boolean default(FALSE) +# retry_after :datetime +# created_at :datetime +# updated_at :datetime +# ip_address_id :integer +# message_id :integer +# route_id :integer +# server_id :integer +# +# Indexes +# +# index_queued_messages_on_domain (domain) +# index_queued_messages_on_message_id (message_id) +# index_queued_messages_on_server_id (server_id) +# +FactoryBot.define do + factory :queued_message do + server + message_id { 1234 } + domain { "example.com" } + batch_key { nil } + + trait :locked do + locked_by { "worker1" } + locked_at { 5.minutes.ago } + end + end +end diff --git a/spec/factories/webhook_factory.rb b/spec/factories/webhook_factory.rb new file mode 100644 index 0000000..8c5f594 --- /dev/null +++ b/spec/factories/webhook_factory.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :webhook do + server + name { "Example Webhook" } + url { "https://example.com" } + all_events { true } + end +end diff --git a/spec/factories/webhook_request_factory.rb b/spec/factories/webhook_request_factory.rb new file mode 100644 index 0000000..7a51f1b --- /dev/null +++ b/spec/factories/webhook_request_factory.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: webhook_requests +# +# id :integer not null, primary key +# attempts :integer default(0) +# error :text(65535) +# event :string(255) +# locked_at :datetime +# locked_by :string(255) +# payload :text(65535) +# retry_after :datetime +# url :string(255) +# uuid :string(255) +# created_at :datetime +# server_id :integer +# webhook_id :integer +# +# Indexes +# +# index_webhook_requests_on_locked_by (locked_by) +# +FactoryBot.define do + factory :webhook_request do + webhook + url { "https://example.com" } + event { "ExampleEvent" } + payload { { "hello" => "world" } } + + before(:create) do |webhook_request| + webhook_request.server = webhook_request.webhook&.server + end + + trait :locked do + locked_by { "test" } + locked_at { 5.minutes.ago } + end + end +end diff --git a/spec/factories/worker_role_factory.rb b/spec/factories/worker_role_factory.rb new file mode 100644 index 0000000..be5f497 --- /dev/null +++ b/spec/factories/worker_role_factory.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :worker_role do + role { "test" } + end +end diff --git a/spec/lib/worker/jobs/process_queued_messages_job.rb b/spec/lib/worker/jobs/process_queued_messages_job.rb new file mode 100644 index 0000000..f2ce50a --- /dev/null +++ b/spec/lib/worker/jobs/process_queued_messages_job.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +require "rails_helper" + +module Worker + module Jobs + + RSpec.describe ProcessQueuedMessagesJob do + subject(:job) { described_class.new(logger: Postal.logger) } + let(:mocked_service) { instance_double(UnqueueMessageService) } + + before do + allow(UnqueueMessageService).to receive(:new).and_return(mocked_service) + allow(mocked_service).to receive(:call).with(any_args) + end + + describe "#call" do + context "when there are no queued messages" do + it "does nothing" do + job.call + expect(UnqueueMessageService).to_not have_received(:new) + end + end + + context "when there is an unlocked queued message for an IP address that is not ours" do + it "does nothing" do + ip_address = create(:ip_address) + queued_message = create(:queued_message, ip_address: ip_address) + job.call + expect(UnqueueMessageService).to_not have_received(:new) + expect(queued_message.reload.locked?).to be false + end + end + + context "when there is an unlocked queued message without an IP address without a retry time" do + it "locks the message and calls the service" do + queued_message = create(:queued_message, ip_address: nil, retry_after: nil) + job.call + expect(UnqueueMessageService).to have_received(:new).with(logger: kind_of(Klogger::Logger), queued_message: queued_message) + expect(mocked_service).to have_received(:call) + expect(queued_message.reload.locked?).to be true + expect(queued_message.locked_by).to eq Postal.locker_name + expect(queued_message.locked_at).to be_within(1.second).of(Time.current) + end + end + + context "when there is an unlocked queued message without an IP address without a retry time in the past" do + it "locks the message and calls the service" do + queued_message = create(:queued_message, ip_address: nil, retry_after: 10.minutes.ago) + job.call + expect(UnqueueMessageService).to have_received(:new).with(logger: kind_of(Klogger::Logger), queued_message: queued_message) + expect(mocked_service).to have_received(:call) + expect(queued_message.reload.locked?).to be true + expect(queued_message.locked_by).to eq Postal.locker_name + expect(queued_message.locked_at).to be_within(1.second).of(Time.current) + end + end + + context "when there is an unlocked queued message without an IP address without a retry time in the future" do + it "does nothing" do + queued_message = create(:queued_message, ip_address: nil, retry_after: 10.minutes.from_now) + job.call + expect(UnqueueMessageService).to_not have_received(:new) + expect(queued_message.reload.locked?).to be false + end + end + + context "when there is a locked queued message without an IP address without a retry time" do + it "does nothing" do + queued_message = create(:queued_message, :locked, ip_address: nil, retry_after: nil) + job.call + expect(UnqueueMessageService).to_not have_received(:new) + expect(queued_message.reload.locked?).to be true + end + end + + context "when there is a locked queued message without an IP address with a retry time in the past" do + it "does nothing" do + queued_message = create(:queued_message, :locked, ip_address: nil, retry_after: 1.month.ago) + job.call + expect(UnqueueMessageService).to_not have_received(:new) + expect(queued_message.reload.locked?).to be true + end + end + + context "when there is an unlocked queued message with an IP address that is ours without a retry time" do + it "locks the message and calls the service" do + ip_address = create(:ip_address, ipv4: "10.20.30.40") + allow(Socket).to receive(:ip_address_list).and_return([Addrinfo.new(["AF_INET", 1, "localhost.localdomain", "10.20.30.40"])]) + queued_message = create(:queued_message, ip_address: ip_address) + job.call + expect(UnqueueMessageService).to have_received(:new).with(logger: kind_of(Klogger::Logger), queued_message: queued_message) + expect(mocked_service).to have_received(:call) + expect(queued_message.reload.locked?).to be true + expect(queued_message.locked_by).to eq Postal.locker_name + expect(queued_message.locked_at).to be_within(1.second).of(Time.current) + end + end + + context "when there is an unlocked queued message with an IP address that is ours without a retry time in the future" do + it "does nothing" do + ip_address = create(:ip_address, ipv4: "10.20.30.40") + allow(Socket).to receive(:ip_address_list).and_return([Addrinfo.new(["AF_INET", 1, "localhost.localdomain", "10.20.30.40"])]) + queued_message = create(:queued_message, ip_address: ip_address, retry_after: 1.month.from_now) + job.call + expect(UnqueueMessageService).to_not have_received(:new) + expect(queued_message.reload.locked?).to be false + end + end + end + end + + end +end diff --git a/spec/lib/worker/jobs/process_webhook_requests_job.rb b/spec/lib/worker/jobs/process_webhook_requests_job.rb new file mode 100644 index 0000000..7624060 --- /dev/null +++ b/spec/lib/worker/jobs/process_webhook_requests_job.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require "rails_helper" + +module Worker + module Jobs + + RSpec.describe ProcessWebhookRequestsJob do + subject(:job) { described_class.new(logger: Postal.logger) } + + before do + allow_any_instance_of(WebhookRequest).to receive(:deliver) + end + + context "when there are no requests to process" do + it "does nothing" do + job.call + expect(job.work_completed?).to be false + end + end + + context "when there is a unlocked request with no retry time" do + it "delivers the request" do + create(:webhook_request) + job.call + expect(job.work_completed?).to be true + end + end + + context "when there is an unlocked request with a retry time in the past" do + it "delivers the request" do + create(:webhook_request, retry_after: 1.minute.ago) + job.call + expect(job.work_completed?).to be true + end + end + + context "when there is an unlocked request with a retry time in the future" do + it "does nothing" do + create(:webhook_request, retry_after: 1.minute.from_now) + job.call + expect(job.work_completed?).to be false + end + end + + context "when there is a locked requested without a retry time" do + it "does nothing" do + create(:webhook_request, :locked) + job.call + expect(job.work_completed?).to be false + end + end + end + + end +end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index ec9f85c..67dc502 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -8,6 +8,7 @@ require "spec_helper" require "factory_bot" require "timecop" require "database_cleaner" +require "webmock/rspec" DatabaseCleaner.allow_remote_database_url = true ActiveRecord::Base.logger = Logger.new("/dev/null") From 8d21adcbd45c279dd21137cabbf6f98f2698b090 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Wed, 14 Feb 2024 17:46:11 +0000 Subject: [PATCH 03/56] docs: add quick contributing instructions --- CONTRIBUTING.md | 49 +++++++++++++++++++++++++++++++++++++++ config/postal.example.yml | 38 ++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 CONTRIBUTING.md create mode 100644 config/postal.example.yml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..0724581 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,49 @@ +# Contributing to Postal + +This doc explains how to go about running Postal in development to allow you to make contributions to the project. + +## Dependencies + +You will need a MySQL database server to get started. Postal needs to be able to make databases within that server whenever new mail servers are created so the permissions that you use should be suitable for that. + +You'll also need Ruby. Postal currently uses Ruby 3.2.1. Install that using whichever version manager takes your fancy - rbenv, asdf, rvm etc. + +## Clone + +You'll need to clone the repository + +``` +git clone git@github.com:postalserver/postal +``` + +Once cloned, you can install the Ruby dependencies using bundler. + +``` +bundle install +``` + +## Configuration + +At present, configuration is handled using a config file. This lives in `config/postal/postal.yml`. An example configuration file is provided in `config/postal.example.yml`. This example is for development use only and not an example for production use. + +You'll also need a key for signing. You can generate one of these like this: + +``` +openssl genrsa -out config/postal/signing.key 2048 +``` + +## Running + +The neatest way to run postal is to ensure that `./bin` is your `$PATH` and then use one of the following commands. + +* `bin/dev` - will run all components of the application using Foreman +* `bin/postal` - will run the Postal binary providing access to running individual components or other tools. + +## Database initialization + +Use the commands below to initialize your database and make your first user. + +``` +postal initialize +postal make-user +``` diff --git a/config/postal.example.yml b/config/postal.example.yml new file mode 100644 index 0000000..fc71488 --- /dev/null +++ b/config/postal.example.yml @@ -0,0 +1,38 @@ +web: + host: postal.example.dev + protocol: https + +web_server: + bind_address: 0.0.0.0 + port: 4010 + +smtp_server: + port: 2525 + +logging: + rails_log: true + stdout: true + +main_db: + host: 127.0.0.1 + username: root + password: + database: postal + +message_db: + host: 127.0.0.1 + username: root + password: + prefix: postal + +smtp: + host: 127.0.0.1 + port: 2525 + username: + password: + from_name: Postal + from_address: postal@yourdomain.com + +rails: + environment: development + secret_key: 7f27856d26e864bafd49d0df37ad3d1339086e86ef0447e0f1814dde5277452fea97dab9e3aad6dfa11bfe359c82ce302d97bf1e58f6103c4408e4fbad4eeccf From 6eb16c3fab0ede68b0784066c5573bd2db1f67dd Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Thu, 15 Feb 2024 09:36:09 +0000 Subject: [PATCH 04/56] chore: add script for generating smtp TLS certificates --- script/generate_tls_certificate.rb | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 script/generate_tls_certificate.rb diff --git a/script/generate_tls_certificate.rb b/script/generate_tls_certificate.rb new file mode 100644 index 0000000..e886536 --- /dev/null +++ b/script/generate_tls_certificate.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require File.expand_path("../lib/postal/config", __dir__) +require "openssl" + +unless File.exist?(Postal.smtp_private_key_path) + key = OpenSSL::PKey::RSA.new(2048).to_s + File.write(Postal.smtp_private_key_path, key) + puts "Created new private key for encrypting SMTP connections" +end + +unless File.exist?(Postal.smtp_certificate_path) + cert = OpenSSL::X509::Certificate.new + cert.subject = cert.issuer = OpenSSL::X509::Name.parse("/C=GB/O=Test/OU=Test/CN=Test") + cert.not_before = Time.now + cert.not_after = Time.now + (365 * 24 * 60 * 60) + cert.public_key = Postal.smtp_private_key.public_key + cert.serial = 0x0 + cert.version = 2 + cert.sign Postal.smtp_private_key, OpenSSL::Digest.new("SHA256") + File.write(Postal.smtp_certificate_path, cert.to_pem) + puts "Created new self signed certificate for encrypting SMTP connections" +end From e0403ba641b978e67766ab9283e0904a00d2977b Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Thu, 15 Feb 2024 11:46:29 +0000 Subject: [PATCH 05/56] refactor: refactor webhook deliveries --- app/models/webhook_request.rb | 44 ------- app/services/webhook_delivery_service.rb | 94 ++++++++++++-- lib/postal/message_db/database.rb | 2 +- lib/postal/message_db/webhooks.rb | 2 +- .../jobs/process_webhook_requests_job.rb | 3 +- .../services/webhook_delivery_service_spec.rb | 119 ++++++++++++++++++ 6 files changed, 209 insertions(+), 55 deletions(-) create mode 100644 spec/app/services/webhook_delivery_service_spec.rb diff --git a/app/models/webhook_request.rb b/app/models/webhook_request.rb index c475155..7609b5a 100644 --- a/app/models/webhook_request.rb +++ b/app/models/webhook_request.rb @@ -28,8 +28,6 @@ class WebhookRequest < ApplicationRecord include HasUUID include HasLocking - RETRIES = { 1 => 2.minutes, 2 => 3.minutes, 3 => 6.minutes, 4 => 10.minutes, 5 => 15.minutes }.freeze - belongs_to :server belongs_to :webhook, optional: true @@ -38,48 +36,6 @@ class WebhookRequest < ApplicationRecord serialize :payload, Hash - def deliver - payload = { event: event, timestamp: created_at.to_f, payload: self.payload, uuid: uuid }.to_json - Postal.logger.tagged(event: event, url: url) do - Postal.logger.info "Sending webhook request" - result = Postal::HTTP.post(url, sign: true, json: payload, timeout: 5) - self.attempts += 1 - self.retry_after = RETRIES[self.attempts]&.from_now - server.message_db.webhooks.record( - event: event, - url: url, - webhook_id: webhook_id, - attempt: self.attempts, - timestamp: Time.now.to_f, - payload: self.payload.to_json, - uuid: uuid, - status_code: result[:code], - body: result[:body], - will_retry: (retry_after ? 0 : 1) - ) - - if result[:code] >= 200 && result[:code] < 300 - Postal.logger.info "Received #{result[:code]} status code. That's OK." - destroy! - webhook&.update_column(:last_used_at, Time.now) - true - else - Postal.logger.error "Received #{result[:code]} status code. That's not OK." - self.error = "Couldn't send to URL. Code received was #{result[:code]}" - if retry_after - Postal.logger.info "Will retry #{retry_after} (this was attempt #{self.attempts})" - self.locked_by = nil - self.locked_at = nil - save! - else - Postal.logger.info "Have tried #{self.attempts} times. Giving up." - destroy! - end - false - end - end - end - class << self def trigger(server, event, payload = {}) diff --git a/app/services/webhook_delivery_service.rb b/app/services/webhook_delivery_service.rb index 617054e..4aeb50c 100644 --- a/app/services/webhook_delivery_service.rb +++ b/app/services/webhook_delivery_service.rb @@ -2,18 +2,96 @@ class WebhookDeliveryService - def initialize(webhook_delivery:) - @webhook_delivery = webhook_delivery + RETRIES = { 1 => 2.minutes, 2 => 3.minutes, 3 => 6.minutes, 4 => 10.minutes, 5 => 15.minutes }.freeze + + def initialize(webhook_request:) + @webhook_request = webhook_request end - # TODO: move the logic from WebhookDelivery#deliver in to this service. - # def call - if @webhook_delivery.deliver - log "Succesfully delivered" - else - log "Delivery failed" + logger.tagged(webhook: @webhook_request.webhook_id, webhook_request: @webhook_request.id) do + generate_payload + send_request + record_attempt + appreciate_http_result + update_webhook_request end end + def success? + @success == true + end + + private + + def generate_payload + @payload = { + event: @webhook_request.event, + timestamp: @webhook_request.created_at.to_f, + payload: @webhook_request.payload, + uuid: @webhook_request.uuid + }.to_json + end + + def send_request + @http_result = Postal::HTTP.post(@webhook_request.url, + sign: true, + json: @payload, + timeout: 5) + + @success = (@http_result[:code] >= 200 && @http_result[:code] < 300) + end + + def record_attempt + @webhook_request.attempts += 1 + + if success? + @webhook_request.retry_after = nil + else + @webhook_request.retry_after = RETRIES[@webhook_request.attempts]&.from_now + end + + @attempt = @webhook_request.server.message_db.webhooks.record( + event: @webhook_request.event, + url: @webhook_request.url, + webhook_id: @webhook_request.webhook_id, + attempt: @webhook_request.attempts, + timestamp: Time.now.to_f, + payload: @webhook_request.payload.to_json, + uuid: @webhook_request.uuid, + status_code: @http_result[:code], + body: @http_result[:body], + will_retry: @webhook_request.retry_after.present? + ) + end + + def appreciate_http_result + if success? + logger.info "Received #{@http_result[:code]} status code. That's OK." + @webhook_request.destroy! + @webhook_request.webhook&.update_column(:last_used_at, Time.current) + return + end + + logger.error "Received #{@http_result[:code]} status code. That's not OK." + @webhook_request.error = "Couldn't send to URL. Code received was #{@http_result[:code]}" + end + + def update_webhook_request + if @webhook_request.retry_after + logger.info "Will retry #{@webhook_request.retry_after} (this was attempt #{@webhook_request.attempts})" + @webhook_request.locked_by = nil + @webhook_request.locked_at = nil + @webhook_request.save! + return + end + + logger.info "Have tried #{@webhook_request.attempts} times. Giving up." + @webhook_request.destroy! + end + + def logger + Postal.logger + end + end diff --git a/lib/postal/message_db/database.rb b/lib/postal/message_db/database.rb index 061e1e4..c5c3e38 100644 --- a/lib/postal/message_db/database.rb +++ b/lib/postal/message_db/database.rb @@ -325,7 +325,7 @@ module Postal result = connection.query(query, cast_booleans: true) time = Time.now.to_f - start_time logger.debug " \e[4;34mMessageDB Query (#{time.round(2)}s) \e[0m \e[33m#{query}\e[0m" - if time.positive? && query =~ /\A(SELECT|UPDATE|DELETE) / + if time > 0.05 && query =~ /\A(SELECT|UPDATE|DELETE) / id = Nifty::Utils::RandomString.generate(length: 6).upcase explain_result = ResultForExplainPrinter.new(connection.query("EXPLAIN #{query}")) logger.info " [#{id}] EXPLAIN #{query}" diff --git a/lib/postal/message_db/webhooks.rb b/lib/postal/message_db/webhooks.rb index cfe4bcd..97bbad5 100644 --- a/lib/postal/message_db/webhooks.rb +++ b/lib/postal/message_db/webhooks.rb @@ -12,7 +12,7 @@ module Postal @database.insert(:webhook_requests, attributes) end - def list(page) + def list(page = 1) result = @database.select_with_pagination(:webhook_requests, page, order: :timestamp, direction: "desc") result[:records] = result[:records].map { |i| Request.new(i) } result diff --git a/lib/worker/jobs/process_webhook_requests_job.rb b/lib/worker/jobs/process_webhook_requests_job.rb index c33de5f..53751a1 100644 --- a/lib/worker/jobs/process_webhook_requests_job.rb +++ b/lib/worker/jobs/process_webhook_requests_job.rb @@ -39,7 +39,8 @@ module Worker def process_requests @requests_to_process.each do |request| work_completed! - request.deliver + + WebhookDeliveryService.new(webhook_request: request).call end end diff --git a/spec/app/services/webhook_delivery_service_spec.rb b/spec/app/services/webhook_delivery_service_spec.rb new file mode 100644 index 0000000..3527e71 --- /dev/null +++ b/spec/app/services/webhook_delivery_service_spec.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe WebhookDeliveryService do + let(:server) { GLOBAL_SERVER } + let(:webhook) { create(:webhook, server: server) } + let(:webhook_request) { create(:webhook_request, :locked, webhook: webhook) } + + subject(:service) { described_class.new(webhook_request: webhook_request) } + + let(:response_status) { 200 } + let(:response_body) { "OK" } + + before do + stub_request(:post, webhook.url).to_return(status: response_status, body: response_body) + end + + after do + server.message_db.provisioner.clean + end + + describe "#call" do + it "sends a request to the webhook's url" do + service.call + expect(WebMock).to have_requested(:post, webhook.url).with({ + body: { + event: webhook_request.event, + timestamp: webhook_request.created_at.to_f, + payload: webhook_request.payload, + uuid: webhook_request.uuid + }.to_json, + headers: { + "Content-Type" => "application/json", + "X-Postal-Signature" => /\A[a-z0-9\/+]+=*\z/i + } + }) + end + + context "when the endpoint returns a 200 OK" do + it "creates a webhook request for the server" do + service.call + expect(server.message_db.webhooks.list(1)[:total]).to eq(1) + webhook_request = server.message_db.webhooks.list(1)[:records].first + expect(webhook_request).to have_attributes( + event: webhook_request.event, + url: webhook_request.url, + status_code: 200, + body: "OK", + uuid: webhook_request.uuid, + will_retry?: false, + payload: webhook_request.payload, + attempt: 1, + timestamp: webhook_request.timestamp + ) + end + + it "deletes the webhook request" do + service.call + expect { webhook_request.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + + it "updates the last used at time on the webhook" do + service.call + expect(webhook.reload.last_used_at).to be_within(1.second).of(Time.current) + end + end + + context "when the request returns a 500 Internal Server Error for the first time" do + let(:response_status) { 500 } + let(:response_body) { "internal server error!" } + + it "unlocks the webhook request if locked" do + expect { service.call }.to change { webhook_request.reload.locked? }.from(true).to(false) + end + + it "updates the retry time and attempt counter" do + service.call + expect(webhook_request.reload.attempts).to eq(1) + expect(webhook_request.retry_after).to be_within(1.second).of(2.minutes.from_now) + end + end + + context "when the request returns a 500 Internal Server Error for the second time" do + let(:webhook_request) { create(:webhook_request, :locked, webhook: webhook, attempts: 1) } + let(:response_status) { 500 } + let(:response_body) { "internal server error!" } + + it "updates the retry time and attempt counter" do + service.call + expect(webhook_request.reload.attempts).to eq(2) + expect(webhook_request.retry_after).to be_within(1.second).of(3.minutes.from_now) + end + end + + context "when the request returns a 500 Internal Server Error for the sixth time" do + let(:webhook_request) { create(:webhook_request, :locked, webhook: webhook, attempts: 5) } + let(:response_status) { 500 } + let(:response_body) { "internal server error!" } + + it "creates a webhook request for the server" do + service.call + expect(server.message_db.webhooks.list(1)[:total]).to eq(1) + webhook_request = server.message_db.webhooks.list(1)[:records].first + expect(webhook_request).to have_attributes( + status_code: 500, + body: "internal server error!", + will_retry?: false, + attempt: 6 + ) + end + + it "deletes the webhook request" do + service.call + expect { webhook_request.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + end +end From 72715fe5f8d8977a0fb058973ba31d781cb7cfe1 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Thu, 15 Feb 2024 20:11:04 +0000 Subject: [PATCH 06/56] chore: upgrade ruby to 3.2.2 and nodejs to 20.x --- .ruby-version | 3 +-- CONTRIBUTING.md | 2 +- Dockerfile | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.ruby-version b/.ruby-version index 667b8b1..be94e6f 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1,2 +1 @@ -3.2.1 - +3.2.2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0724581..4b3a15a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ This doc explains how to go about running Postal in development to allow you to You will need a MySQL database server to get started. Postal needs to be able to make databases within that server whenever new mail servers are created so the permissions that you use should be suitable for that. -You'll also need Ruby. Postal currently uses Ruby 3.2.1. Install that using whichever version manager takes your fancy - rbenv, asdf, rvm etc. +You'll also need Ruby. Postal currently uses Ruby 3.2.2. Install that using whichever version manager takes your fancy - rbenv, asdf, rvm etc. ## Clone diff --git a/Dockerfile b/Dockerfile index b1b17c1..cf07188 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ -FROM ruby:3.2.1-bullseye AS base +FROM ruby:3.2.2-bullseye AS base SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN apt-get update \ && apt-get install -y --no-install-recommends \ software-properties-common dirmngr apt-transport-https \ - && (curl -sL https://deb.nodesource.com/setup_14.x | bash -) \ + && (curl -sL https://deb.nodesource.com/setup_20.x | bash -) \ && rm -rf /var/lib/apt/lists/* # Install main dependencies From 465f4d82476416296e6bd4cdfaaa0f34305edfc9 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Thu, 15 Feb 2024 23:30:24 +0000 Subject: [PATCH 07/56] test: fix test that was failing due to time differences --- .../postal/smtp_server/client/data_spec.rb | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/spec/lib/postal/smtp_server/client/data_spec.rb b/spec/lib/postal/smtp_server/client/data_spec.rb index d79fbb2..b4a9d8f 100644 --- a/spec/lib/postal/smtp_server/client/data_spec.rb +++ b/spec/lib/postal/smtp_server/client/data_spec.rb @@ -50,10 +50,10 @@ module Postal client.handle("HELO test.example.com") client.handle("MAIL FROM: test@test.com") client.handle("RCPT TO: #{route.name}@#{route.domain.name}") - client.handle("DATA") end it "logs headers" do + client.handle("DATA") client.handle("Subject: Test") client.handle("From: test@test.com") client.handle("To: test1@example.com") @@ -66,17 +66,20 @@ module Postal end it "logs content" do - client.handle("Subject: Test") - client.handle("") - client.handle("This is some content for the message.") - client.handle("It will keep going.") - expect(client.instance_variable_get("@data")).to eq <<~DATA - Received: from test.example.com (1.2.3.4 [1.2.3.4]) by #{Postal.config.dns.smtp_server_hostname} with SMTP; #{Time.now.utc.rfc2822}\r - Subject: Test\r - \r - This is some content for the message.\r - It will keep going.\r - DATA + Timecop.freeze do + client.handle("DATA") + client.handle("Subject: Test") + client.handle("") + client.handle("This is some content for the message.") + client.handle("It will keep going.") + expect(client.instance_variable_get("@data")).to eq <<~DATA + Received: from test.example.com (1.2.3.4 [1.2.3.4]) by #{Postal.config.dns.smtp_server_hostname} with SMTP; #{Time.now.utc.rfc2822}\r + Subject: Test\r + \r + This is some content for the message.\r + It will keep going.\r + DATA + end end end end From b4016f6b49900a09b5d10d1d84fcc5cc2518d9ca Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Mon, 19 Feb 2024 22:27:22 +0000 Subject: [PATCH 08/56] test: add tests for message unqueueing This adds a comprehensive set of tests for the message unqueueing service. Additionally, it improves how message databases are used for testing environments. --- app/services/unqueue_message_service.rb | 11 +- lib/postal/message_db/database.rb | 3 +- lib/postal/message_db/message.rb | 22 +- lib/postal/rspec_helpers.rb | 7 - lib/postal/send_result.rb | 5 + lib/test_logger.rb | 47 ++ .../models/outgoing_message_prototype_spec.rb | 27 +- .../incoming_messages_spec.rb | 743 ++++++++++++++++++ .../outgoing_message_spec.rb | 600 ++++++++++++++ .../services/unqueue_message_service_spec.rb | 36 + .../services/webhook_delivery_service_spec.rb | 6 +- spec/factories/address_endpoint_factory.rb | 8 + spec/factories/queued_message_factory.rb | 25 +- spec/factories/server_factory.rb | 5 + spec/factories/smtp_endpoint_factory.rb | 11 + spec/helpers/message_db_mocking.rb | 41 + spec/helpers/message_factory.rb | 78 ++ spec/lib/postal/message_db/database_spec.rb | 3 +- spec/lib/postal/message_parser_spec.rb | 30 +- .../smtp_server/client/finished_spec.rb | 6 +- spec/rails_helper.rb | 28 +- 21 files changed, 1658 insertions(+), 84 deletions(-) create mode 100644 lib/test_logger.rb create mode 100644 spec/app/services/unqueue_message_service/incoming_messages_spec.rb create mode 100644 spec/app/services/unqueue_message_service/outgoing_message_spec.rb create mode 100644 spec/app/services/unqueue_message_service_spec.rb create mode 100644 spec/factories/address_endpoint_factory.rb create mode 100644 spec/factories/smtp_endpoint_factory.rb create mode 100644 spec/helpers/message_db_mocking.rb create mode 100644 spec/helpers/message_factory.rb diff --git a/app/services/unqueue_message_service.rb b/app/services/unqueue_message_service.rb index 5433771..3b797ce 100644 --- a/app/services/unqueue_message_service.rb +++ b/app/services/unqueue_message_service.rb @@ -163,7 +163,10 @@ class UnqueueMessageService queued_message.message.inspect_message if queued_message.message.inspected is_spam = queued_message.message.spam_score > queued_message.server.spam_threshold - queued_message.message.update(spam: true) if is_spam + if is_spam + queued_message.message.update(spam: true) + log "message is spam (scored #{queued_message.message.spam_score}, threshold is #{queued_message.server.spam_threshold})" + end queued_message.message.append_headers( "X-Postal-Spam: #{queued_message.message.spam ? 'yes' : 'no'}", "X-Postal-Spam-Threshold: #{queued_message.server.spam_threshold}", @@ -285,7 +288,7 @@ class UnqueueMessageService # If the message is a hard fail, send a bounce message for this message. log "sending a bounce because message hard failed" if bounce_id = queued_message.send_bounce - log_details += ". " unless log_details =~ /\.\z/ + log_details += "." unless log_details =~ /\.\z/ log_details += " Sent bounce message to sender (see message )" end end @@ -445,7 +448,8 @@ class UnqueueMessageService if recent_hard_fails >= 1 && queued_message.server.message_db.suppression_list.add(:recipient, queued_message.message.rcpt_to, reason: "too many hard fails") log "Added #{queued_message.message.rcpt_to} to suppression list because #{recent_hard_fails} hard fails in 24 hours" result.details += "." if result.details =~ /\.\z/ - result.details += " Recipient added to suppression list (too many hard fails)." + result.details += " " if result.details.present? + result.details += "Recipient added to suppression list (too many hard fails)." end end @@ -477,6 +481,7 @@ class UnqueueMessageService if defined?(Sentry) Sentry.capture_exception(e, extra: { server_id: queued_message.server_id, queued_message_id: queued_message.message_id }) end + queued_message.message&.create_delivery("Error", details: "An internal error occurred while sending " \ "this message. This message will be retried " \ diff --git a/lib/postal/message_db/database.rb b/lib/postal/message_db/database.rb index c5c3e38..3455c8a 100644 --- a/lib/postal/message_db/database.rb +++ b/lib/postal/message_db/database.rb @@ -12,9 +12,10 @@ module Postal end - def initialize(organization_id, server_id) + def initialize(organization_id, server_id, database_name: nil) @organization_id = organization_id @server_id = server_id + @database_name = database_name end attr_reader :organization_id diff --git a/lib/postal/message_db/message.rb b/lib/postal/message_db/message.rb index 46df1c9..818b425 100644 --- a/lib/postal/message_db/message.rb +++ b/lib/postal/message_db/message.rb @@ -39,6 +39,10 @@ module Postal @attributes = attributes end + def reload + self.class.find_one(@database, @attributes["id"]) + end + # # Return the server for this message # @@ -200,9 +204,9 @@ module Postal # #  Save this message # - def save + def save(queue_on_create: true) save_raw_message - persisted? ? _update : _create + persisted? ? _update : _create(queue: queue_on_create) self end @@ -346,8 +350,14 @@ module Postal # # Create a new item in the message queue for this message # - def add_to_message_queue(options = {}) - QueuedMessage.create!(message: self, server_id: @database.server_id, batch_key: batch_key, domain: recipient_domain, route_id: route_id, manual: options[:manual]).id + def add_to_message_queue(**options) + QueuedMessage.create!({ + message: self, + server_id: @database.server_id, + batch_key: batch_key, + domain: recipient_domain, + route_id: route_id + }.merge(options)) end # @@ -572,7 +582,7 @@ module Postal @database.update("messages", @attributes.except(:id), where: { id: @attributes["id"] }) end - def _create + def _create(queue: true) self.timestamp = Time.now.to_f if timestamp.blank? self.status = "Pending" if status.blank? self.token = Nifty::Utils::RandomString.generate(length: 12) if token.blank? @@ -581,7 +591,7 @@ module Postal @database.statistics.increment_all(timestamp, scope) Statistic.global.increment!(:total_messages) Statistic.global.increment!("total_#{scope}".to_sym) - add_to_message_queue + add_to_message_queue if queue end def mail diff --git a/lib/postal/rspec_helpers.rb b/lib/postal/rspec_helpers.rb index 44fd0ab..fd55188 100644 --- a/lib/postal/rspec_helpers.rb +++ b/lib/postal/rspec_helpers.rb @@ -3,13 +3,6 @@ module Postal module RspecHelpers - def with_global_server(&block) - server = Server.find(GLOBAL_SERVER.id) - block.call(server) - ensure - server.message_db.provisioner.clean - end - def create_plain_text_message(server, text, to = "test@example.com", override_attributes = {}) domain = create(:domain, owner: server) attributes = { from: "test@#{domain.name}", subject: "Test Plain Text Message" }.merge(override_attributes) diff --git a/lib/postal/send_result.rb b/lib/postal/send_result.rb index af4c40a..b0505ba 100644 --- a/lib/postal/send_result.rb +++ b/lib/postal/send_result.rb @@ -13,5 +13,10 @@ module Postal attr_accessor :time attr_accessor :suppress_bounce + def initialize + @details = "" + yield self if block_given? + end + end end diff --git a/lib/test_logger.rb b/lib/test_logger.rb new file mode 100644 index 0000000..f07abfd --- /dev/null +++ b/lib/test_logger.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +class TestLogger + + def initialize + @log_lines = [] + @group_set = Klogger::GroupSet.new + @print = false + end + + def print! + @print = true + end + + def add(level, message, **tags) + @group_set.groups.each do |group| + tags = group[:tags].merge(tags) + end + + @log_lines << { level: level, message: message, tags: tags } + puts message if @print + true + end + + [:info, :debug, :warn, :error].each do |level| + define_method(level) do |message, **tags| + add(level, message, **tags) + end + end + + def tagged(**tags, &block) + @group_set.call_without_id(**tags, &block) + end + + def log_line(match) + @log_lines.reverse.each do |log_line| + return log_line if match.is_a?(String) && log_line[:message] == match + return log_line if match.is_a?(Regexp) && log_line[:message] =~ match + end + nil + end + + def has_logged?(match) + !!log_line(match) + end + +end diff --git a/spec/app/models/outgoing_message_prototype_spec.rb b/spec/app/models/outgoing_message_prototype_spec.rb index 01c3d57..80490fd 100644 --- a/spec/app/models/outgoing_message_prototype_spec.rb +++ b/spec/app/models/outgoing_message_prototype_spec.rb @@ -3,21 +3,20 @@ require "rails_helper" describe OutgoingMessagePrototype do + let(:server) { create(:server) } it "should create a new message" do - with_global_server do |server| - domain = create(:domain, owner: server) - prototype = OutgoingMessagePrototype.new(server, "127.0.0.1", "TestSuite", { - from: "test@#{domain.name}", - to: "test@example.com", - subject: "Test Message", - plain_body: "A plain body!" - }) + domain = create(:domain, owner: server) + prototype = OutgoingMessagePrototype.new(server, "127.0.0.1", "TestSuite", { + from: "test@#{domain.name}", + to: "test@example.com", + subject: "Test Message", + plain_body: "A plain body!" + }) - expect(prototype.valid?).to be true - message = prototype.create_message("test@example.com") - expect(message).to be_a Hash - expect(message[:id]).to be_a Integer - expect(message[:token]).to be_a String - end + expect(prototype.valid?).to be true + message = prototype.create_message("test@example.com") + expect(message).to be_a Hash + expect(message[:id]).to be_a Integer + expect(message[:token]).to be_a String end end diff --git a/spec/app/services/unqueue_message_service/incoming_messages_spec.rb b/spec/app/services/unqueue_message_service/incoming_messages_spec.rb new file mode 100644 index 0000000..3925d41 --- /dev/null +++ b/spec/app/services/unqueue_message_service/incoming_messages_spec.rb @@ -0,0 +1,743 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe UnqueueMessageService do + let(:server) { create(:server) } + let(:logger) { TestLogger.new } + let(:queued_message) { create(:queued_message, server: server) } + subject(:service) { described_class.new(queued_message: queued_message, logger: logger) } + + # We're going to, for now, just stop the SMTP sender from doing anything here because + # we don't want to leak out of this test in to the real world. + before do + smtp_sender_mock = double("SMTPSender") + allow(Postal::SMTPSender).to receive(:new).and_return(smtp_sender_mock) + allow(smtp_sender_mock).to receive(:start) + allow(smtp_sender_mock).to receive(:finish) + allow(smtp_sender_mock).to receive(:send_message) do + puts "SMTP SENDING DETECTED!" + end + end + + describe "#call" do + context "for an incoming message" do + let(:route) { create(:route, server: server) } + let(:message) { MessageFactory.incoming(server, route: route) } + let(:queued_message) { create(:queued_message, :locked, message: message) } + + context "when the server is suspended" do + before do + allow(queued_message.server).to receive(:suspended?).and_return(true) + end + + it "logs" do + service.call + expect(logger).to have_logged(/server is suspended/) + end + + it "sets the message status to Held" do + service.call + expect(message.reload.status).to eq "Held" + end + + it "creates a Held delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Held", details: /server has been suspended/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the number of attempts is more than the maximum" do + let(:queued_message) { create(:queued_message, :locked, message: message, attempts: Postal.config.general.maximum_delivery_attempts + 1) } + + it "logs" do + service.call + expect(logger).to have_logged(/message has reached maximum number of attempts/) + end + + it "sends a bounce to the sender" do + expect(Postal::BounceMessage).to receive(:new).with(server, queued_message.message) + service.call + end + + it "sets the message status to HardFail" do + service.call + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /maximum number of delivery attempts.*bounce sent to sender/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the message raw data has been removed" do + before do + message.raw_table = nil + message.save + end + + it "logs" do + service.call + expect(logger).to have_logged(/raw message has been removed/) + end + + it "sets the message status to HardFail" do + service.call + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /Raw message has been removed/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the message is a bounce for an existing message" do + let(:existing_message) { MessageFactory.outgoing(server) } + + let(:message) do + MessageFactory.incoming(server) do |msg, mail| + msg.bounce = true + mail["X-Postal-MsgID"] = existing_message.token + end + end + + it "logs" do + service.call + expect(logger).to have_logged(/message is a bounce/) + end + + it "adds the original message as the bounce ID for the received message" do + service.call + expect(message.reload.bounce_for_id).to eq existing_message.id + end + + it "sets the received message status to Processed" do + service.call + expect(message.reload.status).to eq "Processed" + end + + it "creates a Processed delivery on the received message" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Processed", details: /This has been detected as a bounce message for /i) + end + + it "sets the existing message status to Bounced" do + service.call + expect(existing_message.reload.status).to eq "Bounced" + end + + it "creates a Bounced delivery on the original message" do + service.call + delivery = existing_message.deliveries.last + expect(delivery).to have_attributes(status: "Bounced", details: /received a bounce message for this e-mail. See for/i) + end + + it "triggers a MessageBounced webhook event" do + expect(WebhookRequest).to receive(:trigger).with(server, "MessageBounced", { + original_message: kind_of(Hash), + bounce: kind_of(Hash) + }) + service.call + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the message was a bounce but there's no return path for it" do + let(:message) do + MessageFactory.incoming(server) do |msg| + msg.bounce = true + end + end + + it "logs" do + service.call + expect(logger).to have_logged(/no source messages found, hard failing/) + end + + it "sets the message status to HardFail" do + service.call + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /was a bounce but we couldn't link it with any outgoing message/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the message is not a bounce" do + it "increments the stats for the server" do + expect { service.call }.to change { server.message_db.live_stats.total(5) }.by(1) + end + + it "inspects the message and adds headers" do + expect { service.call }.to change { message.reload.inspected }.from(false).to(true) + new_message = message.reload + expect(new_message.headers).to match hash_including( + "x-postal-spam" => ["no"], + "x-postal-spam-threshold" => ["5.0"], + "x-postal-threat" => ["no"] + ) + end + + it "marks the message as spam if the spam score is higher than the server threshold" do + inspection_result = double("Result", spam_score: server.spam_threshold + 1, threat: false, threat_message: nil, spam_checks: []) + allow(Postal::MessageInspection).to receive(:scan).and_return(inspection_result) + service.call + expect(message.reload.spam).to be true + end + end + + context "when the message has a spam score greater than the server's spam failure threshold" do + before do + inspection_result = double("Result", spam_score: 100, threat: false, threat_message: nil, spam_checks: []) + allow(Postal::MessageInspection).to receive(:scan).and_return(inspection_result) + end + + it "logs" do + service.call + expect(logger).to have_logged(/message has a spam score higher than the server's maxmimum/) + end + + it "sets the message status to HardFail" do + service.call + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /spam score is higher than the failure threshold for this server/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the server mode is Development and the message was not manually queued" do + before do + server.update!(mode: "Development") + end + + after do + server.update!(mode: "Live") + end + + it "logs" do + service.call + expect(logger).to have_logged(/server is in development mode/) + end + + it "sets the message status to Held" do + service.call + expect(message.reload.status).to eq "Held" + end + + it "creates a Held delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Held", details: /server is in development mode/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when there is no route for the incoming message" do + let(:route) { nil } + + it "logs" do + service.call + expect(logger).to have_logged(/no route and\/or endpoint available for processing/i) + end + + it "sets the message status to HardFail" do + service.call + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /does not have a route and\/or endpoint available/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the route's spam mode is Quarantine, the message is spam and not manually queued" do + let(:route) { create(:route, server: server, spam_mode: "Quarantine") } + + before do + inspection_result = double("Result", spam_score: server.spam_threshold + 1, threat: false, threat_message: nil, spam_checks: []) + allow(Postal::MessageInspection).to receive(:scan).and_return(inspection_result) + end + + it "logs" do + service.call + expect(logger).to have_logged(/message is spam and route says to quarantine spam message/i) + end + + it "sets the message status to Held" do + service.call + expect(message.reload.status).to eq "Held" + end + + it "creates a Held delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Held", details: /message placed into quarantine/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the route's spam mode is Fail, the message is spam and not manually queued" do + let(:route) { create(:route, server: server, spam_mode: "Fail") } + + before do + inspection_result = double("Result", spam_score: server.spam_threshold + 1, threat: false, threat_message: nil, spam_checks: []) + allow(Postal::MessageInspection).to receive(:scan).and_return(inspection_result) + end + + it "logs" do + service.call + expect(logger).to have_logged(/message is spam and route says to fail spam message/i) + end + + it "sets the message status to HardFail" do + service.call + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /message is spam and the route specifies it should be failed/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the route's mode is Accept" do + it "logs" do + service.call + expect(logger).to have_logged(/route says to accept without endpoint/i) + end + + it "sets the message status to Processed" do + service.call + expect(message.reload.status).to eq "Processed" + end + + it "creates a Processed delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Processed", details: /message has been accepted but not sent to any endpoints/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the route's mode is Hold" do + let(:route) { create(:route, server: server, mode: "Hold") } + + context "when the message was queued manually" do + let(:queued_message) { create(:queued_message, :locked, server: server, message: message, manual: true) } + + it "logs" do + service.call + expect(logger).to have_logged(/route says to hold and message was queued manually/i) + end + + it "sets the message status to Processed" do + service.call + expect(message.reload.status).to eq "Processed" + end + + it "creates a Processed delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Processed", details: /message has been processed/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the message was not queued manually" do + let(:queued_message) { create(:queued_message, :locked, server: server, message: message, manual: false) } + + it "logs" do + service.call + expect(logger).to have_logged(/route says to hold, marking as held/i) + end + + it "sets the message status to Held" do + service.call + expect(message.reload.status).to eq "Held" + end + + it "creates a Held delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Held", details: /message has been accepted but not sent to any endpoints/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + end + + context "when the route's mode is Bounce" do + let(:route) { create(:route, server: server, mode: "Bounce") } + + it "logs" do + service.call + expect(logger).to have_logged(/route says to bounce/i) + end + + it "sends a bounce" do + expect(Postal::BounceMessage).to receive(:new).with(server, queued_message.message) + service.call + end + + it "sets the message status to HardFail" do + service.call + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /message has been bounced because/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the route's mode is Reject" do + let(:route) { create(:route, server: server, mode: "Reject") } + + it "logs" do + service.call + expect(logger).to have_logged(/route says to bounce/i) + end + + it "sends a bounce" do + expect(Postal::BounceMessage).to receive(:new).with(server, queued_message.message) + service.call + end + + it "sets the message status to HardFail" do + service.call + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /message has been bounced because/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the route's endpoint is an HTTP endpoint" do + let(:endpoint) { create(:http_endpoint, server: server) } + let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) } + + it "sends the message to the HTTPSender" do + http_sender_double = double("HTTPSender") + expect(Postal::HTTPSender).to receive(:new).with(endpoint).and_return(http_sender_double) + expect(http_sender_double).to receive(:start).with(no_args) + expect(http_sender_double).to receive(:finish).with(no_args) + expect(http_sender_double).to receive(:send_message).with(queued_message.message).and_return(Postal::SendResult.new) + service.call + end + end + + context "when the route's endpoint is an SMTP endpoint" do + let(:endpoint) { create(:smtp_endpoint, server: server) } + let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) } + + it "sends the message to the SMTPSender" do + smtp_sender_double = double("SMTPSender") + expect(smtp_sender_double).to receive(:start).with(no_args) + expect(smtp_sender_double).to receive(:finish).with(no_args) + expect(smtp_sender_double).to receive(:send_message).with(queued_message.message).and_return(Postal::SendResult.new) + expect(Postal::SMTPSender).to receive(:new).with(message.recipient_domain, nil, { servers: [endpoint] }).and_return(smtp_sender_double) + service.call + end + end + + context "when the route's endpoint is an Address endpoint" do + let(:endpoint) { create(:address_endpoint, server: server) } + let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) } + + it "sends the message to the SMTPSender" do + smtp_sender_double = double("SMTPSender") + expect(smtp_sender_double).to receive(:start).with(no_args) + expect(smtp_sender_double).to receive(:finish).with(no_args) + expect(smtp_sender_double).to receive(:send_message).with(queued_message.message).and_return(Postal::SendResult.new) + expect(Postal::SMTPSender).to receive(:new).with(endpoint.domain, nil, { force_rcpt_to: endpoint.address }).and_return(smtp_sender_double) + service.call + end + end + + context "when the route's endpoint is an unknown endpoint" do + let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: create(:webhook, server: server)) } + + it "logs" do + service.call + expect(logger).to have_logged(/invalid endpoint for route/i) + end + + it "sets the message status to HardFail" do + service.call + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /invalid endpoint for route/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the message has been sent to a sender" do + let(:endpoint) { create(:smtp_endpoint, server: server) } + let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) } + + let(:send_result) do + Postal::SendResult.new do |result| + result.type = "Sent" + result.details = "Sent successfully" + end + end + + before do + smtp_sender_mock = double("SMTPSender") + allow(Postal::SMTPSender).to receive(:new).and_return(smtp_sender_mock) + allow(smtp_sender_mock).to receive(:start) + allow(smtp_sender_mock).to receive(:finish) + allow(smtp_sender_mock).to receive(:send_message).and_return(send_result) + end + + context "when the sender returns a HardFail and bounces are suppressed" do + before do + send_result.type = "HardFail" + send_result.suppress_bounce = true + end + + it "logs" do + service.call + expect(logger).to have_logged(/suppressing bounce message after hard fail/) + end + + it "does not send a bounce" do + allow(Postal::BounceMessage).to receive(:new) + service.call + expect(Postal::BounceMessage).to_not have_received(:new) + end + end + + context "when the sender returns a HardFail and bounces should be sent" do + before do + send_result.type = "HardFail" + send_result.details = "Failed to send message" + end + + it "logs" do + service.call + expect(logger).to have_logged(/sending a bounce because message hard failed/) + end + + it "sends a bounce" do + expect(Postal::BounceMessage).to receive(:new).with(server, queued_message.message) + service.call + end + + it "sets the message status to HardFail" do + service.call + expect(message.reload.status).to eq "HardFail" + end + + it "creates a delivery with the details and a suffix about the bounce message" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /Failed to send message. Sent bounce message to sender \(see message \)/i) + end + end + + it "creates a delivery with the result from the sender" do + send_result.output = "some output here" + send_result.secure = true + send_result.log_id = "12345" + send_result.time = 2.32 + + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Sent", + details: "Sent successfully", + output: "some output here", + sent_with_ssl: true, + log_id: "12345", + time: 2.32) + end + + context "when the sender wants to retry" do + before do + send_result.type = "SoftFail" + send_result.retry = true + end + + it "logs" do + service.call + expect(logger).to have_logged(/message requeued for trying later, at/i) + end + + it "sets the message status to SoftFail" do + service.call + expect(message.reload.status).to eq "SoftFail" + end + + it "updates the queued message with a new retry time" do + Timecop.freeze do + retry_time = 5.minutes.from_now.change(usec: 0) + service.call + expect(queued_message.reload.retry_after).to eq retry_time + end + end + + it "allocates a new IP address to send the message from and updates the queued message" do + expect(queued_message).to receive(:allocate_ip_address) + service.call + end + + it "does not remove the queued message" do + service.call + expect(queued_message.reload).to be_present + end + end + + context "when the sender does not want a retry" do + it "logs" do + service.call + expect(logger).to have_logged(/message processing completed/i) + end + + it "sets the message status to Sent" do + service.call + expect(message.reload.status).to eq "Sent" + end + + it "marks the endpoint as used" do + route.endpoint.update!(last_used_at: nil) + Timecop.freeze do + expect { service.call }.to change { route.endpoint.reload.last_used_at.to_i }.from(0).to(Time.now.to_i) + end + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + end + + context "when an exception occurrs during processing" do + let(:endpoint) { create(:smtp_endpoint, server: server) } + let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) } + + before do + smtp_sender_mock = double("SMTPSender") + allow(Postal::SMTPSender).to receive(:new).and_return(smtp_sender_mock) + allow(smtp_sender_mock).to receive(:start) + allow(smtp_sender_mock).to receive(:finish) + allow(smtp_sender_mock).to receive(:send_message) do + 1 / 0 + end + end + + it "logs" do + service.call + expect(logger).to have_logged(/internal error: ZeroDivisionError/i) + end + + it "creates an Error delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Error", details: /internal error/i) + end + + it "marks the message for retrying later" do + service.call + expect(queued_message.reload.retry_after).to be_present + end + end + end + end +end diff --git a/spec/app/services/unqueue_message_service/outgoing_message_spec.rb b/spec/app/services/unqueue_message_service/outgoing_message_spec.rb new file mode 100644 index 0000000..49e7e70 --- /dev/null +++ b/spec/app/services/unqueue_message_service/outgoing_message_spec.rb @@ -0,0 +1,600 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe UnqueueMessageService do + let(:server) { create(:server) } + let(:logger) { TestLogger.new } + let(:send_result) do + Postal::SendResult.new do |r| + r.type = "Sent" + end + end + subject(:service) { described_class.new(queued_message: queued_message, logger: logger) } + + # We're going to, for now, just stop the SMTP sender from doing anything here because + # we don't want to leak out of this test in to the real world. + before do + smtp_sender_mock = double("SMTPSender") + allow(Postal::SMTPSender).to receive(:new).and_return(smtp_sender_mock) + allow(smtp_sender_mock).to receive(:start) + allow(smtp_sender_mock).to receive(:finish) + allow(smtp_sender_mock).to receive(:send_message).and_return(send_result) + end + + context "for an outgoing message" do + let(:domain) { create(:domain, server: server) } + let(:credential) { create(:credential, server: server) } + let(:message) { MessageFactory.outgoing(server, domain: domain, credential: credential) } + let(:queued_message) { create(:queued_message, :locked, message: message) } + + context "when the server is suspended" do + let(:server) { create(:server, :suspended) } + + it "logs" do + service.call + expect(logger).to have_logged(/server is suspended/) + end + + it "sets the message status to Held" do + service.call + expect(message.reload.status).to eq "Held" + end + + it "creates a Hold delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Held", details: /server has been suspended/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the number of attempts is more than the maximum" do + let(:queued_message) { create(:queued_message, :locked, message: message, attempts: Postal.config.general.maximum_delivery_attempts + 1) } + + it "logs" do + service.call + expect(logger).to have_logged(/message has reached maximum number of attempts/) + end + + it "adds the recipient to the suppression list and logs this" do + Timecop.freeze do + service.call + entry = server.message_db.suppression_list.get(:recipient, message.rcpt_to) + expect(entry).to match hash_including( + "address" => message.rcpt_to, + "type" => "recipient", + "reason" => "too many soft fails" + ) + end + end + + it "sets the message status to Held" do + service.call + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /maximum number of delivery attempts.*added [\w.@]+ to suppression list/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the message raw data has been removed" do + before do + message.raw_table = nil + message.save + end + + it "logs" do + service.call + expect(logger).to have_logged(/raw message has been removed/) + end + + it "sets the message status to Held" do + service.call + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /Raw message has been removed/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the domain belonging to the message no longer exists" do + before do + domain.destroy + end + + it "logs" do + service.call + expect(logger).to have_logged(/message has no domain/) + end + + it "sets the message status to HardFail" do + service.call + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /Message's domain no longer exist/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the message has no rcpt to address" do + before do + message.update(rcpt_to: "") + end + + it "logs" do + service.call + expect(logger).to have_logged(/message has no 'to' address/) + end + + it "sets the message status to HardFail" do + service.call + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /Message doesn't have an RCPT to/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the message has a x-postal-tag header" do + let(:message) do + MessageFactory.outgoing(server, domain: domain) do |_msg, mail| + mail["x-postal-tag"] = "example-tag" + end + end + + it "logs" do + service.call + expect(logger).to have_logged(/added tag: example-tag/) + end + + it "adds the tag to the message object" do + service.call + expect(message.reload.tag).to eq("example-tag") + end + end + + context "when the credential says to hold the message" do + let(:credential) { create(:credential, hold: true) } + + context "when the message was queued manually" do + let(:queued_message) { create(:queued_message, :locked, message: message, manual: true) } + + it "does not hold the message" do + service.call + deliveries = message.deliveries.find { |d| d.status == "Held" } + expect(deliveries).to be_nil + end + end + + context "when the message was not queued manually" do + it "logs" do + service.call + expect(logger).to have_logged(/credential wants us to hold messages/) + end + + it "sets the message status to Held" do + service.call + expect(message.reload.status).to eq "Held" + end + + it "creates a Held delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Held", details: /Credential is configured to hold all messages authenticated/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + end + + context "when the rcpt address is on the suppression list" do + before do + server.message_db.suppression_list.add(:recipient, message.rcpt_to, reason: "testing") + end + + context "when the message was queued manually" do + let(:queued_message) { create(:queued_message, :locked, message: message, manual: true) } + + it "does not hold the message" do + service.call + deliveries = message.deliveries.find { |d| d.status == "Held" } + expect(deliveries).to be_nil + end + end + + context "when the message was not queued manually" do + it "logs" do + service.call + expect(logger).to have_logged(/recipient is on the suppression list/) + end + + it "sets the message status to Held" do + service.call + expect(message.reload.status).to eq "Held" + end + + it "creates a Held delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Held", details: /Recipient \(#{message.rcpt_to}\) is on the suppression list/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + end + + context "when the message content has not been parsed" do + it "parses the content" do + mocked_parser = double("Result") + allow(mocked_parser).to receive(:actioned?).and_return(false) + allow(mocked_parser).to receive(:tracked_links).and_return(0) + allow(mocked_parser).to receive(:tracked_images).and_return(0) + expect(Postal::MessageParser).to receive(:new).with(kind_of(Postal::MessageDB::Message)).and_return(mocked_parser) + service.call + reloaded_message = message.reload + expect(reloaded_message.parsed).to eq 1 + expect(reloaded_message.tracked_links).to eq 0 + expect(reloaded_message.tracked_images).to eq 0 + end + end + + context "when the server has an outbound spam threshold configured" do + let(:server) { create(:server, outbound_spam_threshold: 5.0) } + + it "logs" do + service.call + expect(logger).to have_logged(/inspecting message/) + expect(logger).to have_logged(/message inspected successfully/) + end + + it "inspects the message" do + inspection_result = double("Result", spam_score: 1.0, threat: false, threat_message: nil, spam_checks: []) + expect(Postal::MessageInspection).to receive(:scan).and_return(inspection_result) + service.call + end + + context "when the message spam score is higher than the threshold" do + before do + inspection_result = double("Result", spam_score: 6.0, threat: false, threat_message: nil, spam_checks: []) + allow(Postal::MessageInspection).to receive(:scan).and_return(inspection_result) + end + + it "logs" do + service.call + expect(logger).to have_logged(/message is spam/) + end + + it "sets the spam boolean on the message" do + service.call + expect(message.reload.spam).to be true + end + + it "sets the message status to HardFail" do + service.call + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /Message is likely spam. Threshold is 5.0 and the message scored 6.0/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + end + + context "when the server does not have a outbound spam threshold configured" do + it "does not inspect the message" do + expect(Postal::MessageInspection).to_not receive(:scan) + service.call + end + end + + context "when the message already has an x-postal-msgid header" do + let(:message) do + MessageFactory.outgoing(server, domain: domain, credential: credential) do |_, mail| + mail["x-postal-msgid"] = "existing-id" + end + end + + it "does not another one" do + service.call + expect(message.reload.headers["x-postal-msgid"]).to eq ["existing-id"] + end + + it "does not add dkim headers" do + service.call + expect(message.reload.headers["dkim-signature"]).to be_nil + end + end + + context "when the message does not have a x-postal-msgid header" do + it "adds it" do + service.call + expect(message.reload.headers["x-postal-msgid"]).to match [match(/[a-zA-Z0-9]{12}/)] + end + + it "adds a dkim header" do + service.call + expect(message.reload.headers["dkim-signature"]).to match [match(/\Av=1; a=rsa-sha256/)] + end + end + + context "when the server has exceeded its send limit" do + let(:server) { create(:server, send_limit: 5) } + + before do + 5.times { server.message_db.live_stats.increment("outgoing") } + end + + it "updates the time the limit was exceeded" do + expect { service.call }.to change { server.reload.send_limit_exceeded_at }.from(nil).to(kind_of(Time)) + end + + it "logs" do + service.call + expect(logger).to have_logged(/server send limit has been exceeded/) + end + + it "sets the message status to Held" do + service.call + expect(message.reload.status).to eq "Held" + end + + it "creates a Held delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Held", details: /Message held because send limit \(5\) has been reached/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the server is approaching its send limit" do + let(:server) { create(:server, send_limit: 10) } + + before do + 9.times { server.message_db.live_stats.increment("outgoing") } + end + + it "updates the time the limit was being approached" do + expect { service.call }.to change { server.reload.send_limit_approaching_at }.from(nil).to(kind_of(Time)) + end + + it "does not set the exceeded time" do + expect { service.call }.to_not change { server.reload.send_limit_exceeded_at } # rubocop:disable Lint/AmbiguousBlockAssociation + end + end + + context "when the server is not exceeded or approaching its limit" do + let(:server) { create(:server, :exceeded_send_limit, send_limit: 10) } + + it "clears the approaching and exceeded limits" do + service.call + server.reload + expect(server.send_limit_approaching_at).to be_nil + expect(server.send_limit_exceeded_at).to be_nil + end + end + + context "when the server is in development mode" do + let(:server) { create(:server, mode: "Development") } + + context "when the message was queued manually" do + let(:queued_message) { create(:queued_message, :locked, message: message, manual: true) } + + it "does not hold the message" do + service.call + deliveries = message.deliveries.find { |d| d.status == "Held" } + expect(deliveries).to be_nil + end + end + + context "when the message was not queued manually" do + it "logs" do + service.call + expect(logger).to have_logged(/server is in development mode/) + end + + it "sets the message status to Held" do + service.call + expect(message.reload.status).to eq "Held" + end + + it "creates a Held delivery" do + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Held", details: /Server is in development mode/i) + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + end + + context "when there are no other impediments" do + it "increments the live stats" do + expect { service.call }.to change { server.message_db.live_stats.total(60) }.from(0).to(1) + end + + context "when there is an IP address assigned to the queued message" do + let(:ip) { create(:ip_address) } + let(:queued_message) { create(:queued_message, :locked, message: message, ip_address: ip) } + + it "sends the message to the SMTP sender with the IP" do + service.call + expect(Postal::SMTPSender).to have_received(:new).with(message.recipient_domain, ip) + end + end + + context "when there is no IP address assigned to the queued message" do + it "sends the message to the SMTP sender without an IP" do + service.call + expect(Postal::SMTPSender).to have_received(:new).with(message.recipient_domain, nil) + end + end + + context "when the message hard fails" do + before do + send_result.type = "HardFail" + end + + context "when the recipient has got no hard fails in the last 24 hours" do + it "does not add to the suppression list" do + service.call + expect(server.message_db.suppression_list.all_with_pagination(1)[:total]).to eq 0 + end + end + + context "when the recipient has more than one hard fail in the last 24 hours" do + before do + 2.times do + MessageFactory.outgoing(server, domain: domain, credential: credential) do |msg| + msg.status = "HardFail" + end + end + end + + it "logs" do + service.call + expect(logger).to have_logged(/added #{message.rcpt_to} to suppression list because 2 hard fails in 24 hours/i) + end + + it "adds the recipient to the suppression list" do + service.call + entry = server.message_db.suppression_list.get(:recipient, message.rcpt_to) + expect(entry).to match hash_including( + "address" => message.rcpt_to, + "type" => "recipient", + "reason" => "too many hard fails" + ) + end + end + end + + context "when the message is sent manually and the recipient is on the suppression list" do + let(:queued_message) { create(:queued_message, :locked, message: message, manual: true) } + + before do + server.message_db.suppression_list.add(:recipient, message.rcpt_to, reason: "testing") + end + + it "logs" do + service.call + expect(logger).to have_logged(/removed #{message.rcpt_to} from suppression list/) + end + + it "removes them from the suppression list" do + service.call + expect(server.message_db.suppression_list.get(:recipient, message.rcpt_to)).to be_nil + end + + it "adds the details to the result details" do + service.call + expect(send_result.details).to include("Recipient removed from suppression list") + end + end + + it "creates a delivery with the appropriate details" do + send_result.details = "Sent successfully to mx.example.com" + service.call + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Sent", details: "Sent successfully to mx.example.com") + end + + context "if the message should be retried" do + before do + send_result.type = "SoftFail" + send_result.retry = true + end + + it "logs" do + service.call + expect(logger).to have_logged(/message requeued for trying later/) + end + + it "sets the message status to SoftFail" do + service.call + expect(message.reload.status).to eq "SoftFail" + end + + it "updates the retry time on the queued message" do + Timecop.freeze do + retry_time = 5.minutes.from_now.change(usec: 0) + service.call + expect(queued_message.reload.retry_after).to eq retry_time + end + end + end + + context "if the message should not be retried" do + it "logs" do + service.call + expect(logger).to have_logged(/message processing complete/) + end + + it "sets the message status to Sent" do + service.call + expect(message.reload.status).to eq "Sent" + end + + it "removes the queued message" do + service.call + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + end + end +end diff --git a/spec/app/services/unqueue_message_service_spec.rb b/spec/app/services/unqueue_message_service_spec.rb new file mode 100644 index 0000000..a4db3fb --- /dev/null +++ b/spec/app/services/unqueue_message_service_spec.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe UnqueueMessageService do + let(:server) { create(:server) } + let(:logger) { TestLogger.new } + let(:queued_message) { create(:queued_message, server: server) } + subject(:service) { described_class.new(queued_message: queued_message, logger: logger) } + + describe "#call" do + context "when the backend message does not exist" do + it "deletes the queued message" do + service.call + expect(logger).to have_logged(/unqueue because backend message has been removed/) + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the message is not ready for processing" do + let(:message) { MessageFactory.outgoing(server) } + let(:queued_message) { create(:queued_message, :retry_in_future, message: message) } + + it "does not do anything" do + service.call + expect(logger).to have_logged(/skipping because message isn't ready for processing/) + end + end + + context "when there are other messages to batch with this one" do + context "when the backend message of a sub-message has been removed" do + it "removes the queued message for that message" + end + end + end +end diff --git a/spec/app/services/webhook_delivery_service_spec.rb b/spec/app/services/webhook_delivery_service_spec.rb index 3527e71..a6f1bf3 100644 --- a/spec/app/services/webhook_delivery_service_spec.rb +++ b/spec/app/services/webhook_delivery_service_spec.rb @@ -3,7 +3,7 @@ require "rails_helper" RSpec.describe WebhookDeliveryService do - let(:server) { GLOBAL_SERVER } + let(:server) { create(:server) } let(:webhook) { create(:webhook, server: server) } let(:webhook_request) { create(:webhook_request, :locked, webhook: webhook) } @@ -16,10 +16,6 @@ RSpec.describe WebhookDeliveryService do stub_request(:post, webhook.url).to_return(status: response_status, body: response_body) end - after do - server.message_db.provisioner.clean - end - describe "#call" do it "sends a request to the webhook's url" do service.call diff --git a/spec/factories/address_endpoint_factory.rb b/spec/factories/address_endpoint_factory.rb new file mode 100644 index 0000000..f5e74c2 --- /dev/null +++ b/spec/factories/address_endpoint_factory.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :address_endpoint do + server + sequence(:address) { |n| "test#{n}@example.com" } + end +end diff --git a/spec/factories/queued_message_factory.rb b/spec/factories/queued_message_factory.rb index b15d456..1634615 100644 --- a/spec/factories/queued_message_factory.rb +++ b/spec/factories/queued_message_factory.rb @@ -27,14 +27,33 @@ # FactoryBot.define do factory :queued_message do - server - message_id { 1234 } domain { "example.com" } - batch_key { nil } + + transient do + message { nil } + end + + after(:build) do |message, evaluator| + if evaluator.message + message.server = evaluator.message.server + message.message_id = evaluator.message.id + message.batch_key = evaluator.message.batch_key + message.domain = evaluator.message.recipient_domain + message.route_id = evaluator.message.route_id + else + message.server ||= create(:server) + message.message_id ||= 0 + end + end trait :locked do locked_by { "worker1" } locked_at { 5.minutes.ago } end + + trait :retry_in_future do + attempts { 2 } + retry_after { 1.hour.from_now } + end end end diff --git a/spec/factories/server_factory.rb b/spec/factories/server_factory.rb index 68d7542..6b9b60d 100644 --- a/spec/factories/server_factory.rb +++ b/spec/factories/server_factory.rb @@ -53,5 +53,10 @@ FactoryBot.define do trait :suspended do suspended_at { Time.current } end + + trait :exceeded_send_limit do + send_limit_approaching_at { 5.minutes.ago } + send_limit_exceeded_at { 1.minute.ago } + end end end diff --git a/spec/factories/smtp_endpoint_factory.rb b/spec/factories/smtp_endpoint_factory.rb new file mode 100644 index 0000000..013f2db --- /dev/null +++ b/spec/factories/smtp_endpoint_factory.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :smtp_endpoint do + server + name { "Example SMTP Endpoint" } + hostname { "example.com" } + ssl_mode { "None" } + port { 25 } + end +end diff --git a/spec/helpers/message_db_mocking.rb b/spec/helpers/message_db_mocking.rb new file mode 100644 index 0000000..5ba4718 --- /dev/null +++ b/spec/helpers/message_db_mocking.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module GlobalMessageDB + + class << self + + def find_or_create + return @db if @db + + @db = Postal::MessageDB::Database.new(1, 1, database_name: "postal-test-message-db") + @db.provisioner.provision + end + + def exists? + !@db.nil? + end + + end + +end + +RSpec.configure do |config| + config.before(:example) do + @mocked_message_dbs = [] + allow_any_instance_of(Server).to receive(:message_db).and_wrap_original do |m| + GlobalMessageDB.find_or_create + + message_db = m.call + @mocked_message_dbs << message_db + allow(message_db).to receive(:database_name).and_return("postal-test-message-db") + message_db + end + end + + config.after(:example) do + if GlobalMessageDB.exists? && @mocked_message_dbs.present? + GlobalMessageDB.find_or_create.provisioner.clean + @mocked_message_dbs = [] + end + end +end diff --git a/spec/helpers/message_factory.rb b/spec/helpers/message_factory.rb new file mode 100644 index 0000000..61e03ca --- /dev/null +++ b/spec/helpers/message_factory.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +# This class can be used to generate a message which can be used for the purposes of +# testing within the given server. +class MessageFactory + + def initialize(server) + @server = server + end + + def incoming(route: nil, &block) + @message = @server.message_db.new_message + @message.scope = "incoming" + @message.rcpt_to = "test@example.com" + @message.mail_from = "john@example.com" + + if route + @message.rcpt_to = route.description + @message.route_id = route.id + end + + create_message(&block) + end + + def outgoing(domain: nil, credential: nil, &block) + @message = @server.message_db.new_message + @message.scope = "outgoing" + @message.rcpt_to = "john@example.com" + @message.mail_from = "test@example.com" + + if domain + @message.mail_from = "test@#{domain.name}" + @message.domain_id = domain.id + end + + if credential + @message.credential_id = credential.id + end + + create_message(&block) + end + + class << self + + def incoming(server, **kwargs, &block) + new(server).incoming(**kwargs, &block) + end + + def outgoing(server, **kwargs, &block) + new(server).outgoing(**kwargs, &block) + end + + end + + private + + def create_message + mail = create_mail(@message.rcpt_to, @message.mail_from) + + if block_given? + yield @message, mail + end + + @message.raw_message = mail.to_s + @message.save(queue_on_create: false) + @message + end + + def create_mail(to, from) + mail = Mail.new + mail.to = to + mail.from = from + mail.subject = "An example message" + mail.body = "Hello world!" + mail + end + +end diff --git a/spec/lib/postal/message_db/database_spec.rb b/spec/lib/postal/message_db/database_spec.rb index 0e2639f..ff9eec4 100644 --- a/spec/lib/postal/message_db/database_spec.rb +++ b/spec/lib/postal/message_db/database_spec.rb @@ -4,7 +4,8 @@ require "rails_helper" describe Postal::MessageDB::Database do context "when provisioned" do - subject(:database) { GLOBAL_SERVER.message_db } + let(:server) { create(:server) } + subject(:database) { server.message_db } it "should be a message db" do expect(database).to be_a Postal::MessageDB::Database diff --git a/spec/lib/postal/message_parser_spec.rb b/spec/lib/postal/message_parser_spec.rb index 3d94cef..6eb645a 100644 --- a/spec/lib/postal/message_parser_spec.rb +++ b/spec/lib/postal/message_parser_spec.rb @@ -3,25 +3,23 @@ require "rails_helper" describe Postal::MessageParser do + let(:server) { create(:server) } + it "should not do anything when there are no tracking domains" do - with_global_server do |server| - expect(server.track_domains.size).to eq 0 - message = create_plain_text_message(server, "Hello world!", "test@example.com") - parser = Postal::MessageParser.new(message) - expect(parser.actioned?).to be false - expect(parser.tracked_links).to eq 0 - expect(parser.tracked_images).to eq 0 - end + expect(server.track_domains.size).to eq 0 + message = create_plain_text_message(server, "Hello world!", "test@example.com") + parser = Postal::MessageParser.new(message) + expect(parser.actioned?).to be false + expect(parser.tracked_links).to eq 0 + expect(parser.tracked_images).to eq 0 end it "should replace links in messages" do - with_global_server do |server| - message = create_plain_text_message(server, "Hello world! http://github.com/atech/postal", "test@example.com") - create(:track_domain, server: server, domain: message.domain) - parser = Postal::MessageParser.new(message) - expect(parser.actioned?).to be true - expect(parser.new_body).to match(/^Hello world! https:\/\/click\.#{message.domain.name}/) - expect(parser.tracked_links).to eq 1 - end + message = create_plain_text_message(server, "Hello world! http://github.com/atech/postal", "test@example.com") + create(:track_domain, server: server, domain: message.domain) + parser = Postal::MessageParser.new(message) + expect(parser.actioned?).to be true + expect(parser.new_body).to match(/^Hello world! https:\/\/click\.#{message.domain.name}/) + expect(parser.tracked_links).to eq 1 end end diff --git a/spec/lib/postal/smtp_server/client/finished_spec.rb b/spec/lib/postal/smtp_server/client/finished_spec.rb index 6438419..aa6b48c 100644 --- a/spec/lib/postal/smtp_server/client/finished_spec.rb +++ b/spec/lib/postal/smtp_server/client/finished_spec.rb @@ -7,7 +7,7 @@ module Postal describe Client do let(:ip_address) { "1.2.3.4" } - let(:server) { GLOBAL_SERVER } # We'll use the global server instance for this + let(:server) { create(:server) } subject(:client) { described_class.new(ip_address) } let(:credential) { create(:credential, server: server, type: "SMTP") } @@ -22,10 +22,6 @@ module Postal client.handle("RCPT TO: #{rcpt_to}") end - after do - server.message_db.provisioner.clean - end - describe "when finished sending data" do context "when the data is larger than the maximum message size" do it "returns an error and resets the state" do diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 67dc502..b46a817 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -14,8 +14,10 @@ DatabaseCleaner.allow_remote_database_url = true ActiveRecord::Base.logger = Logger.new("/dev/null") Dir[File.expand_path("factories/*.rb", __dir__)].each { |f| require f } +Dir[File.expand_path("helpers/**/*.rb", __dir__)].each { |f| require f } ActiveRecord::Migration.maintain_test_schema! + RSpec.configure do |config| config.use_transactional_fixtures = true config.infer_spec_type_from_file_location! @@ -25,29 +27,9 @@ RSpec.configure do |config| config.before(:suite) do # Test that the factories are working as they should and then clean up before getting started on # the rest of the suite. - begin - DatabaseCleaner.start - FactoryBot.lint - ensure - DatabaseCleaner.clean - end - - # We're going to create a global server that can be used by any tests. - # Because the mail databases don't use any transactions, all data left in the - # database will be left there unless removed. DatabaseCleaner.start - - # rubocop:disable Lint/ConstantDefinitionInBlock - GLOBAL_SERVER = FactoryBot.create(:server, provision_database: true) - # rubocop:enable Lint/ConstantDefinitionInBlock - end - - config.after(:suite) do - # Remove the global server after the suite has finished running and then - # clean the database in case it left anything lying around. - if defined?(GLOBAL_SERVER) - GLOBAL_SERVER.destroy - DatabaseCleaner.clean - end + FactoryBot.lint + ensure + DatabaseCleaner.clean end end From 27b7ced3bc022d6a6054e8a6c8847e7533378428 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Mon, 19 Feb 2024 22:37:56 +0000 Subject: [PATCH 09/56] chore: remove non-breaking spaces from comments --- app/models/route.rb | 2 +- lib/postal/http_sender.rb | 2 +- lib/postal/message_db/database.rb | 20 ++++++++++---------- lib/postal/message_db/message.rb | 24 ++++++++++++------------ lib/postal/message_db/provisioner.rb | 2 +- 5 files changed, 25 insertions(+), 25 deletions(-) diff --git a/app/models/route.rb b/app/models/route.rb index 3ef415c..362ff98 100644 --- a/app/models/route.rb +++ b/app/models/route.rb @@ -128,7 +128,7 @@ class Route < ApplicationRecord # # This message will create a suitable number of message objects for messages that - #  are destined for this route. It receives a block which can set the message content + # are destined for this route. It receives a block which can set the message content # but most information is specified already. # # Returns an array of created messages. diff --git a/lib/postal/http_sender.rb b/lib/postal/http_sender.rb index e07a637..69011dd 100644 --- a/lib/postal/http_sender.rb +++ b/lib/postal/http_sender.rb @@ -35,7 +35,7 @@ module Postal result.output = response[:body].to_s[0, 500].strip end if response[:code] >= 200 && response[:code] < 300 - #  This is considered a success + # This is considered a success result.type = "Sent" elsif response[:code] >= 500 && response[:code] < 600 # This is temporary. They might fix their server so it should soft fail. diff --git a/lib/postal/message_db/database.rb b/lib/postal/message_db/database.rb index 3455c8a..e84d761 100644 --- a/lib/postal/message_db/database.rb +++ b/lib/postal/message_db/database.rb @@ -59,8 +59,8 @@ module Postal end # - #  Create a new message with the given attributes. This won't be saved to the database - #  until it has been 'save'd. + # Create a new message with the given attributes. This won't be saved to the database + # until it has been 'save'd. # def new_message(attributes = {}) Message.new(self, attributes) @@ -74,35 +74,35 @@ module Postal end # - #  Return the live stats instance + # Return the live stats instance # def live_stats @live_stats ||= LiveStats.new(self) end # - #  Return the statistics instance + # Return the statistics instance # def statistics @statistics ||= Statistics.new(self) end # - #  Return the provisioner instance + # Return the provisioner instance # def provisioner @provisioner ||= Provisioner.new(self) end # - #  Return the provisioner instance + # Return the provisioner instance # def suppression_list @suppression_list ||= SuppressionList.new(self) end # - #  Return the provisioner instance + # Return the provisioner instance # def webhooks @webhooks ||= Webhooks.new(self) @@ -183,7 +183,7 @@ module Postal end # - #  A paginated version of select + # A paginated version of select # def select_with_pagination(table, page, options = {}) page = page.to_i @@ -253,7 +253,7 @@ module Postal # # Deletes a in the database. Accepts a table name, and some options which - #  are shown below: + # are shown below: # # :where => The condition to apply to the query # @@ -269,7 +269,7 @@ module Postal end # - #  Return the correct database name + # Return the correct database name # def database_name @database_name ||= "#{Postal.config.message_db.prefix}-server-#{@server_id}" diff --git a/lib/postal/message_db/message.rb b/lib/postal/message_db/message.rb index 818b425..3e88eb2 100644 --- a/lib/postal/message_db/message.rb +++ b/lib/postal/message_db/message.rb @@ -103,7 +103,7 @@ module Postal end # - #  Return the time that the last delivery was attempted + # Return the time that the last delivery was attempted # def last_delivery_attempt @last_delivery_attempt ||= @attributes["last_delivery_attempt"] ? Time.zone.at(@attributes["last_delivery_attempt"]) : nil @@ -172,14 +172,14 @@ module Postal end # - #  Return all activity entries + # Return all activity entries # def activity_entries @activity_entries ||= (deliveries + clicks + loads).sort_by(&:timestamp) end # - #  Provide access to set and get acceptable attributes + # Provide access to set and get acceptable attributes # def method_missing(name, value = nil, &block) if @attributes.key?(name.to_s) @@ -202,7 +202,7 @@ module Postal end # - #  Save this message + # Save this message # def save(queue_on_create: true) save_raw_message @@ -223,7 +223,7 @@ module Postal end # - #  Delete the message from the database + # Delete the message from the database # def delete return unless persisted? @@ -232,7 +232,7 @@ module Postal end # - #  Return the headers + # Return the headers # def raw_headers if raw_table @@ -243,7 +243,7 @@ module Postal end # - #  Return the full raw message body for this message. + # Return the full raw message body for this message. # def raw_body if raw_table @@ -303,7 +303,7 @@ module Postal end # - #  Return the HTML body for this message + # Return the HTML body for this message # def html_body mail&.html_body @@ -484,7 +484,7 @@ module Postal end # - #  Create a new link + # Create a new link # def create_link(url) hash = Digest::SHA1.hexdigest(url.to_s) @@ -494,7 +494,7 @@ module Postal end # - #  Return a message object that this message is a reply to + # Return a message object that this message is a reply to # def original_messages return nil unless bounce @@ -531,7 +531,7 @@ module Postal end # - #  Return all spam checks for this message + # Return all spam checks for this message # def spam_checks @spam_checks ||= database.select(:spam_checks, where: { message_id: id }) @@ -552,7 +552,7 @@ module Postal def parse_content parse_result = Postal::MessageParser.new(self) if parse_result.actioned? - #  Somethign was changed, update the raw message + # Somethign was changed, update the raw message @database.update(raw_table, { data: parse_result.new_body }, where: { id: raw_body_id }) @database.update(raw_table, { data: parse_result.new_headers }, where: { id: raw_headers_id }) @raw = parse_result.new_body diff --git a/lib/postal/message_db/provisioner.rb b/lib/postal/message_db/provisioner.rb index aa64d3f..dca2054 100644 --- a/lib/postal/message_db/provisioner.rb +++ b/lib/postal/message_db/provisioner.rb @@ -127,7 +127,7 @@ module Postal end # - #  Remove messages from the messages table that are too old to retain + # Remove messages from the messages table that are too old to retain # def remove_messages(max_age = 60) time = (Time.now.utc.to_date - max_age.days).to_time.end_of_day From 8b611000821520313b7deaf1f119abd4cda10625 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Tue, 20 Feb 2024 10:47:11 +0000 Subject: [PATCH 10/56] test: additional tests for batched messages when unqueueing messages --- app/services/unqueue_message_service.rb | 1 + .../services/unqueue_message_service_spec.rb | 65 ++++++++++++++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/app/services/unqueue_message_service.rb b/app/services/unqueue_message_service.rb index 3b797ce..e0d8886 100644 --- a/app/services/unqueue_message_service.rb +++ b/app/services/unqueue_message_service.rb @@ -464,6 +464,7 @@ class UnqueueMessageService # Log the result queued_message.message.create_delivery(result.type, details: result.details, output: result.output, sent_with_ssl: result.secure, log_id: result.log_id, time: result.time) + if result.retry queued_message.retry_later(result.retry.is_a?(Integer) ? result.retry : nil) log "message requeued for trying later", retry_after: queued_message.retry_after diff --git a/spec/app/services/unqueue_message_service_spec.rb b/spec/app/services/unqueue_message_service_spec.rb index a4db3fb..39e7027 100644 --- a/spec/app/services/unqueue_message_service_spec.rb +++ b/spec/app/services/unqueue_message_service_spec.rb @@ -28,8 +28,71 @@ RSpec.describe UnqueueMessageService do end context "when there are other messages to batch with this one" do + let(:domain) { create(:domain, server: server) } + let(:message) { MessageFactory.outgoing(server, domain: domain) } + let(:queued_message) { create(:queued_message, :locked, message: message) } + let(:send_result) { Postal::SendResult.new } + + before do + smtp_sender_mock = double("SMTPSender") + allow(Postal::SMTPSender).to receive(:new).and_return(smtp_sender_mock) + allow(smtp_sender_mock).to receive(:start) + allow(smtp_sender_mock).to receive(:finish) + allow(smtp_sender_mock).to receive(:send_message).and_return(send_result) + end + + before do + # Create 2 extra messages which are similar to the original + @message2 = MessageFactory.outgoing(server, domain: domain) + @queued_message2 = create(:queued_message, message: @message2) + @message3 = MessageFactory.outgoing(server, domain: domain) + @queued_message3 = create(:queued_message, message: @message3) + end + + it "logs" do + service.call + expect(logger).to have_logged(/found 2 associated messages/) + end + + it "sends processes each message" do + allow(service).to receive(:process_message).and_call_original + service.call + expect(service).to have_received(:process_message).with(queued_message) + expect(service).to have_received(:process_message).with(@queued_message2) + expect(service).to have_received(:process_message).with(@queued_message3) + end + + context "when there is a connect error" do + before do + send_result.type = "SoftFail" + send_result.connect_error = true + send_result.details = "Connection Error" + send_result.retry = true + end + + it "uses the same result for subsequent messages" do + service.call + expect(Postal::SMTPSender).to have_received(:new).once + expect(message.reload.status).to eq "SoftFail" + expect(@message2.reload.status).to eq "SoftFail" + expect(@message3.reload.status).to eq "SoftFail" + end + end + context "when the backend message of a sub-message has been removed" do - it "removes the queued message for that message" + before do + @message2.delete + end + + it "logs" do + service.call + expect(logger).to have_logged(/unqueueing because backend message has been removed/) + end + + it "removes the queued message for that message" do + service.call + expect { @queued_message2.reload }.to raise_error(ActiveRecord::RecordNotFound) + end end end end From 2023200d91964882382fe42ca44f340797023ef8 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Tue, 20 Feb 2024 17:08:02 +0000 Subject: [PATCH 11/56] test: add tests for Server model --- Gemfile | 1 + Gemfile.lock | 3 + app/models/server.rb | 95 ++- config/initializers/smtp.rb | 2 + spec/app/models/server_spec.rb | 14 - spec/factories/domain_factory.rb | 11 + spec/factories/ip_pool_rule_factory.rb | 15 + spec/factories/organization_factory.rb | 5 + spec/factories/server_factory.rb | 1 + spec/{app => }/models/organization_spec.rb | 0 .../models/outgoing_message_prototype_spec.rb | 0 spec/models/server_spec.rb | 614 ++++++++++++++++++ spec/{app => }/models/user_spec.rb | 0 spec/{app => }/models/worker_role_spec.rb | 0 spec/rails_helper.rb | 10 + .../incoming_messages_spec.rb | 0 .../outgoing_message_spec.rb | 0 .../services/unqueue_message_service_spec.rb | 0 .../services/webhook_delivery_service_spec.rb | 0 19 files changed, 707 insertions(+), 64 deletions(-) delete mode 100644 spec/app/models/server_spec.rb create mode 100644 spec/factories/ip_pool_rule_factory.rb rename spec/{app => }/models/organization_spec.rb (100%) rename spec/{app => }/models/outgoing_message_prototype_spec.rb (100%) create mode 100644 spec/models/server_spec.rb rename spec/{app => }/models/user_spec.rb (100%) rename spec/{app => }/models/worker_role_spec.rb (100%) rename spec/{app => }/services/unqueue_message_service/incoming_messages_spec.rb (100%) rename spec/{app => }/services/unqueue_message_service/outgoing_message_spec.rb (100%) rename spec/{app => }/services/unqueue_message_service_spec.rb (100%) rename spec/{app => }/services/webhook_delivery_service_spec.rb (100%) diff --git a/Gemfile b/Gemfile index 9772788..8c9fe26 100644 --- a/Gemfile +++ b/Gemfile @@ -52,6 +52,7 @@ group :development do gem "rspec-rails", require: false gem "rubocop" gem "rubocop-rails" + gem "shoulda-matchers" gem "timecop" gem "webmock" end diff --git a/Gemfile.lock b/Gemfile.lock index 2471753..476e924 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -289,6 +289,8 @@ GEM sentry-ruby (~> 5.8.0) sentry-ruby (5.8.0) concurrent-ruby (~> 1.0, >= 1.0.2) + shoulda-matchers (6.1.0) + activesupport (>= 5.2.0) sprockets (4.2.0) concurrent-ruby (~> 1.0) rack (>= 2.2.4, < 4) @@ -366,6 +368,7 @@ DEPENDENCIES secure_headers sentry-rails sentry-ruby + shoulda-matchers timecop turbolinks (~> 5) uglifier (>= 1.3.0) diff --git a/app/models/server.rb b/app/models/server.rb index 942f40c..43cfe35 100644 --- a/app/models/server.rb +++ b/app/models/server.rb @@ -192,15 +192,22 @@ class Server < ApplicationRecord end def send_limit_approaching? - send_limit && (send_volume >= send_limit * 0.90) + return false unless send_limit + + (send_volume >= send_limit * 0.90) end def send_limit_exceeded? - send_limit && send_volume >= send_limit + return false unless send_limit + + send_volume >= send_limit end def send_limit_warning(type) - AppMailer.send("server_send_limit_#{type}", self).deliver + if organization.notification_addresses.present? + AppMailer.send("server_send_limit_#{type}", self).deliver + end + update_column("send_limit_#{type}_notified_at", Time.now) WebhookRequest.trigger(self, "SendLimit#{type.to_s.capitalize}", server: webhook_hash, volume: send_volume, limit: send_limit) end @@ -209,17 +216,6 @@ class Server < ApplicationRecord @queue_size ||= queued_messages.ready.count end - def stats - { - queue: queue_size, - held: held_messages, - bounce_rate: bounce_rate, - message_rate: message_rate, - throughput: throughput_stats, - size: message_db.total_size - } - end - # Return the domain which can be used to authenticate emails sent from the given e-mail address. # # @param address [String] an e-mail address @@ -274,7 +270,10 @@ class Server < ApplicationRecord self.suspended_at = Time.now self.suspension_reason = reason save! - AppMailer.server_suspended(self).deliver + if organization.notification_addresses.present? + AppMailer.server_suspended(self).deliver + end + true end def unsuspend @@ -283,12 +282,6 @@ class Server < ApplicationRecord save! end - def validate_ip_pool_belongs_to_organization - return unless ip_pool && ip_pool_id_changed? && !organization.ip_pools.include?(ip_pool) - - errors.add :ip_pool_id, "must belong to the organization" - end - def ip_pool_for_message(message) return unless message.scope == "outgoing" @@ -300,46 +293,48 @@ class Server < ApplicationRecord end end end + ip_pool end - def self.triggered_send_limit(type) - servers = where("send_limit_#{type}_at IS NOT NULL AND send_limit_#{type}_at > ?", 3.minutes.ago) - servers.where("send_limit_#{type}_notified_at IS NULL OR send_limit_#{type}_notified_at < ?", 1.hour.ago) + private + + def validate_ip_pool_belongs_to_organization + return unless ip_pool && ip_pool_id_changed? && !organization.ip_pools.include?(ip_pool) + + errors.add :ip_pool_id, "must belong to the organization" end - def self.send_send_limit_notifications - [:approaching, :exceeded].each_with_object({}) do |type, hash| - hash[type] = 0 - servers = triggered_send_limit(type) - next if servers.empty? + class << self - servers.each do |server| - hash[type] += 1 - server.send_limit_warning(type) - end - end - end - - def self.[](id, extra = nil) - server = nil - if id.is_a?(String) - if id =~ /\A(\w+)\/(\w+)\z/ - server = includes(:organization).where(organizations: { permalink: ::Regexp.last_match(1) }, permalink: ::Regexp.last_match(2)).first - end - else - server = where(id: id).first + def triggered_send_limit(type) + servers = where("send_limit_#{type}_at IS NOT NULL AND send_limit_#{type}_at > ?", 3.minutes.ago) + servers.where("send_limit_#{type}_notified_at IS NULL OR send_limit_#{type}_notified_at < ?", 1.hour.ago) end - if extra - if extra.is_a?(String) - server.domains.where(name: extra.to_s).first + def send_send_limit_notifications + [:approaching, :exceeded].each_with_object({}) do |type, hash| + hash[type] = 0 + servers = triggered_send_limit(type) + next if servers.empty? + + servers.each do |server| + hash[type] += 1 + server.send_limit_warning(type) + end + end + end + + def [](id, extra = nil) + if id.is_a?(String) && id =~ /\A(\w+)\/(\w+)\z/ + joins(:organization).where( + organizations: { permalink: ::Regexp.last_match(1) }, permalink: ::Regexp.last_match(2) + ).first else - server.message(extra.to_i) + find_by(id: id.to_i) end - else - server end + end end diff --git a/config/initializers/smtp.rb b/config/initializers/smtp.rb index dcc06a1..fc5c7c1 100644 --- a/config/initializers/smtp.rb +++ b/config/initializers/smtp.rb @@ -3,6 +3,8 @@ require "postal/config" if Postal.config&.smtp + # TODO: by default, we should just send mail through the local Postal + # installation rather than having to actually configure an SMTP server. ActionMailer::Base.delivery_method = :smtp ActionMailer::Base.smtp_settings = { address: Postal.config.smtp.host, user_name: Postal.config.smtp.username, password: Postal.config.smtp.password, port: Postal.config.smtp.port || 25 } end diff --git a/spec/app/models/server_spec.rb b/spec/app/models/server_spec.rb deleted file mode 100644 index f871408..0000000 --- a/spec/app/models/server_spec.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -require "rails_helper" - -describe Server do - context "model" do - subject(:server) { create(:server) } - - it "should have a UUID" do - expect(server.uuid).to be_a String - expect(server.uuid.length).to eq 36 - end - end -end diff --git a/spec/factories/domain_factory.rb b/spec/factories/domain_factory.rb index fe9ca66..4aeebe8 100644 --- a/spec/factories/domain_factory.rb +++ b/spec/factories/domain_factory.rb @@ -42,6 +42,17 @@ FactoryBot.define do sequence(:name) { |n| "example#{n}.com" } verification_method { "DNS" } verified_at { Time.now } + + trait :unverified do + verified_at { nil } + end + + trait :dns_all_ok do + spf_status { "OK" } + dkim_status { "OK" } + mx_status { "OK" } + return_path_status { "OK" } + end end factory :organization_domain, parent: :domain do diff --git a/spec/factories/ip_pool_rule_factory.rb b/spec/factories/ip_pool_rule_factory.rb new file mode 100644 index 0000000..0a75af2 --- /dev/null +++ b/spec/factories/ip_pool_rule_factory.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :ip_pool_rule do + owner factory: :organization + ip_pool + to_text { "google.com" } + + after(:build) do |ip_pool_rule| + if ip_pool_rule.ip_pool.organizations.empty? && ip_pool_rule.owner.is_a?(Organization) + ip_pool_rule.ip_pool.organizations << ip_pool_rule.owner + end + end + end +end diff --git a/spec/factories/organization_factory.rb b/spec/factories/organization_factory.rb index 937a33d..e35812e 100644 --- a/spec/factories/organization_factory.rb +++ b/spec/factories/organization_factory.rb @@ -28,5 +28,10 @@ FactoryBot.define do name { "Acme Inc" } sequence(:permalink) { |n| "org#{n}" } association :owner, factory: :user + + trait :suspended do + suspended_at { 1.day.ago } + suspension_reason { "test" } + end end end diff --git a/spec/factories/server_factory.rb b/spec/factories/server_factory.rb index 6b9b60d..ac19936 100644 --- a/spec/factories/server_factory.rb +++ b/spec/factories/server_factory.rb @@ -52,6 +52,7 @@ FactoryBot.define do trait :suspended do suspended_at { Time.current } + suspension_reason { "Test Reason" } end trait :exceeded_send_limit do diff --git a/spec/app/models/organization_spec.rb b/spec/models/organization_spec.rb similarity index 100% rename from spec/app/models/organization_spec.rb rename to spec/models/organization_spec.rb diff --git a/spec/app/models/outgoing_message_prototype_spec.rb b/spec/models/outgoing_message_prototype_spec.rb similarity index 100% rename from spec/app/models/outgoing_message_prototype_spec.rb rename to spec/models/outgoing_message_prototype_spec.rb diff --git a/spec/models/server_spec.rb b/spec/models/server_spec.rb new file mode 100644 index 0000000..e7b78d3 --- /dev/null +++ b/spec/models/server_spec.rb @@ -0,0 +1,614 @@ +# frozen_string_literal: true + +require "rails_helper" + +describe Server do + subject(:server) { build(:server) } + + describe "relationships" do + it { is_expected.to belong_to(:organization) } + it { is_expected.to belong_to(:ip_pool).optional } + it { is_expected.to have_many(:domains) } + it { is_expected.to have_many(:credentials) } + it { is_expected.to have_many(:smtp_endpoints) } + it { is_expected.to have_many(:http_endpoints) } + it { is_expected.to have_many(:address_endpoints) } + it { is_expected.to have_many(:routes) } + it { is_expected.to have_many(:queued_messages) } + it { is_expected.to have_many(:webhooks) } + it { is_expected.to have_many(:webhook_requests) } + it { is_expected.to have_many(:track_domains) } + it { is_expected.to have_many(:ip_pool_rules) } + end + + describe "validations" do + it { is_expected.to validate_presence_of(:name) } + it { is_expected.to validate_uniqueness_of(:name).scoped_to(:organization_id).case_insensitive } + it { is_expected.to validate_inclusion_of(:mode).in_array(Server::MODES) } + it { is_expected.to validate_uniqueness_of(:permalink).scoped_to(:organization_id).case_insensitive } + it { is_expected.to validate_exclusion_of(:permalink).in_array(Server::RESERVED_PERMALINKS) } + it { is_expected.to allow_value("hello").for(:permalink) } + it { is_expected.to allow_value("hello-world").for(:permalink) } + it { is_expected.to allow_value("hello1234").for(:permalink) } + it { is_expected.not_to allow_value("LARGE").for(:permalink) } + it { is_expected.not_to allow_value(" lots of spaces ").for(:permalink) } + it { is_expected.not_to allow_value("hello+").for(:permalink) } + it { is_expected.not_to allow_value("!!!").for(:permalink) } + it { is_expected.not_to allow_value("[hello]").for(:permalink) } + + describe "ip pool validation" do + let(:org) { create(:organization) } + let(:ip_pool) { create(:ip_pool) } + let(:server) { build(:server, organization: org, ip_pool: ip_pool) } + + context "when the IP pool does not belong to the same organization" do + it "adds an error" do + expect(server.save).to be false + expect(server.errors[:ip_pool_id]).to include(/must belong to the organization/) + end + end + + context "whent he IP pool does belong to the the same organization" do + before do + org.ip_pools << ip_pool + end + + it "does not add an error" do + expect(server.save).to be true + end + end + end + end + + describe "creation" do + let(:server) { build(:server) } + + it "generates a uuid" do + expect { server.save }.to change { server.uuid }.from(nil).to(/[a-f0-9-]{36}/) + end + + it "generates a token" do + expect { server.save }.to change { server.token }.from(nil).to(/[a-z0-9]{6}/) + end + + it "provisions a database" do + expect(server.message_db.provisioner).to receive(:provision).once + server.provision_database = true + server.save + end + end + + describe "deletion" do + it "removes the database" do + expect(server.message_db.provisioner).to receive(:drop).once + server.provision_database = true + server.destroy + end + end + + describe "#status" do + context "when the server is suspended" do + let(:server) { build(:server, :suspended) } + + it "returns Suspended" do + expect(server.status).to eq("Suspended") + end + end + + context "when the server is not suspended" do + it "returns the mode" do + expect(server.status).to eq "Live" + end + end + end + + describe "#full_permalink" do + it "returns the org and server permalinks concatenated" do + expect(server.full_permalink).to eq "#{server.organization.permalink}/#{server.permalink}" + end + end + + describe "#suspended?" do + context "when the server is suspended" do + let(:server) { build(:server, :suspended) } + + it "returns true" do + expect(server).to be_suspended + end + end + + context "when the server is not suspended" do + it "returns false" do + expect(server).not_to be_suspended + end + end + end + + describe "#actual_suspension_reason" do + context "when the server is not suspended" do + it "returns nil" do + expect(server.actual_suspension_reason).to be_nil + end + end + + context "when the server is not suspended by the organization is" do + let(:org) { build(:organization, :suspended, suspension_reason: "org test") } + let(:server) { build(:server, organization: org) } + + it "returns the organization suspension reason" do + expect(server.actual_suspension_reason).to eq "org test" + end + end + + context "when the server is suspended" do + let(:server) { build(:server, :suspended, suspension_reason: "server test") } + + it "returns the suspension reason" do + expect(server.actual_suspension_reason).to eq "server test" + end + end + end + + describe "#to_param" do + it "returns the permalink" do + expect(server.to_param).to eq server.permalink + end + end + + describe "#message_db" do + it "returns a message DB instance" do + expect(server.message_db).to be_a Postal::MessageDB::Database + expect(server.message_db).to have_attributes(server_id: server.id, organization_id: server.organization.id) + end + + it "caches the value" do + call1 = server.message_db + call2 = server.message_db + expect(call1.object_id).to eq(call2.object_id) + end + end + + describe "#message" do + it "delegates to the message db" do + expect(server.message_db).to receive(:message).with(1) + server.message(1) + end + end + + describe "#message_rate" do + it "returns the live stats for the last hour per minute" do + allow(server.message_db.live_stats).to receive(:total).and_return(600) + expect(server.message_rate).to eq 10 + expect(server.message_db.live_stats).to have_received(:total).with(60, types: [:incoming, :outgoing]) + end + end + + describe "#held_messages" do + it "returns the number of held messages" do + expect(server.message_db).to receive(:messages).with(count: true, where: { held: true }).and_return(50) + expect(server.held_messages).to eq 50 + end + end + + describe "#throughput_stats" do + before do + allow(server.message_db.live_stats).to receive(:total).with(60, types: [:incoming]).and_return(50) + allow(server.message_db.live_stats).to receive(:total).with(60, types: [:outgoing]).and_return(100) + end + + context "when the server has a sent limit" do + let(:server) { build(:server, send_limit: 500) } + + it "returns the stats with an outgoing usage percentage" do + expect(server.throughput_stats).to eq({ + incoming: 50, + outgoing: 100, + outgoing_usage: 20.0 + }) + end + end + + context "when the server does not have a sent limit" do + it "returns the stats with no outgoing usage percentage" do + expect(server.throughput_stats).to eq({ + incoming: 50, + outgoing: 100, + outgoing_usage: 0 + }) + end + end + end + + describe "#bounce_rate" do + context "when there are no outgoing emails" do + it "returns zero" do + expect(server.bounce_rate).to eq 0 + end + end + + context "when there are outgoing emails with some bounces" do + it "returns the rate" do + allow(server.message_db.statistics).to receive(:get).with(:daily, [:outgoing, :bounces], kind_of(Time), 30) + .and_return({ + 10.minutes.ago => { outgoing: 150, bounces: 50 }, + 5.minutes.ago => { outgoing: 350, bounces: 30 }, + 1.minutes.ago => { outgoing: 500, bounces: 20 } + }) + expect(server.bounce_rate).to eq 10.0 + end + end + end + + describe "#domain_stats" do + it "returns stats about the domains associated with the server" do + create(:domain, owner: server) # verified, bad dns + create(:domain, :unverified, owner: server) # unverified + create(:domain, :dns_all_ok, owner: server) # verified good dns + + expect(server.domain_stats).to eq [3, 1, 1] + end + end + + describe "#webhook_hash" do + it "returns a hash to represent the server" do + expect(server.webhook_hash).to eq({ + uuid: server.uuid, + name: server.name, + permalink: server.permalink, + organization: server.organization.permalink + }) + end + end + + describe "#send_volume" do + it "returns the number of outgoing messages sent in the last hour" do + allow(server.message_db.live_stats).to receive(:total).with(60, types: [:outgoing]).and_return(50) + expect(server.send_volume).to eq 50 + end + end + + describe "#send_limit_approaching?" do + context "when the server has no send limit" do + it "returns false" do + expect(server.send_limit_approaching?).to be false + end + end + + context "when the server has a send limit" do + let(:server) { build(:server, send_limit: 1000) } + + context "when the server's send volume is less 90% of the limit" do + it "return false" do + allow(server).to receive(:send_volume).and_return(800) + expect(server.send_limit_approaching?).to be false + end + end + + context "when the server's send volume is more than 90% of the limit" do + it "returns true" do + allow(server).to receive(:send_volume).and_return(901) + expect(server.send_limit_approaching?).to be true + end + end + end + end + + describe "#send_limit_warning" do + let(:server) { create(:server, send_limit: 1000) } + + before do + allow(server).to receive(:send_volume).and_return(500) + end + + context "when given the :approaching argument" do + it "sends an email to the org notification addresses" do + server.organization.users << create(:user) + + server.send_limit_warning(:approaching) + delivery = ActionMailer::Base.deliveries.last + expect(delivery).to have_attributes(subject: /mail server is approaching its send limit/i) + end + + it "sets the notification time" do + expect { server.send_limit_warning(:approaching) }.to change { server.send_limit_approaching_notified_at } + .from(nil).to(kind_of(Time)) + end + + it "triggers a webhook" do + expect(WebhookRequest).to receive(:trigger).with(server, "SendLimitApproaching", server: server.webhook_hash, volume: 500, limit: 1000) + server.send_limit_warning(:approaching) + end + end + + context "when given the :exceeded argument" do + it "sends an email to the org notification addresses" do + server.organization.users << create(:user) + + server.send_limit_warning(:exceeded) + delivery = ActionMailer::Base.deliveries.last + expect(delivery).to have_attributes(subject: /mail server has exceeded its send limit/i) + end + + it "sets the notification time" do + expect { server.send_limit_warning(:exceeded) }.to change { server.send_limit_exceeded_notified_at } + .from(nil).to(kind_of(Time)) + end + + it "triggers a webhook" do + expect(WebhookRequest).to receive(:trigger).with(server, "SendLimitExceeded", server: server.webhook_hash, volume: 500, limit: 1000) + server.send_limit_warning(:exceeded) + end + end + end + + describe "#queue_size" do + it "returns the number of queued messages that are ready" do + create(:queued_message, server: server, retry_after: nil) + create(:queued_message, server: server, retry_after: 1.minute.ago) + expect(server.queue_size).to eq 2 + end + end + + describe "#authenticated_domain_for_address" do + context "when the address given is blank" do + it "returns nil" do + expect(server.authenticated_domain_for_address("")).to be nil + expect(server.authenticated_domain_for_address(nil)).to be nil + end + end + + context "when the address given does not have a username & domain component" do + it "returns nil" do + expect(server.authenticated_domain_for_address("blah")).to be nil + end + end + + context "when there is a verified org-level domain matching the address provided" do + it "returns that domain" do + server = create(:server) + domain = create(:domain, owner: server.organization, name: "mangos.io") + expect(server.authenticated_domain_for_address("hello@mangos.io")).to eq domain + end + end + + context "when there is a verified server-level domain matching the address provided" do + it "returns that domain" do + domain = create(:domain, owner: server, name: "oranges.io") + expect(server.authenticated_domain_for_address("hello@oranges.io")).to eq domain + end + end + + context "when there is a verified server-level domain matching the address and a use_for_any" do + it "returns the matching domain" do + domain = create(:domain, owner: server, name: "oranges.io") + create(:domain, owner: server, name: "pears.com", use_for_any: true) + expect(server.authenticated_domain_for_address("hello@oranges.io")).to eq domain + end + end + + context "when there is a verified server-level and org-level domain with the same name" do + it "returns the server-level domain" do + domain = create(:domain, owner: server, name: "lemons.com") + create(:domain, owner: server.organization, name: "lemons.com") + expect(server.authenticated_domain_for_address("hello@lemons.com")).to eq domain + end + end + + context "when there is a verified server-level domain with the 'use_for_any' boolean set with a different name" do + it "returns that domain" do + create(:domain, owner: server, name: "pears.com") + domain = create(:domain, owner: server, name: "apples.io", use_for_any: true) + expect(server.authenticated_domain_for_address("hello@bananas.com")).to eq domain + end + end + + context "when there is no suitable domain" do + it "returns nil" do + server = create(:server) + create(:domain, owner: server, name: "pears.com") + create(:domain, owner: server.organization, name: "pineapples.com") + expect(server.authenticated_domain_for_address("hello@bananas.com")).to be nil + end + end + end + + describe "#find_authenticated_domain_from_headers" do + context "when none of the from addresses have a valid domain" do + it "returns nil" do + expect(server.find_authenticated_domain_from_headers("from" => "test@lemons.com")).to be nil + end + end + + context "when the from addresses has a valid domain" do + it "returns the domain" do + domain = create(:domain, owner: server) + expect(server.find_authenticated_domain_from_headers("from" => "hello@#{domain.name}")).to eq domain + end + end + + context "when there are multiple from addresses" do + context "when none of them match a domain" do + it "returns nil" do + expect(server.find_authenticated_domain_from_headers("from" => ["hello@lemons.com", "hello@apples.com"])).to be nil + end + end + + context "when some but not all match" do + it "returns nil" do + domain = create(:domain, owner: server) + expect(server.find_authenticated_domain_from_headers("from" => ["hello@#{domain.name}", "hello@lemons.com"])).to be nil + end + end + + context "when all match" do + it "returns the first domain that matched" do + domain1 = create(:domain, owner: server) + domain2 = create(:domain, owner: server) + expect(server.find_authenticated_domain_from_headers("from" => ["hello@#{domain1.name}", "hello@#{domain2.name}"])).to eq domain1 + end + end + end + + context "when the server is not allowed to use the sender header" do + context "when the sender header has a valid address" do + it "does not return the domain" do + domain = create(:domain, owner: server) + result = server.find_authenticated_domain_from_headers( + "from" => "hello@lemons.com", + "sender" => "hello@#{domain.name}" + ) + expect(result).to be nil + end + end + end + + context "when the server is allowed to use the sender header" do + let(:server) { build(:server, allow_sender: true) } + + context "when none of the from addresses match but sender domains do" do + it "returns the domain that does match" do + domain = create(:domain, owner: server) + result = server.find_authenticated_domain_from_headers( + "from" => "hello@lemons.com", + "sender" => "hello@#{domain.name}" + ) + expect(result).to eq domain + end + end + end + end + + describe "#suspend" do + let(:server) { create(:server) } + + it "sets the suspension time" do + expect { server.suspend("some reason") }.to change { server.reload.suspended_at }.from(nil).to(kind_of(Time)) + end + + it "sets the suspension reason" do + expect { server.suspend("some reason") }.to change { server.reload.suspension_reason }.from(nil).to("some reason") + end + + context "when there are no notification addresses" do + it "does not send an email" do + server.suspend("some reason") + expect(ActionMailer::Base.deliveries).to be_empty + end + end + + context "when there are notification addresses" do + before do + server.organization.users << create(:user) + end + + it "sends an email" do + server.suspend("some reason") + delivery = ActionMailer::Base.deliveries.last + expect(delivery).to have_attributes(subject: /server has been suspended/i) + end + end + end + + describe "#unsuspend" do + let(:server) { create(:server, :suspended) } + + it "removes the suspension time" do + expect { server.unsuspend }.to change { server.reload.suspended_at }.to(nil) + end + + it "removes the suspension reason" do + expect { server.unsuspend }.to change { server.reload.suspension_reason }.to(nil) + end + end + + describe "#ip_pool_for_message" do + context "when the message is not outgoing" do + let(:message) { MessageFactory.incoming(server) } + + it "returns nil" do + expect(server.ip_pool_for_message(message)).to be nil + end + end + + context "when a server rule matches the message" do + let(:domain) { create(:domain, owner: server) } + let(:ip_pool) { create(:ip_pool, organizations: [server.organization]) } + let(:message) do + MessageFactory.outgoing(server, domain: domain) do |msg| + msg.rcpt_to = "hello@google.com" + end + end + + before do + create(:ip_pool_rule, ip_pool: ip_pool, owner: server, from_text: nil, to_text: "google.com") + end + + it "returns the pool" do + expect(server.ip_pool_for_message(message)).to eq ip_pool + end + end + + context "when an org rule matches the message" do + let(:domain) { create(:domain, owner: server) } + let(:ip_pool) { create(:ip_pool, organizations: [server.organization]) } + let(:message) do + MessageFactory.outgoing(server, domain: domain) do |msg| + msg.rcpt_to = "hello@google.com" + end + end + + before do + create(:ip_pool_rule, ip_pool: ip_pool, owner: server.organization, from_text: nil, to_text: "google.com") + end + + it "returns the pool" do + expect(server.ip_pool_for_message(message)).to eq ip_pool + end + end + + context "when the server has no default pool and no rules match the message" do + let(:domain) { create(:domain, owner: server) } + let(:message) { MessageFactory.outgoing(server, domain: domain) } + + it "returns nil" do + expect(server.ip_pool_for_message(message)).to be nil + end + end + + context "when the server has a default pool and no rules match the message" do + let(:organization) { create(:organization) } + let(:ip_pool) { create(:ip_pool, organizations: [organization]) } + let(:server) { create(:server, organization: organization, ip_pool: ip_pool) } + let(:domain) { create(:domain, owner: server) } + let(:message) { MessageFactory.outgoing(server, domain: domain) } + + it "returns the server's default pool" do + expect(server.ip_pool_for_message(message)).to eq ip_pool + end + end + end + + describe ".[]" do + context "when provided with an integer" do + it "returns the server with that ID" do + server = create(:server) + expect(described_class[server.id]).to eq server + end + + it "returns nil if no server exists with the ID" do + expect(described_class[1234]).to be nil + end + end + + context "when provided with a string" do + it "returns the server that matches the given permalinks" do + server = create(:server) + expect(described_class["#{server.organization.permalink}/#{server.permalink}"]).to eq server + end + + it "returns nil if no server exists" do + expect(described_class["hello/world"]).to be nil + end + end + end +end diff --git a/spec/app/models/user_spec.rb b/spec/models/user_spec.rb similarity index 100% rename from spec/app/models/user_spec.rb rename to spec/models/user_spec.rb diff --git a/spec/app/models/worker_role_spec.rb b/spec/models/worker_role_spec.rb similarity index 100% rename from spec/app/models/worker_role_spec.rb rename to spec/models/worker_role_spec.rb diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index b46a817..4a5d93e 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -9,6 +9,7 @@ require "factory_bot" require "timecop" require "database_cleaner" require "webmock/rspec" +require "shoulda-matchers" DatabaseCleaner.allow_remote_database_url = true ActiveRecord::Base.logger = Logger.new("/dev/null") @@ -16,8 +17,17 @@ ActiveRecord::Base.logger = Logger.new("/dev/null") Dir[File.expand_path("factories/*.rb", __dir__)].each { |f| require f } Dir[File.expand_path("helpers/**/*.rb", __dir__)].each { |f| require f } +ActionMailer::Base.delivery_method = :test + ActiveRecord::Migration.maintain_test_schema! +Shoulda::Matchers.configure do |config| + config.integrate do |with| + with.test_framework :rspec + with.library :rails + end +end + RSpec.configure do |config| config.use_transactional_fixtures = true config.infer_spec_type_from_file_location! diff --git a/spec/app/services/unqueue_message_service/incoming_messages_spec.rb b/spec/services/unqueue_message_service/incoming_messages_spec.rb similarity index 100% rename from spec/app/services/unqueue_message_service/incoming_messages_spec.rb rename to spec/services/unqueue_message_service/incoming_messages_spec.rb diff --git a/spec/app/services/unqueue_message_service/outgoing_message_spec.rb b/spec/services/unqueue_message_service/outgoing_message_spec.rb similarity index 100% rename from spec/app/services/unqueue_message_service/outgoing_message_spec.rb rename to spec/services/unqueue_message_service/outgoing_message_spec.rb diff --git a/spec/app/services/unqueue_message_service_spec.rb b/spec/services/unqueue_message_service_spec.rb similarity index 100% rename from spec/app/services/unqueue_message_service_spec.rb rename to spec/services/unqueue_message_service_spec.rb diff --git a/spec/app/services/webhook_delivery_service_spec.rb b/spec/services/webhook_delivery_service_spec.rb similarity index 100% rename from spec/app/services/webhook_delivery_service_spec.rb rename to spec/services/webhook_delivery_service_spec.rb From 3bbbc70bc1ed27232b570af4bdd73d0ba188cd6f Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Tue, 20 Feb 2024 17:08:56 +0000 Subject: [PATCH 12/56] test: move TestLogger to spec/helpers --- {lib => spec/helpers}/test_logger.rb | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {lib => spec/helpers}/test_logger.rb (100%) diff --git a/lib/test_logger.rb b/spec/helpers/test_logger.rb similarity index 100% rename from lib/test_logger.rb rename to spec/helpers/test_logger.rb From 1a4158699c452966308af02d7f2a26dd094be208 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Tue, 20 Feb 2024 21:33:56 +0000 Subject: [PATCH 13/56] refactor: refactor DNS resolution This commit also adds some of tests for the Domain model. It was during the writing of these tests that the DNS resolution refactoring requirement became apparent. --- app/controllers/domains_controller.rb | 2 +- app/models/concerns/has_dns_checks.rb | 13 +- app/models/domain.rb | 55 ++-- app/models/track_domain.rb | 3 +- app/util/dns_resolver.rb | 148 +++++++++++ lib/postal/mx_lookup.rb | 36 --- lib/postal/received_header.rb | 15 +- lib/postal/smtp_sender.rb | 17 +- spec/lib/postal/received_header_spec.rb | 2 +- spec/models/domain_spec.rb | 317 ++++++++++++++++++++++++ 10 files changed, 506 insertions(+), 102 deletions(-) create mode 100644 app/util/dns_resolver.rb delete mode 100644 lib/postal/mx_lookup.rb create mode 100644 spec/models/domain_spec.rb diff --git a/app/controllers/domains_controller.rb b/app/controllers/domains_controller.rb index f1ef258..146df1d 100644 --- a/app/controllers/domains_controller.rb +++ b/app/controllers/domains_controller.rb @@ -71,7 +71,7 @@ class DomainsController < ApplicationController when "Email" if params[:code] if @domain.verification_token == params[:code].to_s.strip - @domain.verify + @domain.mark_as_verified redirect_to_with_json [:setup, organization, @server, @domain], notice: "#{@domain.name} has been verified successfully. You now need to configure your DNS records." else respond_to do |wants| diff --git a/app/models/concerns/has_dns_checks.rb b/app/models/concerns/has_dns_checks.rb index a4d6ec5..30e2814 100644 --- a/app/models/concerns/has_dns_checks.rb +++ b/app/models/concerns/has_dns_checks.rb @@ -43,8 +43,8 @@ module HasDNSChecks # def check_spf_record - result = resolver.getresources(name, Resolv::DNS::Resource::IN::TXT) - spf_records = result.map(&:data).grep(/\Av=spf1/) + result = resolver.txt(name) + spf_records = result.grep(/\Av=spf1/) if spf_records.empty? self.spf_status = "Missing" self.spf_error = "No SPF record exists for this domain" @@ -73,8 +73,7 @@ module HasDNSChecks def check_dkim_record domain = "#{dkim_record_name}.#{name}" - result = resolver.getresources(domain, Resolv::DNS::Resource::IN::TXT) - records = result.map(&:data) + records = resolver.txt(domain) if records.empty? self.dkim_status = "Missing" self.dkim_error = "No TXT records were returned for #{domain}" @@ -104,8 +103,7 @@ module HasDNSChecks # def check_mx_records - result = resolver.getresources(name, Resolv::DNS::Resource::IN::MX) - records = result.map(&:exchange) + records = resolver.mx(name).map(&:last) if records.empty? self.mx_status = "Missing" self.mx_error = "There are no MX records for #{name}" @@ -134,8 +132,7 @@ module HasDNSChecks # def check_return_path_record - result = resolver.getresources(return_path_domain, Resolv::DNS::Resource::IN::CNAME) - records = result.map { |r| r.name.to_s.downcase } + records = resolver.cname(return_path_domain) if records.empty? self.return_path_status = "Missing" self.return_path_error = "There is no return path record at #{return_path_domain}" diff --git a/app/models/domain.rb b/app/models/domain.rb index 4a845f8..6c3b2ff 100644 --- a/app/models/domain.rb +++ b/app/models/domain.rb @@ -77,7 +77,9 @@ class Domain < ApplicationRecord verified_at.present? end - def verify + def mark_as_verified + return false if verified? + self.verified_at = Time.now save! end @@ -94,6 +96,8 @@ class Domain < ApplicationRecord end def dkim_key + return nil unless dkim_private_key + @dkim_key ||= OpenSSL::PKey::RSA.new(dkim_private_key) end @@ -114,28 +118,37 @@ class Domain < ApplicationRecord end def dkim_record + return if dkim_key.nil? + public_key = dkim_key.public_key.to_s.gsub(/-+[A-Z ]+-+\n/, "").gsub(/\n/, "") "v=DKIM1; t=s; h=sha256; p=#{public_key};" end def dkim_identifier + return nil unless dkim_identifier_string + Postal.config.dns.dkim_identifier + "-#{dkim_identifier_string}" end def dkim_record_name - "#{dkim_identifier}._domainkey" + identifier = dkim_identifier + return if identifier.nil? + + "#{identifier}._domainkey" end def return_path_domain "#{Postal.config.dns.custom_return_path_prefix}.#{name}" end - def nameservers - @nameservers ||= get_nameservers - end - + # Returns a DNSResolver instance that can be used to perform DNS lookups needed for + # the verification and DNS checking for this domain. + # + # @return [DNSResolver] def resolver - @resolver ||= Postal.config.general.use_local_ns_for_domains? ? Resolv::DNS.new : Resolv::DNS.new(nameserver: nameservers) + return DNSResolver.local if Postal.config.general.use_local_ns_for_domains? + + @resolver ||= DNSResolver.for_domain(name) end def dns_verification_string @@ -145,32 +158,14 @@ class Domain < ApplicationRecord def verify_with_dns return false unless verification_method == "DNS" - result = resolver.getresources(name, Resolv::DNS::Resource::IN::TXT) - if result.map { |d| d.data.to_s.strip }.include?(dns_verification_string) + result = resolver.txt(name) + + if result.include?(dns_verification_string) self.verified_at = Time.now - save - else - false + return save end - end - private - - def get_nameservers - local_resolver = Resolv::DNS.new - ns_records = [] - parts = name.split(".") - (parts.size - 1).times do |n| - d = parts[n, parts.size - n + 1].join(".") - ns_records = local_resolver.getresources(d, Resolv::DNS::Resource::IN::NS).map { |s| s.name.to_s } - break if ns_records.present? - end - return [] if ns_records.blank? - - ns_records = ns_records.map { |r| local_resolver.getresources(r, Resolv::DNS::Resource::IN::A).map { |s| s.address.to_s } }.flatten - return [] if ns_records.blank? - - ns_records + false end end diff --git a/app/models/track_domain.rb b/app/models/track_domain.rb index 810b6ea..a662beb 100644 --- a/app/models/track_domain.rb +++ b/app/models/track_domain.rb @@ -54,8 +54,7 @@ class TrackDomain < ApplicationRecord end def check_dns - result = domain.resolver.getresources(full_name, Resolv::DNS::Resource::IN::CNAME) - records = result.map { |r| r.name.to_s.downcase } + records = domain.resolver.cname(full_name) if records.empty? self.dns_status = "Missing" self.dns_error = "There is no record at #{full_name}" diff --git a/app/util/dns_resolver.rb b/app/util/dns_resolver.rb new file mode 100644 index 0000000..ffb5543 --- /dev/null +++ b/app/util/dns_resolver.rb @@ -0,0 +1,148 @@ +# frozen_string_literal: true + +class DNSResolver + + attr_reader :nameservers + attr_reader :timeout + + def initialize(nameservers: nil, timeout: 5) + @nameservers = nameservers + @timeout = timeout + end + + # Return all A records for the given name + # + # @param [String] name + # @return [Array] + def a(name) + dns do |dns| + dns.getresources(name, Resolv::DNS::Resource::IN::A).map do |s| + s.address.to_s + end + end + end + + # Return all AAAA records for the given name + # + # @param [String] name + # @return [Array] + def aaaa(name) + dns do |dns| + dns.getresources(name, Resolv::DNS::Resource::IN::AAAA).map do |s| + s.address.to_s + end + end + end + + # Return all TXT records for the given name + # + # @param [String] name + # @return [Array] + def txt(name) + dns do |dns| + dns.getresources(name, Resolv::DNS::Resource::IN::TXT).map do |s| + s.data.to_s.strip + end + end + end + + # Return all CNAME records for the given name + # + # @param [String] name + # @return [Array] + def cname(name) + dns do |dns| + dns.getresources(name, Resolv::DNS::Resource::IN::CNAME).map do |s| + s.name.to_s.downcase + end + end + end + + # Return all MX records for the given name + # + # @param [String] name + # @return [Array>] + def mx(name) + dns do |dns| + records = dns.getresources(name, Resolv::DNS::Resource::IN::MX).map do |m| + [m.preference.to_i, m.exchange.to_s] + end + records.sort do |a, b| + if a[0] == b[0] + [-1, 1].sample + else + a[0] <=> b[0] + end + end + end + end + + # Return the effective nameserver names for a given domain name. + # + # @param [String] name + # @return [Array] + def effective_ns(name) + records = [] + dns do |dns| + parts = name.split(".") + (parts.size - 1).times do |n| + d = parts[n, parts.size - n + 1].join(".") + + records = dns.getresources(d, Resolv::DNS::Resource::IN::NS).map do |s| + s.name.to_s + end + + break if records.present? + end + end + + records + end + + # Return the hostname for a given IP address. + # Returns the IP address itself if no hostname can be determined. + # + # @param [String] ip_address + # @return [String] + def ip_to_hostname(ip_address) + dns do |dns| + dns.getname(ip_address)&.to_s + end + rescue Resolv::ResolvError + ip_address + end + + private + + def dns + Resolv::DNS.open(nameserver: @nameservers || []) do |dns| + dns.timeouts = [@timeout, @timeout / 2] + yield dns + end + end + + class << self + + # Return a resolver which will use the nameservers for the given domain + # + # @param [String] name + # @return [DNSResolver] + def for_domain(name) + resolver = new + nameservers = resolver.effective_ns(name) + ips = nameservers.map do |ns| + resolver.a(ns) + end.flatten.uniq + new(nameservers: ips) + end + + # Return a local resolver to use for lookups + # + # @return [DNSResolver] + def local + @local ||= new + end + + end + +end diff --git a/lib/postal/mx_lookup.rb b/lib/postal/mx_lookup.rb deleted file mode 100644 index 21aa7eb..0000000 --- a/lib/postal/mx_lookup.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -module Postal - class MXLookup - - class << self - - def lookup(domain) - records = resolve(domain) - records = sort(records) - records.map { |m| m[1] } - end - - private - - def sort(records) - records.sort do |a, b| - if a[0] == b[0] - [-1, 1].sample - else - a[0] <=> b[0] - end - end - end - - def resolve(domain) - Resolv::DNS.open do |dns| - dns.timeouts = [10, 5] - dns.getresources(domain, Resolv::DNS::Resource::IN::MX).map { |m| [m.preference.to_i, m.exchange.to_s] } - end - end - - end - - end -end diff --git a/lib/postal/received_header.rb b/lib/postal/received_header.rb index a9bec15..b019d69 100644 --- a/lib/postal/received_header.rb +++ b/lib/postal/received_header.rb @@ -19,26 +19,13 @@ module Postal header = "by #{our_hostname} with #{method.to_s.upcase}; #{Time.now.utc.rfc2822}" if server.nil? || server.privacy_mode == false - hostname = resolve_hostname(ip_address) + hostname = DNSResolver.local.ip_to_hostname(ip_address) header = "from #{helo} (#{hostname} [#{ip_address}]) #{header}" end header end - private - - def resolve_hostname(ip_address) - Resolv::DNS.open do |dns| - dns.timeouts = [10, 5] - begin - dns.getname(ip_address) - rescue StandardError - ip_address - end - end - end - end end diff --git a/lib/postal/smtp_sender.rb b/lib/postal/smtp_sender.rb index 764b057..2857c30 100644 --- a/lib/postal/smtp_sender.rb +++ b/lib/postal/smtp_sender.rb @@ -225,7 +225,7 @@ module Postal def servers @options[:servers] || self.class.relay_hosts || @servers ||= begin - mx_servers = MXLookup.lookup(@domain) + mx_servers = DNSResolver.local.mx(@domain).map(&:last) if mx_servers.empty? mx_servers = [@domain] # This will be resolved to an A or AAAA record later end @@ -243,16 +243,13 @@ module Postal def lookup_ip_address(type, hostname) records = [] - Resolv::DNS.open do |dns| - dns.timeouts = [10, 5] - case type - when :a - records = dns.getresources(hostname, Resolv::DNS::Resource::IN::A) - when :aaaa - records = dns.getresources(hostname, Resolv::DNS::Resource::IN::AAAA) - end + case type + when :a + records = DNSResolver.local.a(hostname) + when :aaaa + records = DNSResolver.local.aaaa(hostname) end - records.first&.address&.to_s&.downcase + records.first&.to_s&.downcase end class << self diff --git a/spec/lib/postal/received_header_spec.rb b/spec/lib/postal/received_header_spec.rb index 541da80..0a9c6bb 100644 --- a/spec/lib/postal/received_header_spec.rb +++ b/spec/lib/postal/received_header_spec.rb @@ -4,7 +4,7 @@ require "rails_helper" describe Postal::ReceivedHeader do before do - allow(Resolv::DNS).to receive(:open).and_return("hostname.com") + allow(DNSResolver.local).to receive(:ip_to_hostname).and_return("hostname.com") end describe ".generate" do diff --git a/spec/models/domain_spec.rb b/spec/models/domain_spec.rb new file mode 100644 index 0000000..7911ef4 --- /dev/null +++ b/spec/models/domain_spec.rb @@ -0,0 +1,317 @@ +# frozen_string_literal: true + +require "rails_helper" + +describe Domain do + subject(:domain) { build(:domain) } + + describe "relationships" do + it { is_expected.to belong_to(:server).optional } + it { is_expected.to belong_to(:owner).optional } + it { is_expected.to have_many(:routes) } + it { is_expected.to have_many(:track_domains) } + end + + describe "validations" do + it { is_expected.to validate_presence_of(:name) } + it { is_expected.to validate_uniqueness_of(:name).scoped_to([:owner_type, :owner_id]).case_insensitive.with_message("is already added") } + it { is_expected.to allow_value("example.com").for(:name) } + it { is_expected.to allow_value("example.co.uk").for(:name) } + it { is_expected.to_not allow_value("EXAMPLE.COM").for(:name) } + it { is_expected.to_not allow_value("example.com ").for(:name) } + it { is_expected.to_not allow_value("example com").for(:name) } + it { is_expected.to validate_inclusion_of(:verification_method).in_array(Domain::VERIFICATION_METHODS) } + end + + describe "creation" do + it "creates a new dkim identifier string" do + expect { domain.save }.to change { domain.dkim_identifier_string }.from(nil).to(match(/\A[a-zA-Z0-9]{6}\z/)) + end + + it "generates a new dkim key" do + expect { domain.save }.to change { domain.dkim_private_key }.from(nil).to(match(/\A-+BEGIN RSA PRIVATE KEY-+/)) + end + + it "generates a UUID" do + expect { domain.save }.to change { domain.uuid }.from(nil).to(/[a-f0-9-]{36}/) + end + end + + describe ".verified" do + it "returns verified domains only" do + verified_domain = create(:domain) + create(:domain, :unverified) + expect(described_class.verified).to eq [verified_domain] + end + end + + context "when verification method changes" do + context "to DNS" do + let(:domain) { create(:domain, :unverified, verification_method: "Email") } + + it "generates a DNS suitable verification token" do + domain.verification_method = "DNS" + expect { domain.save }.to change { domain.verification_token }.from(match(/\A\d{6}\z/)).to(match(/\A[A-Za-z0-9+]{32}\z/)) + end + end + + context "to Email" do + let(:domain) { create(:domain, :unverified, verification_method: "DNS") } + + it "generates an email suitable verification token" do + domain.verification_method = "Email" + expect { domain.save }.to change { domain.verification_token }.from(match(/\A[A-Za-z0-9+]{32}\z/)).to(match(/\A\d{6}\z/)) + end + end + end + + describe "#verified?" do + context "when the domain is verified" do + it "returns true" do + expect(domain.verified?).to be true + end + end + + context "when the domain is not verified" do + let(:domain) { build(:domain, :unverified) } + + it "returns false" do + expect(domain.verified?).to be false + end + end + end + + describe "#mark_as_verified" do + context "when already verified" do + it "returns false" do + expect(domain.mark_as_verified).to be false + end + end + + context "when unverified" do + let(:domain) { create(:domain, :unverified) } + + it "sets the verification time" do + expect { domain.mark_as_verified }.to change { domain.verified_at }.from(nil).to(kind_of(Time)) + end + end + end + + describe "#parent_domains" do + context "at level 1" do + let(:domain) { build(:domain, name: "example.com") } + + it "returns the current domain only" do + expect(domain.parent_domains).to eq ["example.com"] + end + end + + context "at level 2" do + let(:domain) { build(:domain, name: "test.example.com") } + + it "returns the current domain plus its parent" do + expect(domain.parent_domains).to eq ["test.example.com", "example.com"] + end + end + + context "at level 3 (and higher)" do + let(:domain) { build(:domain, name: "sub.test.example.com") } + + it "returns the current domain plus its parents" do + expect(domain.parent_domains).to eq ["sub.test.example.com", "test.example.com", "example.com"] + end + end + end + + describe "#generate_dkim_key" do + it "generates a new dkim key" do + expect { domain.generate_dkim_key }.to change { domain.dkim_private_key }.from(nil).to(match(/\A-+BEGIN RSA PRIVATE KEY-+/)) + end + end + + describe "#dkim_key" do + context "when the domain has a DKIM key" do + let(:domain) { create(:domain) } + + it "returns the dkim key as a OpenSSL::PKey::RSA" do + expect(domain.dkim_key).to be_a OpenSSL::PKey::RSA + expect(domain.dkim_key.to_s).to eq domain.dkim_private_key + end + end + + context "when the domain has no DKIM key" do + let(:domain) { build(:domain) } + + it "returns nil" do + expect(domain.dkim_key).to be_nil + end + end + end + + describe "#to_param" do + context "when the domain has not been saved" do + it "returns nil" do + expect(domain.to_param).to be_nil + end + end + context "when the domain has been saved" do + before do + domain.save + end + + it "returns the UUID" do + expect(domain.to_param).to eq domain.uuid + end + end + end + + describe "#verification_email_addresses" do + let(:domain) { build(:domain, name: "example.com") } + + it "returns the verification email addresses" do + expect(domain.verification_email_addresses).to eq [ + "webmaster@example.com", + "postmaster@example.com", + "admin@example.com", + "administrator@example.com", + "hostmaster@example.com" + ] + end + end + + describe "#spf_record" do + it "returns the SPF record" do + expect(domain.spf_record).to eq "v=spf1 a mx include:#{Postal.config.dns.spf_include} ~all" + end + end + + describe "#dkim_record" do + context "when the domain has no DKIM key" do + it "returns nil" do + expect(domain.dkim_record).to be_nil + end + end + + context "when the domain has a DKIM key" do + before do + domain.save + end + + it "returns the DKIM record" do + expect(domain.dkim_record).to match(/\Av=DKIM1; t=s; h=sha256; p=.*;\z/) + end + end + end + + describe "#dkim_identifier" do + context "when the domain has no dkim identifier string" do + it "returns nil" do + expect(domain.dkim_identifier).to be_nil + end + end + + context "when the domain has a dkim identifier string" do + before do + domain.save + end + + it "returns the DKIM identifier" do + expect(domain.dkim_identifier).to eq "#{Postal.config.dns.dkim_identifier}-#{domain.dkim_identifier_string}" + end + end + end + + describe "#dkim_record_name" do + context "when the domain has no dkim identifier string" do + it "returns nil" do + expect(domain.dkim_record_name).to be_nil + end + end + + context "when the domain has a dkim identifier string" do + before do + domain.save + end + + it "returns the DKIM identifier" do + expect(domain.dkim_record_name).to eq "#{Postal.config.dns.dkim_identifier}-#{domain.dkim_identifier_string}._domainkey" + end + end + end + + describe "#return_path_domain" do + it "returns the return path domain" do + expect(domain.return_path_domain).to eq "#{Postal.config.dns.custom_return_path_prefix}.#{domain.name}" + end + end + + describe "#dns_verification_string" do + let(:domain) { create(:domain, verification_method: "DNS") } + + it "returns the DNS verification string" do + expect(domain.dns_verification_string).to eq "#{Postal.config.dns.domain_verify_prefix} #{domain.verification_token}" + end + end + + describe "#resolver" do + context "when the local nameservers should be used" do + before do + allow(Postal.config.general).to receive(:use_local_ns_for_domains?).and_return(true) + end + + it "uses the local DNS" do + expect(domain.resolver).to eq DNSResolver.local + end + end + + context "when local nameservers should not be used" do + it "uses the a resolver for this domain" do + allow(DNSResolver).to receive(:for_domain).with(domain.name).and_return(DNSResolver.new(nameservers: ["1.2.3.4"])) + expect(domain.resolver).to be_a DNSResolver + expect(domain.resolver.nameservers).to eq ["1.2.3.4"] + end + end + end + + describe "#verify_with_dns" do + context "when the verification method is not DNS" do + let(:domain) { build(:domain, verification_method: "Email") } + + it "returns false" do + expect(domain.verify_with_dns).to be false + end + end + + context "when a TXT record is found that matches" do + let(:domain) { create(:domain, :unverified) } + + before do + allow(domain.resolver).to receive(:txt).with(domain.name).and_return([domain.dns_verification_string]) + end + + it "returns true" do + expect(domain.verify_with_dns).to be true + end + + it "sets the verification time" do + expect { domain.verify_with_dns }.to change { domain.verified_at }.from(nil).to(kind_of(Time)) + end + end + + context "when no TXT record is found" do + let(:domain) { create(:domain, :unverified) } + + before do + allow(domain.resolver).to receive(:txt).with(domain.name).and_return(["something", "something else"]) + end + + it "returns false" do + expect(domain.verify_with_dns).to be false + end + + it "does not set the verification time" do + expect { domain.verify_with_dns }.to_not change { domain.verified_at } # rubocop:disable Lint/AmbiguousBlockAssociation + end + end + end +end From ee8631152534ca52fe4aba5009691bc1a3830b91 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Tue, 20 Feb 2024 21:36:20 +0000 Subject: [PATCH 14/56] test: move Postal::RpecHelpers to spec/helpers --- lib/postal/rspec_helpers.rb | 17 ----------------- spec/helpers/general_helpers.rb | 15 +++++++++++++++ spec/rails_helper.rb | 2 +- 3 files changed, 16 insertions(+), 18 deletions(-) delete mode 100644 lib/postal/rspec_helpers.rb create mode 100644 spec/helpers/general_helpers.rb diff --git a/lib/postal/rspec_helpers.rb b/lib/postal/rspec_helpers.rb deleted file mode 100644 index fd55188..0000000 --- a/lib/postal/rspec_helpers.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -module Postal - module RspecHelpers - - def create_plain_text_message(server, text, to = "test@example.com", override_attributes = {}) - domain = create(:domain, owner: server) - attributes = { from: "test@#{domain.name}", subject: "Test Plain Text Message" }.merge(override_attributes) - attributes[:to] = to - attributes[:plain_body] = text - message = OutgoingMessagePrototype.new(server, "127.0.0.1", "testsuite", attributes) - result = message.create_message(to) - server.message_db.message(result[:id]) - end - - end -end diff --git a/spec/helpers/general_helpers.rb b/spec/helpers/general_helpers.rb new file mode 100644 index 0000000..0b982f5 --- /dev/null +++ b/spec/helpers/general_helpers.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module GeneralHelpers + + def create_plain_text_message(server, text, to = "test@example.com", override_attributes = {}) + domain = create(:domain, owner: server) + attributes = { from: "test@#{domain.name}", subject: "Test Plain Text Message" }.merge(override_attributes) + attributes[:to] = to + attributes[:plain_body] = text + message = OutgoingMessagePrototype.new(server, "127.0.0.1", "testsuite", attributes) + result = message.create_message(to) + server.message_db.message(result[:id]) + end + +end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 4a5d93e..8827741 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -32,7 +32,7 @@ RSpec.configure do |config| config.use_transactional_fixtures = true config.infer_spec_type_from_file_location! config.include FactoryBot::Syntax::Methods - config.include Postal::RspecHelpers + config.include GeneralHelpers config.before(:suite) do # Test that the factories are working as they should and then clean up before getting started on From 8765d8e57a9dce62dc09b46b21c12dd424867925 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Tue, 20 Feb 2024 21:48:57 +0000 Subject: [PATCH 15/56] test: add tests for DNSResolver --- app/util/dns_resolver.rb | 3 +- spec/util/dns_resolver_spec.rb | 85 ++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 spec/util/dns_resolver_spec.rb diff --git a/app/util/dns_resolver.rb b/app/util/dns_resolver.rb index ffb5543..0526584 100644 --- a/app/util/dns_resolver.rb +++ b/app/util/dns_resolver.rb @@ -115,7 +115,8 @@ class DNSResolver private def dns - Resolv::DNS.open(nameserver: @nameservers || []) do |dns| + kwargs = @nameservers ? { nameserver: @nameservers } : {} + Resolv::DNS.open(**kwargs) do |dns| dns.timeouts = [@timeout, @timeout / 2] yield dns end diff --git a/spec/util/dns_resolver_spec.rb b/spec/util/dns_resolver_spec.rb new file mode 100644 index 0000000..d3f0d43 --- /dev/null +++ b/spec/util/dns_resolver_spec.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe DNSResolver do + subject(:resolver) { described_class.new } + + # Now, we could mock everything in here which would give us some comfort + # but I do think that we'll benefit more from having a full E2E test here + # so we'll test this using values which we know to be fairly static and + # that are within our control. + + describe "#a" do + it "returns a list of IP addresses" do + expect(resolver.a("www.test.postalserver.io").sort).to eq ["1.2.3.4", "2.3.4.5"] + end + end + + describe "#aaaa" do + it "returns a list of IP addresses" do + expect(resolver.aaaa("www.test.postalserver.io").sort).to eq ["2a00:67a0:a::1", "2a00:67a0:a::2"] + end + end + + describe "#txt" do + it "returns a list of TXT records" do + expect(resolver.txt("test.postalserver.io").sort).to eq [ + "an example txt record", + "another example" + ] + end + end + + describe "#cname" do + it "returns a list of CNAME records" do + expect(resolver.cname("cname.test.postalserver.io")).to eq ["www.test.postalserver.io"] + end + end + + describe "#mx" do + it "returns a list of MX records" do + expect(resolver.mx("test.postalserver.io")).to eq [ + [10, "mx1.test.postalserver.io"], + [20, "mx2.test.postalserver.io"] + ] + end + end + + describe "#effective_ns" do + it "returns the nameserver names that are authoritative for the given domain" do + expect(resolver.effective_ns("postalserver.io").sort).to eq [ + "prestigious-honeybadger.katapultdns.com", + "the-cake-is-a-lie.katapultdns.com" + ] + end + end + + describe "#ip_to_hostname" do + it "returns the hostname for the given IP" do + expect(resolver.ip_to_hostname("151.252.1.100")).to eq "ns1.katapultdns.com" + end + end + + describe ".for_domain" do + it "finds the effective nameservers for a given domain and returns them" do + resolver = described_class.for_domain("test.postalserver.io") + expect(resolver.nameservers.sort).to eq ["151.252.1.100", "151.252.2.100"] + end + end + + describe ".local" do + it "returns a resolver with no nameservers" do + resolver = described_class.local + expect(resolver.nameservers).to be nil + end + end + + context "when using a resolver for a domain" do + subject(:resolver) { described_class.for_domain("test.postalserver.io") } + + it "will not return domains that are not hosted on that server" do + expect(resolver.a("example.com")).to eq [] + end + end +end From ed6da11b65811e39c2641bb3c9bc2474ca006c0a Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Wed, 21 Feb 2024 21:01:38 +0000 Subject: [PATCH 16/56] chore: add annotations to factories and models specs --- spec/factories/address_endpoint_factory.rb | 12 +++++++ spec/factories/ip_pool_rule_factory.rb | 14 ++++++++ spec/factories/smtp_endpoint_factory.rb | 17 +++++++++ spec/factories/webhook_factory.rb | 20 +++++++++++ spec/factories/worker_role_factory.rb | 13 +++++++ spec/models/domain_spec.rb | 35 ++++++++++++++++++ spec/models/organization_spec.rb | 22 ++++++++++++ spec/models/server_spec.rb | 41 ++++++++++++++++++++++ spec/models/user_spec.rb | 24 +++++++++++++ spec/models/worker_role_spec.rb | 13 +++++++ 10 files changed, 211 insertions(+) diff --git a/spec/factories/address_endpoint_factory.rb b/spec/factories/address_endpoint_factory.rb index f5e74c2..3fea88e 100644 --- a/spec/factories/address_endpoint_factory.rb +++ b/spec/factories/address_endpoint_factory.rb @@ -1,5 +1,17 @@ # frozen_string_literal: true +# == Schema Information +# +# Table name: address_endpoints +# +# id :integer not null, primary key +# address :string(255) +# last_used_at :datetime +# uuid :string(255) +# created_at :datetime not null +# updated_at :datetime not null +# server_id :integer +# FactoryBot.define do factory :address_endpoint do server diff --git a/spec/factories/ip_pool_rule_factory.rb b/spec/factories/ip_pool_rule_factory.rb index 0a75af2..8275557 100644 --- a/spec/factories/ip_pool_rule_factory.rb +++ b/spec/factories/ip_pool_rule_factory.rb @@ -1,5 +1,19 @@ # frozen_string_literal: true +# == Schema Information +# +# Table name: ip_pool_rules +# +# id :integer not null, primary key +# from_text :text(65535) +# owner_type :string(255) +# to_text :text(65535) +# uuid :string(255) +# created_at :datetime not null +# updated_at :datetime not null +# ip_pool_id :integer +# owner_id :integer +# FactoryBot.define do factory :ip_pool_rule do owner factory: :organization diff --git a/spec/factories/smtp_endpoint_factory.rb b/spec/factories/smtp_endpoint_factory.rb index 013f2db..b8a4b76 100644 --- a/spec/factories/smtp_endpoint_factory.rb +++ b/spec/factories/smtp_endpoint_factory.rb @@ -1,5 +1,22 @@ # frozen_string_literal: true +# == Schema Information +# +# Table name: smtp_endpoints +# +# id :integer not null, primary key +# disabled_until :datetime +# error :text(65535) +# hostname :string(255) +# last_used_at :datetime +# name :string(255) +# port :integer +# ssl_mode :string(255) +# uuid :string(255) +# created_at :datetime +# updated_at :datetime +# server_id :integer +# FactoryBot.define do factory :smtp_endpoint do server diff --git a/spec/factories/webhook_factory.rb b/spec/factories/webhook_factory.rb index 8c5f594..10be33d 100644 --- a/spec/factories/webhook_factory.rb +++ b/spec/factories/webhook_factory.rb @@ -1,5 +1,25 @@ # frozen_string_literal: true +# == Schema Information +# +# Table name: webhooks +# +# id :integer not null, primary key +# all_events :boolean default(FALSE) +# enabled :boolean default(TRUE) +# last_used_at :datetime +# name :string(255) +# sign :boolean default(TRUE) +# url :string(255) +# uuid :string(255) +# created_at :datetime +# updated_at :datetime +# server_id :integer +# +# Indexes +# +# index_webhooks_on_server_id (server_id) +# FactoryBot.define do factory :webhook do server diff --git a/spec/factories/worker_role_factory.rb b/spec/factories/worker_role_factory.rb index be5f497..db1379d 100644 --- a/spec/factories/worker_role_factory.rb +++ b/spec/factories/worker_role_factory.rb @@ -1,5 +1,18 @@ # frozen_string_literal: true +# == Schema Information +# +# Table name: worker_roles +# +# id :bigint not null, primary key +# acquired_at :datetime +# role :string(255) +# worker :string(255) +# +# Indexes +# +# index_worker_roles_on_role (role) UNIQUE +# FactoryBot.define do factory :worker_role do role { "test" } diff --git a/spec/models/domain_spec.rb b/spec/models/domain_spec.rb index 7911ef4..d2e451f 100644 --- a/spec/models/domain_spec.rb +++ b/spec/models/domain_spec.rb @@ -1,5 +1,40 @@ # frozen_string_literal: true +# == Schema Information +# +# Table name: domains +# +# id :integer not null, primary key +# dkim_error :string(255) +# dkim_identifier_string :string(255) +# dkim_private_key :text(65535) +# dkim_status :string(255) +# dns_checked_at :datetime +# incoming :boolean default(TRUE) +# mx_error :string(255) +# mx_status :string(255) +# name :string(255) +# outgoing :boolean default(TRUE) +# owner_type :string(255) +# return_path_error :string(255) +# return_path_status :string(255) +# spf_error :string(255) +# spf_status :string(255) +# use_for_any :boolean +# uuid :string(255) +# verification_method :string(255) +# verification_token :string(255) +# verified_at :datetime +# created_at :datetime +# updated_at :datetime +# owner_id :integer +# server_id :integer +# +# Indexes +# +# index_domains_on_server_id (server_id) +# index_domains_on_uuid (uuid) +# require "rails_helper" describe Domain do diff --git a/spec/models/organization_spec.rb b/spec/models/organization_spec.rb index 2340af4..96163a6 100644 --- a/spec/models/organization_spec.rb +++ b/spec/models/organization_spec.rb @@ -1,5 +1,27 @@ # frozen_string_literal: true +# == Schema Information +# +# Table name: organizations +# +# id :integer not null, primary key +# deleted_at :datetime +# name :string(255) +# permalink :string(255) +# suspended_at :datetime +# suspension_reason :string(255) +# time_zone :string(255) +# uuid :string(255) +# created_at :datetime +# updated_at :datetime +# ip_pool_id :integer +# owner_id :integer +# +# Indexes +# +# index_organizations_on_permalink (permalink) +# index_organizations_on_uuid (uuid) +# require "rails_helper" describe Organization do diff --git a/spec/models/server_spec.rb b/spec/models/server_spec.rb index e7b78d3..5712b7b 100644 --- a/spec/models/server_spec.rb +++ b/spec/models/server_spec.rb @@ -1,5 +1,46 @@ # frozen_string_literal: true +# == Schema Information +# +# Table name: servers +# +# id :integer not null, primary key +# allow_sender :boolean default(FALSE) +# deleted_at :datetime +# domains_not_to_click_track :text(65535) +# log_smtp_data :boolean default(FALSE) +# message_retention_days :integer +# mode :string(255) +# name :string(255) +# outbound_spam_threshold :decimal(8, 2) +# permalink :string(255) +# postmaster_address :string(255) +# privacy_mode :boolean default(FALSE) +# raw_message_retention_days :integer +# raw_message_retention_size :integer +# send_limit :integer +# send_limit_approaching_at :datetime +# send_limit_approaching_notified_at :datetime +# send_limit_exceeded_at :datetime +# send_limit_exceeded_notified_at :datetime +# spam_failure_threshold :decimal(8, 2) +# spam_threshold :decimal(8, 2) +# suspended_at :datetime +# suspension_reason :string(255) +# token :string(255) +# uuid :string(255) +# created_at :datetime +# updated_at :datetime +# ip_pool_id :integer +# organization_id :integer +# +# Indexes +# +# index_servers_on_organization_id (organization_id) +# index_servers_on_permalink (permalink) +# index_servers_on_token (token) +# index_servers_on_uuid (uuid) +# require "rails_helper" describe Server do diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index ca9366b..023ef71 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -1,5 +1,29 @@ # frozen_string_literal: true +# == Schema Information +# +# Table name: users +# +# id :integer not null, primary key +# admin :boolean default(FALSE) +# email_address :string(255) +# email_verification_token :string(255) +# email_verified_at :datetime +# first_name :string(255) +# last_name :string(255) +# password_digest :string(255) +# password_reset_token :string(255) +# password_reset_token_valid_until :datetime +# time_zone :string(255) +# uuid :string(255) +# created_at :datetime +# updated_at :datetime +# +# Indexes +# +# index_users_on_email_address (email_address) +# index_users_on_uuid (uuid) +# require "rails_helper" describe User do diff --git a/spec/models/worker_role_spec.rb b/spec/models/worker_role_spec.rb index 4776960..123152f 100644 --- a/spec/models/worker_role_spec.rb +++ b/spec/models/worker_role_spec.rb @@ -1,5 +1,18 @@ # frozen_string_literal: true +# == Schema Information +# +# Table name: worker_roles +# +# id :bigint not null, primary key +# acquired_at :datetime +# role :string(255) +# worker :string(255) +# +# Indexes +# +# index_worker_roles_on_role (role) UNIQUE +# require "rails_helper" RSpec.describe WorkerRole do From 77faf886b39ca0c3fc8d923bd129c26a0d9f19c3 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Wed, 21 Feb 2024 21:05:36 +0000 Subject: [PATCH 17/56] refactor: move Postal::DKIMHeader to app/util/dkim_header --- app/util/dkim_header.rb | 130 +++++++++++++++++ lib/postal/dkim_header.rb | 132 ------------------ lib/postal/message_db/message.rb | 2 +- spec/{lib/postal => util}/dkim_header_spec.rb | 2 +- 4 files changed, 132 insertions(+), 134 deletions(-) create mode 100644 app/util/dkim_header.rb delete mode 100644 lib/postal/dkim_header.rb rename spec/{lib/postal => util}/dkim_header_spec.rb (97%) diff --git a/app/util/dkim_header.rb b/app/util/dkim_header.rb new file mode 100644 index 0000000..6ed1c5c --- /dev/null +++ b/app/util/dkim_header.rb @@ -0,0 +1,130 @@ +# frozen_string_literal: true + +class DKIMHeader + + def initialize(domain, message) + if domain && domain.dkim_status == "OK" + @domain_name = domain.name + @dkim_key = domain.dkim_key + @dkim_identifier = domain.dkim_identifier + else + @domain_name = Postal.config.dns.return_path + @dkim_key = Postal.signing_key + @dkim_identifier = Postal.config.dns.dkim_identifier + end + @domain = domain + @message = message + @raw_headers, @raw_body = @message.gsub(/\r?\n/, "\r\n").split(/\r\n\r\n/, 2) + end + + def dkim_header + "DKIM-Signature: v=1; " + dkim_properties.join("\r\n\t") + signature.scan(/.{1,72}/).join("\r\n\t") + end + + private + + def headers + @headers ||= @raw_headers.to_s.gsub(/\r?\n\s/, " ").split(/\r?\n/) + end + + def header_names + normalized_headers.map { |h| h.split(":")[0].strip } + end + + def normalized_headers + [].tap do |new_headers| + dkim_headers = headers.select do |h| + h.match(/ + ^( + from|sender|reply-to|subject|date|message-id|to|cc|mime-version|content-type|content-transfer-encoding| + resent-to|resent-cc|resent-from|resent-sender|resent-message-id|in-reply-to|references|list-id|list-help| + list-owner|list-unsubscribe|list-subscribe|list-post + ):/ix) + end + dkim_headers.each do |h| + new_headers << normalize_header(h) + end + end + end + + def normalize_header(content) + content = content.dup + + # From the DKIM RFC6376 + # https://datatracker.ietf.org/doc/html/rfc6376#section-3.4.2 + + # Split the key and value. + key, value = content.split(":", 2) + + # Convert all header field names (not the header field values) to + # lowercase. For example, convert "SUBJect: AbC" to "subject: AbC". + key.downcase! + + # Unfold all header field continuation lines as described in [RFC5322] + value.gsub!(/\r?\n[ \t]+/, " ") + + # Convert all sequences of one or more WSP characters to a single SP character. + value.gsub!(/[ \t]+/, " ") + + # Delete all WSP characters at the end of each unfolded header field value. + value.gsub!(/[ \t]*\z/, "") + + # Delete any WSP characters remaining after the colon separating the header field name from the header field value. + value.gsub!(/\A[ \t]*/, "") + + # Join together + key + ":" + value + end + + def normalized_body + @normalized_body ||= begin + content = @raw_body.dup + + # From the DKIM RFC6376 + # https://datatracker.ietf.org/doc/html/rfc6376#section-3.4.4 + + # a. Reduce whitespace + # + # * Reduce all sequences of WSP within a line to a single SP character. + content.gsub!(/[ \t]+/, " ") + + # * Ignore all whitespace at the end of lines. Implementations MUST NOT + # remove the CRLF at the end of the line. + content.gsub!(/ \r\n/, "\r\n") + + # b. Ignore all empty lines at the end of the message body. + content.gsub!(/[ \r\n]*\z/, "") + + content += "\r\n" + content + end + end + + def body_hash + @body_hash ||= Base64.encode64(Digest::SHA256.digest(normalized_body)).strip + end + + def dkim_properties + @dkim_properties ||= [].tap do |header| + header << "a=rsa-sha256; c=relaxed/relaxed;" + header << "d=#{@domain_name};" + header << "s=#{@dkim_identifier}; t=#{Time.now.utc.to_i};" + header << "bh=#{body_hash};" + header << "h=#{header_names.join(':')};" + header << "b=" + end + end + + def dkim_header_for_signing + "dkim-signature:v=1; #{dkim_properties.join(' ')}" + end + + def signable_header_string + (normalized_headers + [dkim_header_for_signing]).join("\r\n") + end + + def signature + Base64.encode64(@dkim_key.sign(OpenSSL::Digest.new("SHA256"), signable_header_string)).gsub("\n", "") + end + +end diff --git a/lib/postal/dkim_header.rb b/lib/postal/dkim_header.rb deleted file mode 100644 index b2ba856..0000000 --- a/lib/postal/dkim_header.rb +++ /dev/null @@ -1,132 +0,0 @@ -# frozen_string_literal: true - -module Postal - class DKIMHeader - - def initialize(domain, message) - if domain && domain.dkim_status == "OK" - @domain_name = domain.name - @dkim_key = domain.dkim_key - @dkim_identifier = domain.dkim_identifier - else - @domain_name = Postal.config.dns.return_path - @dkim_key = Postal.signing_key - @dkim_identifier = Postal.config.dns.dkim_identifier - end - @domain = domain - @message = message - @raw_headers, @raw_body = @message.gsub(/\r?\n/, "\r\n").split(/\r\n\r\n/, 2) - end - - def dkim_header - "DKIM-Signature: v=1; " + dkim_properties.join("\r\n\t") + signature.scan(/.{1,72}/).join("\r\n\t") - end - - private - - def headers - @headers ||= @raw_headers.to_s.gsub(/\r?\n\s/, " ").split(/\r?\n/) - end - - def header_names - normalized_headers.map { |h| h.split(":")[0].strip } - end - - def normalized_headers - [].tap do |new_headers| - dkim_headers = headers.select do |h| - h.match(/ - ^( - from|sender|reply-to|subject|date|message-id|to|cc|mime-version|content-type|content-transfer-encoding| - resent-to|resent-cc|resent-from|resent-sender|resent-message-id|in-reply-to|references|list-id|list-help| - list-owner|list-unsubscribe|list-subscribe|list-post - ):/ix) - end - dkim_headers.each do |h| - new_headers << normalize_header(h) - end - end - end - - def normalize_header(content) - content = content.dup - - # From the DKIM RFC6376 - # https://datatracker.ietf.org/doc/html/rfc6376#section-3.4.2 - - # Split the key and value. - key, value = content.split(":", 2) - - # Convert all header field names (not the header field values) to - # lowercase. For example, convert "SUBJect: AbC" to "subject: AbC". - key.downcase! - - # Unfold all header field continuation lines as described in [RFC5322] - value.gsub!(/\r?\n[ \t]+/, " ") - - # Convert all sequences of one or more WSP characters to a single SP character. - value.gsub!(/[ \t]+/, " ") - - # Delete all WSP characters at the end of each unfolded header field value. - value.gsub!(/[ \t]*\z/, "") - - # Delete any WSP characters remaining after the colon separating the header field name from the header field value. - value.gsub!(/\A[ \t]*/, "") - - # Join together - key + ":" + value - end - - def normalized_body - @normalized_body ||= begin - content = @raw_body.dup - - # From the DKIM RFC6376 - # https://datatracker.ietf.org/doc/html/rfc6376#section-3.4.4 - - # a. Reduce whitespace - # - # * Reduce all sequences of WSP within a line to a single SP character. - content.gsub!(/[ \t]+/, " ") - - # * Ignore all whitespace at the end of lines. Implementations MUST NOT - # remove the CRLF at the end of the line. - content.gsub!(/ \r\n/, "\r\n") - - # b. Ignore all empty lines at the end of the message body. - content.gsub!(/[ \r\n]*\z/, "") - - content += "\r\n" - content - end - end - - def body_hash - @body_hash ||= Base64.encode64(Digest::SHA256.digest(normalized_body)).strip - end - - def dkim_properties - @dkim_properties ||= [].tap do |header| - header << "a=rsa-sha256; c=relaxed/relaxed;" - header << "d=#{@domain_name};" - header << "s=#{@dkim_identifier}; t=#{Time.now.utc.to_i};" - header << "bh=#{body_hash};" - header << "h=#{header_names.join(':')};" - header << "b=" - end - end - - def dkim_header_for_signing - "dkim-signature:v=1; #{dkim_properties.join(' ')}" - end - - def signable_header_string - (normalized_headers + [dkim_header_for_signing]).join("\r\n") - end - - def signature - Base64.encode64(@dkim_key.sign(OpenSSL::Digest.new("SHA256"), signable_header_string)).gsub("\n", "") - end - - end -end diff --git a/lib/postal/message_db/message.rb b/lib/postal/message_db/message.rb index 3e88eb2..4257955 100644 --- a/lib/postal/message_db/message.rb +++ b/lib/postal/message_db/message.rb @@ -413,7 +413,7 @@ module Postal def add_outgoing_headers headers = [] if domain - dkim = Postal::DKIMHeader.new(domain, raw_message) + dkim = DKIMHeader.new(domain, raw_message) headers << dkim.dkim_header end headers << "X-Postal-MsgID: #{token}" diff --git a/spec/lib/postal/dkim_header_spec.rb b/spec/util/dkim_header_spec.rb similarity index 97% rename from spec/lib/postal/dkim_header_spec.rb rename to spec/util/dkim_header_spec.rb index 0342e15..ae16584 100644 --- a/spec/lib/postal/dkim_header_spec.rb +++ b/spec/util/dkim_header_spec.rb @@ -2,7 +2,7 @@ require "rails_helper" -describe Postal::DKIMHeader do +describe DKIMHeader do examples = Rails.root.join("spec/examples/dkim_signing/*.msg") Dir[examples].each do |path| contents = File.read(path) From 05d2ec4d042c350a2c8580b1c3ee308ccaf34293 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Wed, 21 Feb 2024 21:06:51 +0000 Subject: [PATCH 18/56] refactor: move Postal::BounceMessage to app/models/bounce_message --- app/models/bounce_message.rb | 55 ++++++++++++++++++ app/models/queued_message.rb | 2 +- lib/postal/bounce_message.rb | 57 ------------------- .../incoming_messages_spec.rb | 12 ++-- 4 files changed, 62 insertions(+), 64 deletions(-) create mode 100644 app/models/bounce_message.rb delete mode 100644 lib/postal/bounce_message.rb diff --git a/app/models/bounce_message.rb b/app/models/bounce_message.rb new file mode 100644 index 0000000..b6b9f5d --- /dev/null +++ b/app/models/bounce_message.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +class BounceMessage + + def initialize(server, message) + @server = server + @message = message + end + + def raw_message + mail = Mail.new + mail.to = @message.mail_from + mail.from = "Mail Delivery Service <#{@message.route.description}>" + mail.subject = "Mail Delivery Failed (#{@message.subject})" + mail.text_part = body + mail.attachments["Original Message.eml"] = { mime_type: "message/rfc822", encoding: "quoted-printable", content: @message.raw_message } + mail.message_id = "<#{SecureRandom.uuid}@#{Postal.config.dns.return_path}>" + mail.to_s + end + + def queue + message = @server.message_db.new_message + message.scope = "outgoing" + message.rcpt_to = @message.mail_from + message.mail_from = @message.route.description + message.domain_id = @message.domain&.id + message.raw_message = raw_message + message.bounce = true + message.bounce_for_id = @message.id + message.save + message.id + end + + def postmaster_address + @server.postmaster_address || "postmaster@#{@message.domain&.name || Postal.config.web.host}" + end + + private + + def body + <<~BODY + This is the mail delivery service responsible for delivering mail to #{@message.route.description}. + + The message you've sent cannot be delivered. Your original message is attached to this message. + + For further assistance please contact #{postmaster_address}. Please include the details below to help us identify the issue. + + Message Token: #{@message.token}@#{@server.token} + Orginal Message ID: #{@message.message_id} + Mail from: #{@message.mail_from} + Rcpt To: #{@message.rcpt_to} + BODY + end + +end diff --git a/app/models/queued_message.rb b/app/models/queued_message.rb index bbe7468..69988de 100644 --- a/app/models/queued_message.rb +++ b/app/models/queued_message.rb @@ -46,7 +46,7 @@ class QueuedMessage < ApplicationRecord def send_bounce return unless message.send_bounces? - Postal::BounceMessage.new(server, message).queue + BounceMessage.new(server, message).queue end def allocate_ip_address diff --git a/lib/postal/bounce_message.rb b/lib/postal/bounce_message.rb deleted file mode 100644 index e5a98d0..0000000 --- a/lib/postal/bounce_message.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -module Postal - class BounceMessage - - def initialize(server, message) - @server = server - @message = message - end - - def raw_message - mail = Mail.new - mail.to = @message.mail_from - mail.from = "Mail Delivery Service <#{@message.route.description}>" - mail.subject = "Mail Delivery Failed (#{@message.subject})" - mail.text_part = body - mail.attachments["Original Message.eml"] = { mime_type: "message/rfc822", encoding: "quoted-printable", content: @message.raw_message } - mail.message_id = "<#{SecureRandom.uuid}@#{Postal.config.dns.return_path}>" - mail.to_s - end - - def queue - message = @server.message_db.new_message - message.scope = "outgoing" - message.rcpt_to = @message.mail_from - message.mail_from = @message.route.description - message.domain_id = @message.domain&.id - message.raw_message = raw_message - message.bounce = true - message.bounce_for_id = @message.id - message.save - message.id - end - - def postmaster_address - @server.postmaster_address || "postmaster@#{@message.domain&.name || Postal.config.web.host}" - end - - private - - def body - <<~BODY - This is the mail delivery service responsible for delivering mail to #{@message.route.description}. - - The message you've sent cannot be delivered. Your original message is attached to this message. - - For further assistance please contact #{postmaster_address}. Please include the details below to help us identify the issue. - - Message Token: #{@message.token}@#{@server.token} - Orginal Message ID: #{@message.message_id} - Mail from: #{@message.mail_from} - Rcpt To: #{@message.rcpt_to} - BODY - end - - end -end diff --git a/spec/services/unqueue_message_service/incoming_messages_spec.rb b/spec/services/unqueue_message_service/incoming_messages_spec.rb index 3925d41..e9fe122 100644 --- a/spec/services/unqueue_message_service/incoming_messages_spec.rb +++ b/spec/services/unqueue_message_service/incoming_messages_spec.rb @@ -62,7 +62,7 @@ RSpec.describe UnqueueMessageService do end it "sends a bounce to the sender" do - expect(Postal::BounceMessage).to receive(:new).with(server, queued_message.message) + expect(BounceMessage).to receive(:new).with(server, queued_message.message) service.call end @@ -449,7 +449,7 @@ RSpec.describe UnqueueMessageService do end it "sends a bounce" do - expect(Postal::BounceMessage).to receive(:new).with(server, queued_message.message) + expect(BounceMessage).to receive(:new).with(server, queued_message.message) service.call end @@ -479,7 +479,7 @@ RSpec.describe UnqueueMessageService do end it "sends a bounce" do - expect(Postal::BounceMessage).to receive(:new).with(server, queued_message.message) + expect(BounceMessage).to receive(:new).with(server, queued_message.message) service.call end @@ -598,9 +598,9 @@ RSpec.describe UnqueueMessageService do end it "does not send a bounce" do - allow(Postal::BounceMessage).to receive(:new) + allow(BounceMessage).to receive(:new) service.call - expect(Postal::BounceMessage).to_not have_received(:new) + expect(BounceMessage).to_not have_received(:new) end end @@ -616,7 +616,7 @@ RSpec.describe UnqueueMessageService do end it "sends a bounce" do - expect(Postal::BounceMessage).to receive(:new).with(server, queued_message.message) + expect(BounceMessage).to receive(:new).with(server, queued_message.message) service.call end From 870b26c2f2f80a04fe905a7cc8e859c8eee50e70 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Wed, 21 Feb 2024 21:10:11 +0000 Subject: [PATCH 19/56] refactor: move Postal::QueryString to app/util/query_string --- app/controllers/messages_controller.rb | 2 +- app/util/query_string.rb | 36 +++++++++++++++++++++ lib/postal/query_string.rb | 38 ---------------------- spec/lib/postal/query_string_spec.rb | 44 -------------------------- spec/util/query_string_spec.rb | 44 ++++++++++++++++++++++++++ 5 files changed, 81 insertions(+), 83 deletions(-) create mode 100644 app/util/query_string.rb delete mode 100644 lib/postal/query_string.rb delete mode 100644 spec/lib/postal/query_string_spec.rb create mode 100644 spec/util/query_string_spec.rb diff --git a/app/controllers/messages_controller.rb b/app/controllers/messages_controller.rb index c028cdb..2e6a25d 100644 --- a/app/controllers/messages_controller.rb +++ b/app/controllers/messages_controller.rb @@ -161,7 +161,7 @@ class MessagesController < ApplicationController if @query = (params[:query] || session["msg_query_#{@server.id}_#{scope}"]).presence session["msg_query_#{@server.id}_#{scope}"] = @query - qs = Postal::QueryString.new(@query) + qs = QueryString.new(@query) if qs.empty? flash.now[:alert] = "It doesn't appear you entered anything to filter on. Please double check your query." else diff --git a/app/util/query_string.rb b/app/util/query_string.rb new file mode 100644 index 0000000..b2c9de3 --- /dev/null +++ b/app/util/query_string.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +class QueryString + + def initialize(string) + @string = string.strip + " " + end + + def [](value) + hash[value.to_s] + end + + delegate :empty?, to: :hash + + def hash + @hash ||= @string.scan(/([a-z]+):\s*(?:(\d{2,4}-\d{2}-\d{2}\s\d{2}:\d{2})|"(.*?)"|(.*?))(\s|\z)/).each_with_object({}) do |(key, date, string_with_spaces, value), hash| + if date + actual_value = date + elsif string_with_spaces + actual_value = string_with_spaces + elsif value == "[blank]" + actual_value = nil + else + actual_value = value + end + + if hash.keys.include?(key.to_s) + hash[key.to_s] = [hash[key.to_s]].flatten + hash[key.to_s] << actual_value + else + hash[key.to_s] = actual_value + end + end + end + +end diff --git a/lib/postal/query_string.rb b/lib/postal/query_string.rb deleted file mode 100644 index 3ca28c0..0000000 --- a/lib/postal/query_string.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Postal - class QueryString - - def initialize(string) - @string = string.strip + " " - end - - def [](value) - hash[value.to_s] - end - - delegate :empty?, to: :hash - - def hash - @hash ||= @string.scan(/([a-z]+):\s*(?:(\d{2,4}-\d{2}-\d{2}\s\d{2}:\d{2})|"(.*?)"|(.*?))(\s|\z)/).each_with_object({}) do |(key, date, string_with_spaces, value), hash| - if date - actual_value = date - elsif string_with_spaces - actual_value = string_with_spaces - elsif value == "[blank]" - actual_value = nil - else - actual_value = value - end - - if hash.keys.include?(key.to_s) - hash[key.to_s] = [hash[key.to_s]].flatten - hash[key.to_s] << actual_value - else - hash[key.to_s] = actual_value - end - end - end - - end -end diff --git a/spec/lib/postal/query_string_spec.rb b/spec/lib/postal/query_string_spec.rb deleted file mode 100644 index 6be6283..0000000 --- a/spec/lib/postal/query_string_spec.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -require "rails_helper" - -describe Postal::QueryString do - it "should work with a single item" do - qs = Postal::QueryString.new("to: test@example.com") - expect(qs.hash["to"]).to eq "test@example.com" - end - - it "should work with a multiple items" do - qs = Postal::QueryString.new("to: test@example.com from: another@example.com") - expect(qs.hash["to"]).to eq "test@example.com" - expect(qs.hash["from"]).to eq "another@example.com" - end - - it "should not require a space after the field name" do - qs = Postal::QueryString.new("to:test@example.com from:another@example.com") - expect(qs.hash["to"]).to eq "test@example.com" - expect(qs.hash["from"]).to eq "another@example.com" - end - - it "should return nil when it receives blank" do - qs = Postal::QueryString.new("to:[blank]") - expect(qs.hash["to"]).to eq nil - end - - it "should handle dates with spaces" do - qs = Postal::QueryString.new("date: 2017-02-12 15:20") - expect(qs.hash["date"]).to eq("2017-02-12 15:20") - end - - it "should return an array for multiple items" do - qs = Postal::QueryString.new("to: test@example.com to: another@example.com") - expect(qs.hash["to"]).to be_a(Array) - expect(qs.hash["to"][0]).to eq "test@example.com" - expect(qs.hash["to"][1]).to eq "another@example.com" - end - - it "should work with a z in the string" do - qs = Postal::QueryString.new("to: testaz@example.com") - expect(qs.hash["to"]).to eq "testaz@example.com" - end -end diff --git a/spec/util/query_string_spec.rb b/spec/util/query_string_spec.rb new file mode 100644 index 0000000..2233b04 --- /dev/null +++ b/spec/util/query_string_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require "rails_helper" + +describe QueryString do + it "works with a single item" do + qs = described_class.new("to: test@example.com") + expect(qs.hash["to"]).to eq "test@example.com" + end + + it "works with a multiple items" do + qs = described_class.new("to: test@example.com from: another@example.com") + expect(qs.hash["to"]).to eq "test@example.com" + expect(qs.hash["from"]).to eq "another@example.com" + end + + it "does not require a space after the field name" do + qs = described_class.new("to:test@example.com from:another@example.com") + expect(qs.hash["to"]).to eq "test@example.com" + expect(qs.hash["from"]).to eq "another@example.com" + end + + it "returns nil when it receives blank" do + qs = described_class.new("to:[blank]") + expect(qs.hash["to"]).to eq nil + end + + it "handles dates with spaces" do + qs = described_class.new("date: 2017-02-12 15:20") + expect(qs.hash["date"]).to eq("2017-02-12 15:20") + end + + it "returns an array for multiple items" do + qs = described_class.new("to: test@example.com to: another@example.com") + expect(qs.hash["to"]).to be_a(Array) + expect(qs.hash["to"][0]).to eq "test@example.com" + expect(qs.hash["to"][1]).to eq "another@example.com" + end + + it "works with a z in the string" do + qs = described_class.new("to: testaz@example.com") + expect(qs.hash["to"]).to eq "testaz@example.com" + end +end From 07eb15246f524f495b473cf97ddd51f661f8b5bf Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Tue, 20 Feb 2024 21:48:57 +0000 Subject: [PATCH 20/56] docs: update docs for how IP address allocation works closes #2209 --- app/views/ip_addresses/_form.html.haml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app/views/ip_addresses/_form.html.haml b/app/views/ip_addresses/_form.html.haml index ea8053f..e6f534e 100644 --- a/app/views/ip_addresses/_form.html.haml +++ b/app/views/ip_addresses/_form.html.haml @@ -17,16 +17,15 @@ %p.fieldSet__text This priority will determine the likelihood of this IP address being selected for use when sending a message. The higher the number the more likely the IP - is to be chosen. By defalt, the priority is set to the maximum value of 100. + is to be chosen. By default, the priority is set to the maximum value of 100. This can be used to warm up new IP addresses by adding them with a low priority. To give an indication of how this works, if you have three IPs with 1, 50 and 100 as their priorities, and you send 100,000 emails, the priority 1 address will receive - a tiny percentage, the priority 50 will receive roughly 25% and the priority 100 will - receive roughly 75%. + a tiny percentage, the priority 50 will receive roughly one third of e-mails and the + priority 100 will receive roughly two thirds. .fieldSetSubmit.buttonSet = f.submit :class => 'button button--positive js-form-submit' .fieldSetSubmit__delete - if @ip_address.persisted? = link_to "Delete IP address", [@ip_pool, @ip_address], :class => 'button button--danger', :method => :delete, :remote => true, :data => {:confirm => "Are you sure you wish to remove this IP from the pool?"} - From a44e1f9081b51ae516278b08d93d0c8c67071c1f Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Thu, 22 Feb 2024 22:26:27 +0000 Subject: [PATCH 21/56] refactor: refactors message dequeueing (#2810) --- app/lib/message_dequeuer/base.rb | 108 +++ .../incoming_message_processor.rb | 215 +++++ app/lib/message_dequeuer/initial_processor.rb | 62 ++ .../outgoing_message_processor.rb | 190 +++++ .../single_message_processor.rb | 83 ++ app/lib/message_dequeuer/state.rb | 26 + app/services/unqueue_message_service.rb | 493 ------------ app/util/message_dequeuer.rb | 14 + .../jobs/process_queued_messages_job.rb | 2 +- spec/lib/message_dequeuer/base_spec.rb | 38 + .../incoming_message_processor_spec.rb | 640 +++++++++++++++ .../initial_message_processor_spec.rb | 94 +++ .../outgoing_message_processor_spec.rb} | 302 +++---- .../single_message_processor_spec.rb | 134 ++++ spec/lib/message_dequeuer/state_spec.rb | 42 + ...rb => process_queued_messages_job_spec.rb} | 31 +- ...b => process_webhook_requests_job_spec.rb} | 11 +- .../incoming_messages_spec.rb | 743 ------------------ spec/services/unqueue_message_service_spec.rb | 99 --- spec/util/message_dequeuer_spec.rb | 18 + 20 files changed, 1808 insertions(+), 1537 deletions(-) create mode 100644 app/lib/message_dequeuer/base.rb create mode 100644 app/lib/message_dequeuer/incoming_message_processor.rb create mode 100644 app/lib/message_dequeuer/initial_processor.rb create mode 100644 app/lib/message_dequeuer/outgoing_message_processor.rb create mode 100644 app/lib/message_dequeuer/single_message_processor.rb create mode 100644 app/lib/message_dequeuer/state.rb delete mode 100644 app/services/unqueue_message_service.rb create mode 100644 app/util/message_dequeuer.rb create mode 100644 spec/lib/message_dequeuer/base_spec.rb create mode 100644 spec/lib/message_dequeuer/incoming_message_processor_spec.rb create mode 100644 spec/lib/message_dequeuer/initial_message_processor_spec.rb rename spec/{services/unqueue_message_service/outgoing_message_spec.rb => lib/message_dequeuer/outgoing_message_processor_spec.rb} (71%) create mode 100644 spec/lib/message_dequeuer/single_message_processor_spec.rb create mode 100644 spec/lib/message_dequeuer/state_spec.rb rename spec/lib/worker/jobs/{process_queued_messages_job.rb => process_queued_messages_job_spec.rb} (73%) rename spec/lib/worker/jobs/{process_webhook_requests_job.rb => process_webhook_requests_job_spec.rb} (72%) delete mode 100644 spec/services/unqueue_message_service/incoming_messages_spec.rb delete mode 100644 spec/services/unqueue_message_service_spec.rb create mode 100644 spec/util/message_dequeuer_spec.rb diff --git a/app/lib/message_dequeuer/base.rb b/app/lib/message_dequeuer/base.rb new file mode 100644 index 0000000..c2e84bc --- /dev/null +++ b/app/lib/message_dequeuer/base.rb @@ -0,0 +1,108 @@ +# frozen_string_literal: true + +module MessageDequeuer + class Base + + class StopProcessing < StandardError + end + + attr_reader :queued_message + attr_reader :logger + attr_reader :state + + def initialize(queued_message, logger:, state: nil) + @queued_message = queued_message + @logger = logger + @state = state || State.new + end + + def process + raise NotImplemented + end + + class << self + + def process(message, **kwargs) + new(message, **kwargs).process + end + + end + + private + + def stop_processing + raise StopProcessing + end + + def catch_stops + yield if block_given? + true + rescue StopProcessing + false + end + + def remove_from_queue + @queued_message.destroy + end + + def create_delivery(type, **kwargs) + @queued_message.message.create_delivery(type, **kwargs) + end + + def log(text, **tags) + logger.info text, **tags + end + + def increment_live_stats + queued_message.message.database.live_stats.increment(queued_message.message.scope) + end + + def hold_if_server_development_mode + return if queued_message.manual? + return unless queued_message.server.mode == "Development" + + log "server is in development mode, holding" + create_delivery "Held", details: "Server is in development mode." + remove_from_queue + stop_processing + end + + def log_sender_result + log_details = @result.details + + if @additional_delivery_details + log_details += "." unless log_details =~ /\.\z/ + log_details += " " + log_details += @additional_delivery_details + end + + create_delivery @result.type, details: log_details, + output: @result.output&.strip, + sent_with_ssl: @result.secure, + log_id: @result.log_id, + time: @result.time + end + + def handle_exception(exception) + log "internal error: #{exception.class}: #{exception.message}" + exception.backtrace.each { |line| log(line) } + + queued_message.retry_later unless queued_message.destroyed? + log "message requeued for trying later, at #{queued_message.retry_after}" + + if defined?(Sentry) + Sentry.capture_exception(exception, extra: { + server_id: queued_message.server_id, + queued_message_id: queued_message.message_id + }) + end + + queued_message.message&.create_delivery("Error", + details: "An internal error occurred while sending " \ + "this message. This message will be retried " \ + "automatically.", + output: "#{exception.class}: #{exception.message}") + end + + end +end diff --git a/app/lib/message_dequeuer/incoming_message_processor.rb b/app/lib/message_dequeuer/incoming_message_processor.rb new file mode 100644 index 0000000..d42a1dc --- /dev/null +++ b/app/lib/message_dequeuer/incoming_message_processor.rb @@ -0,0 +1,215 @@ +# frozen_string_literal: true + +module MessageDequeuer + class IncomingMessageProcessor < Base + + attr_reader :route + + def process + log "message is incoming" + + catch_stops do + handle_bounces + increment_live_stats + inspect_message + fail_if_spam + hold_if_server_development_mode + find_route + hold_or_reject_spam + accept_mail_without_endpoints + hold_messages + bounce_messages + send_message_to_sender + send_bounce_on_hard_fail + log_sender_result + finish_processing + end + rescue StandardError => e + handle_exception(e) + end + + private + + def handle_bounces + return unless queued_message.message.bounce + + log "message is a bounce" + original_messages = queued_message.message.original_messages + unless original_messages.empty? + queued_message.message.original_messages.each do |orig_msg| + queued_message.message.update(bounce_for_id: orig_msg.id, domain_id: orig_msg.domain_id) + create_delivery "Processed", details: "This has been detected as a bounce message for ." + orig_msg.bounce!(queued_message.message) + log "bounce linked with message #{orig_msg.id}" + end + remove_from_queue + stop_processing + end + + # This message was sent to the return path but hasn't been matched + # to an original message. If we have a route for this, route it + # otherwise we'll drop at this point. + return unless queued_message.message.route_id.nil? + + log "no source messages found, hard failing" + create_delivery "HardFail", details: "This message was a bounce but we couldn't link it with any outgoing message and there was no route for it." + remove_from_queue + stop_processing + end + + def inspect_message + return if queued_message.message.inspected + + log "inspecting message" + queued_message.message.inspect_message + return unless queued_message.message.inspected + + is_spam = queued_message.message.spam_score > queued_message.server.spam_threshold + if is_spam + queued_message.message.update(spam: true) + log "message is spam (scored #{queued_message.message.spam_score}, threshold is #{queued_message.server.spam_threshold})" + end + + queued_message.message.append_headers( + "X-Postal-Spam: #{queued_message.message.spam ? 'yes' : 'no'}", + "X-Postal-Spam-Threshold: #{queued_message.server.spam_threshold}", + "X-Postal-Spam-Score: #{queued_message.message.spam_score}", + "X-Postal-Threat: #{queued_message.message.threat ? 'yes' : 'no'}" + ) + log "message inspected, headers added", spam: queued_message.message.spam?, spam_score: queued_message.message.spam_score, threat: queued_message.message.threat? + end + + def fail_if_spam + return if queued_message.message.spam_score < queued_message.server.spam_failure_threshold + + log "message has a spam score higher than the server's maxmimum, hard failing", server_threshold: queued_message.server.spam_failure_threshold + create_delivery "HardFail", + details: "Message's spam score is higher than the failure threshold for this server. " \ + "Threshold is currently #{queued_message.server.spam_failure_threshold}." + remove_from_queue + stop_processing + end + + def find_route + @route = queued_message.message.route + return if @route + + log "no route and/or endpoint available for processing, hard failing" + create_delivery "HardFail", details: "Message does not have a route and/or endpoint available for delivery." + remove_from_queue + stop_processing + end + + def hold_or_reject_spam + return unless queued_message.message.spam + return if queued_message.manual? + + case @route.spam_mode + when "Quarantine" + log "message is spam and route says to quarantine spam message, holding" + create_delivery "Held", details: "Message placed into quarantine." + when "Fail" + log "message is spam and route says to fail spam message, hard failing" + create_delivery "HardFail", details: "Message is spam and the route specifies it should be failed." + else + return + end + + remove_from_queue + stop_processing + end + + def accept_mail_without_endpoints + return unless @route.mode == "Accept" + + log "route says to accept without endpoint, marking as processed" + create_delivery "Processed", details: "Message has been accepted but not sent to any endpoints." + remove_from_queue + stop_processing + end + + def hold_messages + return unless @route.mode == "Hold" + + if queued_message.manual? + log "route says to hold and message was queued manually, marking as processed" + create_delivery "Processed", details: "Message has been processed." + else + log "route says to hold, marking as held" + create_delivery "Held", details: "Message has been accepted but not sent to any endpoints." + end + + remove_from_queue + stop_processing + end + + def bounce_messages + return unless route.mode == "Bounce" || route.mode == "Reject" + + log "route says to bounce, hard failing and sending bounce" + + if id = queued_message.send_bounce + log "bounce sent with id #{id}" + create_delivery "HardFail", details: "Message has been bounced because the route asks for this. See message " + end + + remove_from_queue + stop_processing + end + + def send_message_to_sender + @result = @state.send_result + return if @result + + case queued_message.message.endpoint + when SMTPEndpoint + sender = @state.sender_for(Postal::SMTPSender, queued_message.message.recipient_domain, nil, servers: [queued_message.message.endpoint]) + when HTTPEndpoint + sender = @state.sender_for(Postal::HTTPSender, queued_message.message.endpoint) + when AddressEndpoint + sender = @state.sender_for(Postal::SMTPSender, queued_message.message.endpoint.domain, nil, force_rcpt_to: queued_message.message.endpoint.address) + else + log "invalid endpoint for route (#{queued_message.message.endpoint_type})" + create_delivery "HardFail", details: "Invalid endpoint for route." + remove_from_queue + stop_processing + end + + @result = sender.send_message(queued_message.message) + return unless @result.connect_error + + @state.send_result = @result + end + + def send_bounce_on_hard_fail + return unless @result.type == "HardFail" + + if @result.suppress_bounce + log "suppressing bounce message after hard fail" + return + end + + return unless queued_message.message.send_bounces? + + log "sending a bounce because message hard failed" + return unless bounce_id = queued_message.send_bounce + + @additional_delivery_details = "Sent bounce message to sender (see message )" + end + + def finish_processing + if @result.retry + queued_message.retry_later(@result.retry.is_a?(Integer) ? @result.retry : nil) + log "message requeued for trying later, at #{queued_message.retry_after}" + queued_message.allocate_ip_address + queued_message.update_column(:ip_address_id, queued_message.ip_address&.id) + stop_processing + end + + log "message processing completed" + queued_message.message.endpoint.mark_as_used + remove_from_queue + end + + end +end diff --git a/app/lib/message_dequeuer/initial_processor.rb b/app/lib/message_dequeuer/initial_processor.rb new file mode 100644 index 0000000..46ebda1 --- /dev/null +++ b/app/lib/message_dequeuer/initial_processor.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module MessageDequeuer + class InitialProcessor < Base + + attr_accessor :send_result + + def process + logger.tagged(original_queued_message: @queued_message.id) do + logger.info "starting message unqueue" + begin + catch_stops do + check_message_exists + check_message_is_ready + find_other_messages_for_batch + + # Process the original message and then all of those + # found for batching. + process_message(@queued_message) + @other_messages.each { |message| process_message(message) } + end + ensure + @state.finished + end + logger.info "finished message unqueue" + end + end + + private + + def check_message_exists + @queued_message.message + rescue Postal::MessageDB::Message::NotFound + log "unqueue because backend message has been removed." + remove_from_queue + stop_processing + end + + def check_message_is_ready + return if @queued_message.ready? + + log "skipping because message isn't ready for processing" + @queued_message.unlock + stop_processing + end + + def find_other_messages_for_batch + @other_messages = @queued_message.batchable_messages(100) + log "found #{@other_messages.size} associated messages to process at the same time", batch_key: @queued_message.batch_key + rescue StandardError + @queued_message.unlock + raise + end + + def process_message(queued_message) + logger.tagged(queued_message: queued_message.id) do + SingleMessageProcessor.process(queued_message, logger: @logger, state: @state) + end + end + + end +end diff --git a/app/lib/message_dequeuer/outgoing_message_processor.rb b/app/lib/message_dequeuer/outgoing_message_processor.rb new file mode 100644 index 0000000..af27c8f --- /dev/null +++ b/app/lib/message_dequeuer/outgoing_message_processor.rb @@ -0,0 +1,190 @@ +# frozen_string_literal: true + +module MessageDequeuer + class OutgoingMessageProcessor < Base + + def process + catch_stops do + check_domain + check_rcpt_to + add_tag + hold_if_credential_is_set_to_hold + hold_if_recipient_on_suppression_list + parse_content + inspect_message + fail_if_spam + add_outgoing_headers + check_send_limits + increment_live_stats + hold_if_server_development_mode + send_message_to_sender + add_recipient_to_suppression_list_on_too_many_hard_fails + remove_recipient_from_suppression_list_on_success + log_sender_result + finish_processing + end + rescue StandardError => e + handle_exception(e) + end + + private + + def check_domain + return if queued_message.message.domain + + log "message has no domain, hard failing" + create_delivery "HardFail", details: "Message's domain no longer exist" + remove_from_queue + stop_processing + end + + def check_rcpt_to + return unless queued_message.message.rcpt_to.blank? + + log "message has no 'to' address, hard failing" + create_delivery "HardFail", details: "Message doesn't have an RCPT to" + remove_from_queue + stop_processing + end + + def add_tag + return if queued_message.message.tag + return unless tag = queued_message.message.headers["x-postal-tag"] + + log "added tag: #{tag.last}" + queued_message.message.update(tag: tag.last) + end + + def hold_if_credential_is_set_to_hold + return if queued_message.manual? + return if queued_message.message.credential.nil? + return unless queued_message.message.credential.hold? + + log "credential wants us to hold messages, holding" + create_delivery "Held", details: "Credential is configured to hold all messages authenticated by it." + remove_from_queue + stop_processing + end + + def hold_if_recipient_on_suppression_list + return if queued_message.manual? + return unless sl = queued_message.server.message_db.suppression_list.get(:recipient, queued_message.message.rcpt_to) + + log "recipient is on the suppression list, holding" + create_delivery "Held", details: "Recipient (#{queued_message.message.rcpt_to}) is on the suppression list (reason: #{sl['reason']})" + remove_from_queue + stop_processing + end + + def parse_content + return unless queued_message.message.should_parse? + + log "parsing message content as it hasn't been parsed before" + queued_message.message.parse_content + end + + def inspect_message + return if queued_message.message.inspected + return unless queued_message.server.outbound_spam_threshold + + log "inspecting message" + queued_message.message.inspect_message + return unless queued_message.message.inspected + + if queued_message.message.spam_score >= queued_message.server.outbound_spam_threshold + queued_message.message.update(spam: true) + end + + log "message inspected successfully", spam: queued_message.message.spam?, spam_score: queued_message.message.spam_score + end + + def fail_if_spam + return unless queued_message.message.spam + + log "message is spam (#{queued_message.message.spam_score}), hard failing", server_threshold: queued_message.server.outbound_spam_threshold + create_delivery "HardFail", + details: "Message is likely spam. Threshold is #{queued_message.server.outbound_spam_threshold} and " \ + "the message scored #{queued_message.message.spam_score}." + remove_from_queue + stop_processing + end + + def add_outgoing_headers + return if queued_message.message.has_outgoing_headers? + + queued_message.message.add_outgoing_headers + end + + def check_send_limits + if queued_message.server.send_limit_exceeded? + # If we're over the limit, we're going to be holding this message + log "server send limit has been exceeded, holding", send_limit: queued_message.server.send_limit + queued_message.server.update_columns(send_limit_exceeded_at: Time.now, send_limit_approaching_at: nil) + create_delivery "Held", details: "Message held because send limit (#{queued_message.server.send_limit}) has been reached." + remove_from_queue + stop_processing + elsif queued_message.server.send_limit_approaching? + # If we're approaching the limit, just say we are but continue to process the message + queued_message.server.update_columns(send_limit_approaching_at: Time.now, send_limit_exceeded_at: nil) + else + queued_message.server.update_columns(send_limit_approaching_at: nil, send_limit_exceeded_at: nil) + end + end + + def send_message_to_sender + @result = @state.send_result + return if @result + + sender = @state.sender_for(Postal::SMTPSender, + queued_message.message.recipient_domain, + queued_message.ip_address) + + @result = sender.send_message(queued_message.message) + return unless @result.connect_error + + @state.send_result = @result + end + + def add_recipient_to_suppression_list_on_too_many_hard_fails + return unless @result.type == "HardFail" + + recent_hard_fails = queued_message.server.message_db.select(:messages, + where: { + rcpt_to: queued_message.message.rcpt_to, + status: "HardFail", + timestamp: { greater_than: 24.hours.ago.to_f } + }, + count: true) + return if recent_hard_fails < 1 + + added = queued_message.server.message_db.suppression_list.add(:recipient, queued_message.message.rcpt_to, + reason: "too many hard fails") + return unless added + + log "Added #{queued_message.message.rcpt_to} to suppression list because #{recent_hard_fails} hard fails in 24 hours" + @additional_delivery_details = "Recipient added to suppression list (too many hard fails)" + end + + def remove_recipient_from_suppression_list_on_success + return unless @result.type == "Sent" + + removed = queued_message.server.message_db.suppression_list.remove(:recipient, queued_message.message.rcpt_to) + return unless removed + + log "removed #{queued_message.message.rcpt_to} from suppression list" + @additional_delivery_details = "Recipient removed from suppression list" + end + + def finish_processing + if @result.retry + queued_message.retry_later(@result.retry.is_a?(Integer) ? @result.retry : nil) + log "message requeued for trying later", retry_after: queued_message.retry_after + stop_processing + end + + log "message processing complete" + remove_from_queue + end + + end +end diff --git a/app/lib/message_dequeuer/single_message_processor.rb b/app/lib/message_dequeuer/single_message_processor.rb new file mode 100644 index 0000000..2ac0277 --- /dev/null +++ b/app/lib/message_dequeuer/single_message_processor.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module MessageDequeuer + class SingleMessageProcessor < Base + + def process + catch_stops do + check_message_exists + check_server_suspension + check_delivery_attempts + check_raw_message_exists + + processor = nil + case queued_message.message.scope + when "incoming" + processor = IncomingMessageProcessor + when "outgoing" + processor = OutgoingMessageProcessor + else + create_delivery "HardFail", details: "Scope #{queued_message.message.scope} is not valid" + remove_from_queue + stop_processing + end + + processor.process(queued_message, logger: @logger, state: @state) + end + rescue StandardError => e + handle_exception(e) + end + + private + + def check_message_exists + queued_message.message + rescue Postal::MessageDB::Message::NotFound + log "unqueueing because backend message has been removed" + remove_from_queue + stop_processing + end + + def check_server_suspension + return unless queued_message.server.suspended? + + log "server is suspended, holding message" + create_delivery "Held", details: "Mail server has been suspended. No e-mails can be processed at present. Contact support for assistance." + remove_from_queue + stop_processing + end + + def check_delivery_attempts + return if queued_message.attempts < Postal.config.general.maximum_delivery_attempts + + details = "Maximum number of delivery attempts (#{queued_message.attempts}) has been reached." + if queued_message.message.scope == "incoming" + # Send bounces to incoming e-mails when they are hard failed + if bounce_id = queued_message.send_bounce + details += " Bounce sent to sender (see message )" + end + elsif queued_message.message.scope == "outgoing" + # Add the recipient to the suppression list + if queued_message.server.message_db.suppression_list.add(:recipient, queued_message.message.rcpt_to, reason: "too many soft fails") + log "added #{queued_message.message.rcpt_to} to suppression list because maximum attempts has been reached" + details += " Added #{queued_message.message.rcpt_to} to suppression list because delivery has failed #{queued_message.attempts} times." + end + end + + log "message has reached maximum number of attempts, hard failing" + create_delivery "HardFail", details: details + remove_from_queue + stop_processing + end + + def check_raw_message_exists + return if queued_message.message.raw_message? + + log "raw message has been removed, not sending" + create_delivery "HardFail", details: "Raw message has been removed. Cannot send message." + remove_from_queue + stop_processing + end + + end +end diff --git a/app/lib/message_dequeuer/state.rb b/app/lib/message_dequeuer/state.rb new file mode 100644 index 0000000..949a9d6 --- /dev/null +++ b/app/lib/message_dequeuer/state.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module MessageDequeuer + class State + + attr_accessor :send_result + + def sender_for(klass, *args) + @cached_senders ||= {} + @cached_senders[[klass, args]] ||= begin + klass_instance = klass.new(*args) + klass_instance.start + klass_instance + end + end + + def finished + @cached_senders&.each_value do |sender| + sender.finish + rescue StandardError + false + end + end + + end +end diff --git a/app/services/unqueue_message_service.rb b/app/services/unqueue_message_service.rb deleted file mode 100644 index e0d8886..0000000 --- a/app/services/unqueue_message_service.rb +++ /dev/null @@ -1,493 +0,0 @@ -# frozen_string_literal: true - -class UnqueueMessageService - - def initialize(queued_message:, logger:) - @queued_message = queued_message - @logger = logger - end - - def call - @logger.tagged(original_queued_message: @queued_message.id) do - log "starting message unqueue" - process_original_message - log "finished message unqueue" - end - end - - private - - def process_original_message - begin - @queued_message.message - rescue Postal::MessageDB::Message::NotFound - log "unqueue because backend message has been removed." - @queued_message.destroy - return - end - - unless @queued_message.ready? - log "skipping because message isn't ready for processing" - return - end - - begin - other_messages = @queued_message.batchable_messages(100) - log "found #{other_messages.size} associated messages to process at the same time", batch_key: @queued_message.batch_key - rescue StandardError - @queued_message.unlock - raise - end - - ([@queued_message] + other_messages).each do |queued_message| - @logger.tagged(queued_message: queued_message.id) do - process_message(queued_message) - end - end - ensure - begin - @sender&.finish - rescue StandardError - nil - end - end - - # rubocop:disable Naming/MemoizedInstanceVariableName - def cached_sender(klass, *args) - @sender ||= begin - sender = klass.new(*args) - sender.start - sender - end - end - # rubocop:enable Naming/MemoizedInstanceVariableName - - def log(message, **tags) - @logger.info(message, **tags) - end - - def process_message(queued_message) - begin - queued_message.message - rescue Postal::MessageDB::Message::NotFound - log "unqueueing because backend message has been removed" - queued_message.destroy - return - end - - log "processing message" - - # - # If the server is suspended, hold all messages - # - if queued_message.server.suspended? - log "server is suspended, holding message" - queued_message.message.create_delivery("Held", details: "Mail server has been suspended. No e-mails can be processed at present. Contact support for assistance.") - queued_message.destroy - return - end - - # We might not be able to send this any more, check the attempts - if queued_message.attempts >= Postal.config.general.maximum_delivery_attempts - details = "Maximum number of delivery attempts (#{queued_message.attempts}) has been reached." - if queued_message.message.scope == "incoming" - # Send bounces to incoming e-mails when they are hard failed - if bounce_id = queued_message.send_bounce - details += " Bounce sent to sender (see message )" - end - elsif queued_message.message.scope == "outgoing" - # Add the recipient to the suppression list - if queued_message.server.message_db.suppression_list.add(:recipient, queued_message.message.rcpt_to, reason: "too many soft fails") - log "added #{queued_message.message.rcpt_to} to suppression list because maximum attempts has been reached" - details += " Added #{queued_message.message.rcpt_to} to suppression list because delivery has failed #{queued_message.attempts} times." - end - end - queued_message.message.create_delivery("HardFail", details: details) - queued_message.destroy - log "message has reached maximum number of attempts, hard failing" - return - end - - # If the raw message has been removed (removed by retention) - unless queued_message.message.raw_message? - log "raw message has been removed, not sending" - queued_message.message.create_delivery("HardFail", details: "Raw message has been removed. Cannot send message.") - queued_message.destroy - return - end - - # - # Handle Incoming Messages - # - if queued_message.message.scope == "incoming" - log "message is incoming" - - # - # If this is a bounce, we need to handle it as such - # - if queued_message.message.bounce - log "message is a bounce" - original_messages = queued_message.message.original_messages - unless original_messages.empty? - queued_message.message.original_messages.each do |orig_msg| - queued_message.message.update(bounce_for_id: orig_msg.id, domain_id: orig_msg.domain_id) - queued_message.message.create_delivery("Processed", details: "This has been detected as a bounce message for .") - orig_msg.bounce!(queued_message.message) - log "bounce linked with message #{orig_msg.id}" - end - queued_message.destroy - return - end - - # This message was sent to the return path but hasn't been matched - # to an original message. If we have a route for this, route it - # otherwise we'll drop at this point. - if queued_message.message.route_id.nil? - log "no source messages found, hard failing" - queued_message.message.create_delivery("HardFail", details: "This message was a bounce but we couldn't link it with any outgoing message and there was no route for it.") - queued_message.destroy - return - end - end - - # - # Update live stats - # - queued_message.message.database.live_stats.increment(queued_message.message.scope) - - # - # Inspect incoming messages - # - unless queued_message.message.inspected - log "inspecting message" - queued_message.message.inspect_message - if queued_message.message.inspected - is_spam = queued_message.message.spam_score > queued_message.server.spam_threshold - if is_spam - queued_message.message.update(spam: true) - log "message is spam (scored #{queued_message.message.spam_score}, threshold is #{queued_message.server.spam_threshold})" - end - queued_message.message.append_headers( - "X-Postal-Spam: #{queued_message.message.spam ? 'yes' : 'no'}", - "X-Postal-Spam-Threshold: #{queued_message.server.spam_threshold}", - "X-Postal-Spam-Score: #{queued_message.message.spam_score}", - "X-Postal-Threat: #{queued_message.message.threat ? 'yes' : 'no'}" - ) - log "message inspected, headers added", spam: queued_message.message.spam?, spam_score: queued_message.message.spam_score, threat: queued_message.message.threat? - end - end - - # - # If this message has a SPAM score higher than is permitted - # - if queued_message.message.spam_score >= queued_message.server.spam_failure_threshold - log "message has a spam score higher than the server's maxmimum, hard failing", server_threshold: queued_message.server.spam_failure_threshold - queued_message.message.create_delivery("HardFail", - details: "Message's spam score is higher than the failure threshold for this server. " \ - "Threshold is currently #{queued_message.server.spam_failure_threshold}.") - queued_message.destroy - return - end - - # If the server is in development mode, hold it - if queued_message.server.mode == "Development" && !queued_message.manual? - log "server is in development mode, holding" - queued_message.message.create_delivery("Held", details: "Server is in development mode.") - queued_message.destroy - return - end - - # - # Find out what sort of message we're supposed to be sending and dispatch this request over to - # the sender. - # - if route = queued_message.message.route - - # If the route says we're holding quananteed mail and this is spam, we'll hold this - if route.spam_mode == "Quarantine" && queued_message.message.spam && !queued_message.manual? - log "message is spam and route says to quarantine spam message, holding" - queued_message.message.create_delivery("Held", details: "Message placed into quarantine.") - queued_message.destroy - return - end - - # If the route says we're holding quananteed mail and this is spam, we'll hold this - if route.spam_mode == "Fail" && queued_message.message.spam && !queued_message.manual? - log "message is spam and route says to fail spam message, hard failing" - queued_message.message.create_delivery("HardFail", details: "Message is spam and the route specifies it should be failed.") - queued_message.destroy - return - end - - # - # Messages that should be blindly accepted are blindly accepted - # - if route.mode == "Accept" - log "route says to accept without endpoint, marking as processed" - queued_message.message.create_delivery("Processed", details: "Message has been accepted but not sent to any endpoints.") - queued_message.destroy - return - end - - # - # Messages that should be accepted and held should be held - # - if route.mode == "Hold" - if queued_message.manual? - log "route says to hold and message was queued manually, marking as processed" - queued_message.message.create_delivery("Processed", details: "Message has been processed.") - else - log "route says to hold, marking as held" - queued_message.message.create_delivery("Held", details: "Message has been accepted but not sent to any endpoints.") - end - queued_message.destroy - return - end - - # - # Messages that should be bounced should be bounced (or rejected if they got this far) - # - if route.mode == "Bounce" || route.mode == "Reject" - log "route says to bounce, hard failing and sending bounce" - if id = queued_message.send_bounce - log "bounce sent with id #{id}" - queued_message.message.create_delivery("HardFail", details: "Message has been bounced because the route asks for this. See message ") - end - queued_message.destroy - return - end - - if @fixed_result - result = @fixed_result - else - case queued_message.message.endpoint - when SMTPEndpoint - sender = cached_sender(Postal::SMTPSender, queued_message.message.recipient_domain, nil, servers: [queued_message.message.endpoint]) - when HTTPEndpoint - sender = cached_sender(Postal::HTTPSender, queued_message.message.endpoint) - when AddressEndpoint - sender = cached_sender(Postal::SMTPSender, queued_message.message.endpoint.domain, nil, force_rcpt_to: queued_message.message.endpoint.address) - else - log "invalid endpoint for route (#{queued_message.message.endpoint_type})" - queued_message.message.create_delivery("HardFail", details: "Invalid endpoint for route.") - queued_message.destroy - return - end - result = sender.send_message(queued_message.message) - if result.connect_error - @fixed_result = result - end - end - - # Log the result - log_details = result.details - if result.type == "HardFail" && result.suppress_bounce - # The delivery hard failed, but requested that no bounce be sent - log "suppressing bounce message after hard fail" - elsif result.type == "HardFail" && queued_message.message.send_bounces? - # If the message is a hard fail, send a bounce message for this message. - log "sending a bounce because message hard failed" - if bounce_id = queued_message.send_bounce - log_details += "." unless log_details =~ /\.\z/ - log_details += " Sent bounce message to sender (see message )" - end - end - - queued_message.message.create_delivery(result.type, details: log_details, output: result.output&.strip, sent_with_ssl: result.secure, log_id: result.log_id, time: result.time) - - if result.retry - queued_message.retry_later(result.retry.is_a?(Integer) ? result.retry : nil) - log "message requeued for trying later, at #{queued_message.retry_after}" - queued_message.allocate_ip_address - queued_message.update_column(:ip_address_id, queued_message.ip_address&.id) - else - log "message processing completed" - queued_message.message.endpoint.mark_as_used - queued_message.destroy - end - else - log "no route and/or endpoint available for processing, hard failing" - queued_message.message.create_delivery("HardFail", details: "Message does not have a route and/or endpoint available for delivery.") - queued_message.destroy - return - end - end - - # - # Handle Outgoing Messages - # - return unless queued_message.message.scope == "outgoing" - - log "message is outgoing" - - if queued_message.message.domain.nil? - log "message has no domain, hard failing" - queued_message.message.create_delivery("HardFail", details: "Message's domain no longer exist") - queued_message.destroy - return - end - - # - # If there's no to address, we can't do much. Fail it. - # - if queued_message.message.rcpt_to.blank? - log "message has no 'to' address, hard failing" - queued_message.message.create_delivery("HardFail", details: "Message doesn't have an RCPT to") - queued_message.destroy - return - end - - # Extract a tag and add it to the message if one doesn't exist - if queued_message.message.tag.nil? && tag = queued_message.message.headers["x-postal-tag"] - log "added tag: #{tag.last}" - queued_message.message.update(tag: tag.last) - end - - # - # If the credentials for this message is marked as holding and this isn't manual, hold it - # - if !queued_message.manual? && queued_message.message.credential && queued_message.message.credential.hold? - log "credential wants us to hold messages, holding" - queued_message.message.create_delivery("Held", details: "Credential is configured to hold all messages authenticated by it.") - queued_message.destroy - return - end - - # - # If the recipient is on the suppression list and this isn't a manual queueing block sending - # - if !queued_message.manual? && sl = queued_message.server.message_db.suppression_list.get(:recipient, queued_message.message.rcpt_to) - log "recipient is on the suppression list, holding" - queued_message.message.create_delivery("Held", details: "Recipient (#{queued_message.message.rcpt_to}) is on the suppression list (reason: #{sl['reason']})") - queued_message.destroy - return - end - - # Parse the content of the message as appropriate - if queued_message.message.should_parse? - log "parsing message content as it hasn't been parsed before" - queued_message.message.parse_content - end - - # Inspect outgoing messages when there's a threshold set for the server - if !queued_message.message.inspected && queued_message.server.outbound_spam_threshold - log "inspecting message" - queued_message.message.inspect_message - if queued_message.message.inspected - if queued_message.message.spam_score >= queued_message.server.outbound_spam_threshold - queued_message.message.update(spam: true) - end - log "message inspected successfully", spam: queued_message.message.spam?, spam_score: queued_message.message.spam_score - end - end - - if queued_message.message.spam - log "message is spam (#{queued_message.message.spam_score}), hard failing", server_threshold: queued_message.server.outbound_spam_threshold - queued_message.message.create_delivery("HardFail", - details: "Message is likely spam. Threshold is #{queued_message.server.outbound_spam_threshold} and " \ - "the message scored #{queued_message.message.spam_score}.") - queued_message.destroy - return - end - - # Add outgoing headers - unless queued_message.message.has_outgoing_headers? - queued_message.message.add_outgoing_headers - end - - # Check send limits - if queued_message.server.send_limit_exceeded? - # If we're over the limit, we're going to be holding this message - log "server send limit has been exceeded, holding", send_limit: queued_message.server.send_limit - queued_message.server.update_columns(send_limit_exceeded_at: Time.now, send_limit_approaching_at: nil) - queued_message.message.create_delivery("Held", details: "Message held because send limit (#{queued_message.server.send_limit}) has been reached.") - queued_message.destroy - return - elsif queued_message.server.send_limit_approaching? - # If we're approaching the limit, just say we are but continue to process the message - queued_message.server.update_columns(send_limit_approaching_at: Time.now, send_limit_exceeded_at: nil) - else - queued_message.server.update_columns(send_limit_approaching_at: nil, send_limit_exceeded_at: nil) - end - - # Update the live stats for this message. - queued_message.message.database.live_stats.increment(queued_message.message.scope) - - # If the server is in development mode, hold it - if queued_message.server.mode == "Development" && !queued_message.manual? - log "server is in development mode, holding" - queued_message.message.create_delivery("Held", details: "Server is in development mode.") - queued_message.destroy - return - end - - # Send the outgoing message to the SMTP sender - - if @fixed_result - result = @fixed_result - else - sender = cached_sender(Postal::SMTPSender, queued_message.message.recipient_domain, queued_message.ip_address) - result = sender.send_message(queued_message.message) - if result.connect_error - @fixed_result = result - end - end - - # - # If the message has been hard failed, check to see how many other recent hard fails we've had for the address - # and if there are more than 2, suppress the address for 30 days. - # - if result.type == "HardFail" - recent_hard_fails = queued_message.server.message_db.select(:messages, - where: { - rcpt_to: queued_message.message.rcpt_to, - status: "HardFail", - timestamp: { greater_than: 24.hours.ago.to_f } - }, - count: true) - if recent_hard_fails >= 1 && queued_message.server.message_db.suppression_list.add(:recipient, queued_message.message.rcpt_to, reason: "too many hard fails") - log "Added #{queued_message.message.rcpt_to} to suppression list because #{recent_hard_fails} hard fails in 24 hours" - result.details += "." if result.details =~ /\.\z/ - result.details += " " if result.details.present? - result.details += "Recipient added to suppression list (too many hard fails)." - end - end - - # - # If a message is sent successfully, remove the users from the suppression list - # - if result.type == "Sent" && queued_message.server.message_db.suppression_list.remove(:recipient, queued_message.message.rcpt_to) - log "removed #{queued_message.message.rcpt_to} from suppression list" - result.details += "." if result.details =~ /\.\z/ - result.details += " Recipient removed from suppression list." - end - - # Log the result - queued_message.message.create_delivery(result.type, details: result.details, output: result.output, sent_with_ssl: result.secure, log_id: result.log_id, time: result.time) - - if result.retry - queued_message.retry_later(result.retry.is_a?(Integer) ? result.retry : nil) - log "message requeued for trying later", retry_after: queued_message.retry_after - else - log "message processing complete" - queued_message.destroy - end - rescue StandardError => e - log "internal error: #{e.class}: #{e.message}" - e.backtrace.each { |line| log(line) } - - queued_message.retry_later - log "message requeued for trying later, at #{queued_message.retry_after}" - - if defined?(Sentry) - Sentry.capture_exception(e, extra: { server_id: queued_message.server_id, queued_message_id: queued_message.message_id }) - end - - queued_message.message&.create_delivery("Error", - details: "An internal error occurred while sending " \ - "this message. This message will be retried " \ - "automatically.", - output: "#{e.class}: #{e.message}", log_id: "J-#{id}") - end - -end diff --git a/app/util/message_dequeuer.rb b/app/util/message_dequeuer.rb new file mode 100644 index 0000000..1ab1f6e --- /dev/null +++ b/app/util/message_dequeuer.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module MessageDequeuer + + class << self + + def process(message, logger:) + processor = InitialProcessor.new(message, logger: logger) + processor.process + end + + end + +end diff --git a/lib/worker/jobs/process_queued_messages_job.rb b/lib/worker/jobs/process_queued_messages_job.rb index 60c8ba3..18489b5 100644 --- a/lib/worker/jobs/process_queued_messages_job.rb +++ b/lib/worker/jobs/process_queued_messages_job.rb @@ -64,7 +64,7 @@ module Worker def process_messages @messages_to_process.each do |message| work_completed! - UnqueueMessageService.new(queued_message: message, logger: logger).call + MessageDequeuer.process(message, logger: logger) end end diff --git a/spec/lib/message_dequeuer/base_spec.rb b/spec/lib/message_dequeuer/base_spec.rb new file mode 100644 index 0000000..1092e23 --- /dev/null +++ b/spec/lib/message_dequeuer/base_spec.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require "rails_helper" + +module MessageDequeuer + + RSpec.describe Base do + describe ".new" do + context "when given state" do + it "uses that state" do + base = described_class.new(nil, logger: nil, state: 1234) + expect(base.state).to eq 1234 + end + end + + context "when not given state" do + it "creates a new state" do + base = described_class.new(nil, logger: nil) + expect(base.state).to be_a State + end + end + end + + describe ".process" do + it "creates a new instances of the class and calls process" do + message = create(:queued_message) + logger = TestLogger.new + + mock = double("Base") + expect(mock).to receive(:process).once + expect(described_class).to receive(:new).with(message, logger: logger).and_return(mock) + + described_class.process(message, logger: logger) + end + end + end + +end diff --git a/spec/lib/message_dequeuer/incoming_message_processor_spec.rb b/spec/lib/message_dequeuer/incoming_message_processor_spec.rb new file mode 100644 index 0000000..5ac111a --- /dev/null +++ b/spec/lib/message_dequeuer/incoming_message_processor_spec.rb @@ -0,0 +1,640 @@ +# frozen_string_literal: true + +require "rails_helper" + +module MessageDequeuer + + RSpec.describe IncomingMessageProcessor do + let(:server) { create(:server) } + let(:state) { State.new } + let(:logger) { TestLogger.new } + let(:route) { create(:route, server: server) } + let(:message) { MessageFactory.incoming(server, route: route) } + let(:queued_message) { create(:queued_message, :locked, message: message) } + + subject(:processor) { described_class.new(queued_message, logger: logger, state: state) } + + context "when the message was a bounce but there's no return path for it" do + let(:message) do + MessageFactory.incoming(server) do |msg| + msg.bounce = true + end + end + + it "logs" do + processor.process + expect(logger).to have_logged(/no source messages found, hard failing/) + end + + it "sets the message status to HardFail" do + processor.process + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + processor.process + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /was a bounce but we couldn't link it with any outgoing message/i) + end + + it "removes the queued message" do + processor.process + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the message is a bounce for an existing message" do + let(:existing_message) { MessageFactory.outgoing(server) } + + let(:message) do + MessageFactory.incoming(server) do |msg, mail| + msg.bounce = true + mail["X-Postal-MsgID"] = existing_message.token + end + end + + it "logs" do + processor.process + expect(logger).to have_logged(/message is a bounce/) + end + + it "adds the original message as the bounce ID for the received message" do + processor.process + expect(message.reload.bounce_for_id).to eq existing_message.id + end + + it "sets the received message status to Processed" do + processor.process + expect(message.reload.status).to eq "Processed" + end + + it "creates a Processed delivery on the received message" do + processor.process + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Processed", details: /This has been detected as a bounce message for /i) + end + + it "sets the existing message status to Bounced" do + processor.process + expect(existing_message.reload.status).to eq "Bounced" + end + + it "creates a Bounced delivery on the original message" do + processor.process + delivery = existing_message.deliveries.last + expect(delivery).to have_attributes(status: "Bounced", details: /received a bounce message for this e-mail. See for/i) + end + + it "triggers a MessageBounced webhook event" do + expect(WebhookRequest).to receive(:trigger).with(server, "MessageBounced", { + original_message: kind_of(Hash), + bounce: kind_of(Hash) + }) + processor.process + end + + it "removes the queued message" do + processor.process + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the message is not a bounce" do + it "increments the stats for the server" do + expect { processor.process }.to change { server.message_db.live_stats.total(5) }.by(1) + end + + it "inspects the message and adds headers" do + expect { processor.process }.to change { message.reload.inspected }.from(false).to(true) + new_message = message.reload + expect(new_message.headers).to match hash_including( + "x-postal-spam" => ["no"], + "x-postal-spam-threshold" => ["5.0"], + "x-postal-threat" => ["no"] + ) + end + + it "marks the message as spam if the spam score is higher than the server threshold" do + inspection_result = double("Result", spam_score: server.spam_threshold + 1, threat: false, threat_message: nil, spam_checks: []) + allow(Postal::MessageInspection).to receive(:scan).and_return(inspection_result) + processor.process + expect(message.reload.spam).to be true + end + end + + context "when the message has a spam score greater than the server's spam failure threshold" do + before do + inspection_result = double("Result", spam_score: 100, threat: false, threat_message: nil, spam_checks: []) + allow(Postal::MessageInspection).to receive(:scan).and_return(inspection_result) + end + + it "logs" do + processor.process + expect(logger).to have_logged(/message has a spam score higher than the server's maxmimum/) + end + + it "sets the message status to HardFail" do + processor.process + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + processor.process + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /spam score is higher than the failure threshold for this server/i) + end + + it "removes the queued message" do + processor.process + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the server mode is Development and the message was not manually queued" do + before do + server.update!(mode: "Development") + end + + after do + server.update!(mode: "Live") + end + + it "logs" do + processor.process + expect(logger).to have_logged(/server is in development mode/) + end + + it "sets the message status to Held" do + processor.process + expect(message.reload.status).to eq "Held" + end + + it "creates a Held delivery" do + processor.process + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Held", details: /server is in development mode/i) + end + + it "removes the queued message" do + processor.process + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when there is no route for the incoming message" do + let(:route) { nil } + + it "logs" do + processor.process + expect(logger).to have_logged(/no route and\/or endpoint available for processing/i) + end + + it "sets the message status to HardFail" do + processor.process + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + processor.process + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /does not have a route and\/or endpoint available/i) + end + + it "removes the queued message" do + processor.process + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the route's spam mode is Quarantine, the message is spam and not manually queued" do + let(:route) { create(:route, server: server, spam_mode: "Quarantine") } + + before do + inspection_result = double("Result", spam_score: server.spam_threshold + 1, threat: false, threat_message: nil, spam_checks: []) + allow(Postal::MessageInspection).to receive(:scan).and_return(inspection_result) + end + + it "logs" do + processor.process + expect(logger).to have_logged(/message is spam and route says to quarantine spam message/i) + end + + it "sets the message status to Held" do + processor.process + expect(message.reload.status).to eq "Held" + end + + it "creates a Held delivery" do + processor.process + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Held", details: /message placed into quarantine/i) + end + + it "removes the queued message" do + processor.process + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the route's spam mode is Fail, the message is spam and not manually queued" do + let(:route) { create(:route, server: server, spam_mode: "Fail") } + + before do + inspection_result = double("Result", spam_score: server.spam_threshold + 1, threat: false, threat_message: nil, spam_checks: []) + allow(Postal::MessageInspection).to receive(:scan).and_return(inspection_result) + end + + it "logs" do + processor.process + expect(logger).to have_logged(/message is spam and route says to fail spam message/i) + end + + it "sets the message status to HardFail" do + processor.process + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + processor.process + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /message is spam and the route specifies it should be failed/i) + end + + it "removes the queued message" do + processor.process + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the route's mode is Accept" do + it "logs" do + processor.process + expect(logger).to have_logged(/route says to accept without endpoint/i) + end + + it "sets the message status to Processed" do + processor.process + expect(message.reload.status).to eq "Processed" + end + + it "creates a Processed delivery" do + processor.process + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Processed", details: /message has been accepted but not sent to any endpoints/i) + end + + it "removes the queued message" do + processor.process + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the route's mode is Hold" do + let(:route) { create(:route, server: server, mode: "Hold") } + + context "when the message was queued manually" do + let(:queued_message) { create(:queued_message, :locked, server: server, message: message, manual: true) } + + it "logs" do + processor.process + expect(logger).to have_logged(/route says to hold and message was queued manually/i) + end + + it "sets the message status to Processed" do + processor.process + expect(message.reload.status).to eq "Processed" + end + + it "creates a Processed delivery" do + processor.process + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Processed", details: /message has been processed/i) + end + + it "removes the queued message" do + processor.process + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the message was not queued manually" do + let(:queued_message) { create(:queued_message, :locked, server: server, message: message, manual: false) } + + it "logs" do + processor.process + expect(logger).to have_logged(/route says to hold, marking as held/i) + end + + it "sets the message status to Held" do + processor.process + expect(message.reload.status).to eq "Held" + end + + it "creates a Held delivery" do + processor.process + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Held", details: /message has been accepted but not sent to any endpoints/i) + end + + it "removes the queued message" do + processor.process + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + end + + context "when the route's mode is Bounce" do + let(:route) { create(:route, server: server, mode: "Bounce") } + + it "logs" do + processor.process + expect(logger).to have_logged(/route says to bounce/i) + end + + it "sends a bounce" do + expect(BounceMessage).to receive(:new).with(server, queued_message.message) + processor.process + end + + it "sets the message status to HardFail" do + processor.process + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + processor.process + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /message has been bounced because/i) + end + + it "removes the queued message" do + processor.process + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the route's mode is Reject" do + let(:route) { create(:route, server: server, mode: "Reject") } + + it "logs" do + processor.process + expect(logger).to have_logged(/route says to bounce/i) + end + + it "sends a bounce" do + expect(BounceMessage).to receive(:new).with(server, queued_message.message) + processor.process + end + + it "sets the message status to HardFail" do + processor.process + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + processor.process + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /message has been bounced because/i) + end + + it "removes the queued message" do + processor.process + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the route's endpoint is an HTTP endpoint" do + let(:endpoint) { create(:http_endpoint, server: server) } + let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) } + + it "gets a sender from the state and sends the message to it" do + http_sender_double = double("HTTPSender") + expect(http_sender_double).to receive(:send_message).with(queued_message.message).and_return(Postal::SendResult.new) + expect(state).to receive(:sender_for).with(Postal::HTTPSender, endpoint).and_return(http_sender_double) + processor.process + end + end + + context "when the route's endpoint is an SMTP endpoint" do + let(:endpoint) { create(:smtp_endpoint, server: server) } + let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) } + + it "gets a sender from the state and sends the message to it" do + smtp_sender_double = double("SMTPSender") + expect(smtp_sender_double).to receive(:send_message).with(queued_message.message).and_return(Postal::SendResult.new) + expect(state).to receive(:sender_for).with(Postal::SMTPSender, message.recipient_domain, nil, { servers: [endpoint] }).and_return(smtp_sender_double) + processor.process + end + end + + context "when the route's endpoint is an Address endpoint" do + let(:endpoint) { create(:address_endpoint, server: server) } + let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) } + + it "gets a sender from the state and sends the message to it" do + smtp_sender_double = double("SMTPSender") + expect(smtp_sender_double).to receive(:send_message).with(queued_message.message).and_return(Postal::SendResult.new) + expect(state).to receive(:sender_for).with(Postal::SMTPSender, endpoint.domain, nil, { force_rcpt_to: endpoint.address }).and_return(smtp_sender_double) + processor.process + end + end + + context "when the route's endpoint is an unknown endpoint" do + let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: create(:webhook, server: server)) } + + it "logs" do + processor.process + expect(logger).to have_logged(/invalid endpoint for route/i) + end + + it "sets the message status to HardFail" do + processor.process + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + processor.process + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /invalid endpoint for route/i) + end + + it "removes the queued message" do + processor.process + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the message has been sent to a sender" do + let(:endpoint) { create(:smtp_endpoint, server: server) } + let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) } + + let(:send_result) do + Postal::SendResult.new do |result| + result.type = "Sent" + result.details = "Sent successfully" + end + end + + before do + smtp_sender_mock = double("SMTPSender") + allow(Postal::SMTPSender).to receive(:new).and_return(smtp_sender_mock) + allow(smtp_sender_mock).to receive(:start) + allow(smtp_sender_mock).to receive(:finish) + allow(smtp_sender_mock).to receive(:send_message).and_return(send_result) + end + + context "when the sender returns a HardFail and bounces are suppressed" do + before do + send_result.type = "HardFail" + send_result.suppress_bounce = true + end + + it "logs" do + processor.process + expect(logger).to have_logged(/suppressing bounce message after hard fail/) + end + + it "does not send a bounce" do + allow(BounceMessage).to receive(:new) + processor.process + expect(BounceMessage).to_not have_received(:new) + end + end + + context "when the sender returns a HardFail and bounces should be sent" do + before do + send_result.type = "HardFail" + send_result.details = "Failed to send message" + end + + it "logs" do + processor.process + expect(logger).to have_logged(/sending a bounce because message hard failed/) + end + + it "sends a bounce" do + expect(BounceMessage).to receive(:new).with(server, queued_message.message) + processor.process + end + + it "sets the message status to HardFail" do + processor.process + expect(message.reload.status).to eq "HardFail" + end + + it "creates a delivery with the details and a suffix about the bounce message" do + processor.process + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /Failed to send message. Sent bounce message to sender \(see message \)/i) + end + end + + it "creates a delivery with the result from the sender" do + send_result.output = "some output here" + send_result.secure = true + send_result.log_id = "12345" + send_result.time = 2.32 + + processor.process + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Sent", + details: "Sent successfully", + output: "some output here", + sent_with_ssl: true, + log_id: "12345", + time: 2.32) + end + + context "when the sender wants to retry" do + before do + send_result.type = "SoftFail" + send_result.retry = true + end + + it "logs" do + processor.process + expect(logger).to have_logged(/message requeued for trying later, at/i) + end + + it "sets the message status to SoftFail" do + processor.process + expect(message.reload.status).to eq "SoftFail" + end + + it "updates the queued message with a new retry time" do + Timecop.freeze do + retry_time = 5.minutes.from_now.change(usec: 0) + processor.process + expect(queued_message.reload.retry_after).to eq retry_time + end + end + + it "allocates a new IP address to send the message from and updates the queued message" do + expect(queued_message).to receive(:allocate_ip_address) + processor.process + end + + it "does not remove the queued message" do + processor.process + expect(queued_message.reload).to be_present + end + end + + context "when the sender does not want a retry" do + it "logs" do + processor.process + expect(logger).to have_logged(/message processing completed/i) + end + + it "sets the message status to Sent" do + processor.process + expect(message.reload.status).to eq "Sent" + end + + it "marks the endpoint as used" do + route.endpoint.update!(last_used_at: nil) + Timecop.freeze do + expect { processor.process }.to change { route.endpoint.reload.last_used_at.to_i }.from(0).to(Time.now.to_i) + end + end + + it "removes the queued message" do + processor.process + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + end + + context "when an exception occurrs during processing" do + let(:endpoint) { create(:smtp_endpoint, server: server) } + let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) } + + before do + smtp_sender_mock = double("SMTPSender") + allow(Postal::SMTPSender).to receive(:new).and_return(smtp_sender_mock) + allow(smtp_sender_mock).to receive(:start) + allow(smtp_sender_mock).to receive(:finish) + allow(smtp_sender_mock).to receive(:send_message) do + 1 / 0 + end + end + + it "logs" do + processor.process + expect(logger).to have_logged(/internal error: ZeroDivisionError/i) + end + + it "creates an Error delivery" do + processor.process + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Error", details: /internal error/i) + end + + it "marks the message for retrying later" do + processor.process + expect(queued_message.reload.retry_after).to be_present + end + end + end + +end diff --git a/spec/lib/message_dequeuer/initial_message_processor_spec.rb b/spec/lib/message_dequeuer/initial_message_processor_spec.rb new file mode 100644 index 0000000..b11b2f8 --- /dev/null +++ b/spec/lib/message_dequeuer/initial_message_processor_spec.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require "rails_helper" + +module MessageDequeuer + + RSpec.describe InitialProcessor do + let(:server) { create(:server) } + let(:logger) { TestLogger.new } + let(:route) { create(:route, server: server) } + let(:message) { MessageFactory.incoming(server, route: route) } + let(:queued_message) { create(:queued_message, :locked, message: message) } + + subject(:processor) { described_class.new(queued_message, logger: logger) } + + it "has state when not given any" do + expect(processor.state).to be_a State + end + + context "when associated message does not exist" do + let(:queued_message) { create(:queued_message, :locked, message_id: 12_345) } + + it "logs" do + processor.process + expect(logger).to have_logged(/unqueue because backend message has been removed/) + end + + it "removes from queued message" do + processor.process + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the queued message is not ready for processing" do + let(:queued_message) { create(:queued_message, :locked, message: message, retry_after: 1.hour.from_now) } + + it "logs" do + processor.process + expect(logger).to have_logged(/skipping because message isn't ready for processing/) + end + + it "unlocks and keeps the queued message" do + processor.process + expect(queued_message.reload).to_not be_locked + end + end + + context "when there are no other batchable messages" do + it "calls the single message processor for the initial message" do + expect(SingleMessageProcessor).to receive(:process).with(queued_message, + logger: logger, + state: processor.state) + processor.process + end + end + + context "when there are batchable messages" do + before do + @message2 = MessageFactory.incoming(server, route: route) + @queued_message2 = create(:queued_message, message: @message2) + @message3 = MessageFactory.incoming(server, route: route) + @queued_message3 = create(:queued_message, message: @message3) + end + + it "calls the single message process for the initial message and all batchable messages" do + [queued_message, @queued_message2, @queued_message3].each do |msg| + expect(SingleMessageProcessor).to receive(:process).with(msg, + logger: logger, + state: processor.state) + end + processor.process + end + end + + context "when an error occurs while finding batchable messages" do + before do + allow(queued_message).to receive(:batchable_messages) { 1 / 0 } + end + + it "unlocks the queued message and raises the error" do + expect { processor.process }.to raise_error(ZeroDivisionError) + expect(queued_message.reload).to_not be_locked + end + end + + context "when finished" do + it "notifies the state that processing is complete" do + expect(processor.state).to receive(:finished) + processor.process + end + end + end + +end diff --git a/spec/services/unqueue_message_service/outgoing_message_spec.rb b/spec/lib/message_dequeuer/outgoing_message_processor_spec.rb similarity index 71% rename from spec/services/unqueue_message_service/outgoing_message_spec.rb rename to spec/lib/message_dequeuer/outgoing_message_processor_spec.rb index 49e7e70..a455175 100644 --- a/spec/services/unqueue_message_service/outgoing_message_spec.rb +++ b/spec/lib/message_dequeuer/outgoing_message_processor_spec.rb @@ -2,145 +2,40 @@ require "rails_helper" -RSpec.describe UnqueueMessageService do - let(:server) { create(:server) } - let(:logger) { TestLogger.new } - let(:send_result) do - Postal::SendResult.new do |r| - r.type = "Sent" - end - end - subject(:service) { described_class.new(queued_message: queued_message, logger: logger) } +module MessageDequeuer - # We're going to, for now, just stop the SMTP sender from doing anything here because - # we don't want to leak out of this test in to the real world. - before do - smtp_sender_mock = double("SMTPSender") - allow(Postal::SMTPSender).to receive(:new).and_return(smtp_sender_mock) - allow(smtp_sender_mock).to receive(:start) - allow(smtp_sender_mock).to receive(:finish) - allow(smtp_sender_mock).to receive(:send_message).and_return(send_result) - end - - context "for an outgoing message" do + RSpec.describe OutgoingMessageProcessor do + let(:server) { create(:server) } + let(:state) { State.new } + let(:logger) { TestLogger.new } let(:domain) { create(:domain, server: server) } let(:credential) { create(:credential, server: server) } let(:message) { MessageFactory.outgoing(server, domain: domain, credential: credential) } let(:queued_message) { create(:queued_message, :locked, message: message) } - context "when the server is suspended" do - let(:server) { create(:server, :suspended) } - - it "logs" do - service.call - expect(logger).to have_logged(/server is suspended/) - end - - it "sets the message status to Held" do - service.call - expect(message.reload.status).to eq "Held" - end - - it "creates a Hold delivery" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "Held", details: /server has been suspended/i) - end - - it "removes the queued message" do - service.call - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "when the number of attempts is more than the maximum" do - let(:queued_message) { create(:queued_message, :locked, message: message, attempts: Postal.config.general.maximum_delivery_attempts + 1) } - - it "logs" do - service.call - expect(logger).to have_logged(/message has reached maximum number of attempts/) - end - - it "adds the recipient to the suppression list and logs this" do - Timecop.freeze do - service.call - entry = server.message_db.suppression_list.get(:recipient, message.rcpt_to) - expect(entry).to match hash_including( - "address" => message.rcpt_to, - "type" => "recipient", - "reason" => "too many soft fails" - ) - end - end - - it "sets the message status to Held" do - service.call - expect(message.reload.status).to eq "HardFail" - end - - it "creates a HardFail delivery" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "HardFail", details: /maximum number of delivery attempts.*added [\w.@]+ to suppression list/i) - end - - it "removes the queued message" do - service.call - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "when the message raw data has been removed" do - before do - message.raw_table = nil - message.save - end - - it "logs" do - service.call - expect(logger).to have_logged(/raw message has been removed/) - end - - it "sets the message status to Held" do - service.call - expect(message.reload.status).to eq "HardFail" - end - - it "creates a HardFail delivery" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "HardFail", details: /Raw message has been removed/i) - end - - it "removes the queued message" do - service.call - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end + subject(:processor) { described_class.new(queued_message, logger: logger, state: state) } context "when the domain belonging to the message no longer exists" do - before do - domain.destroy - end + let(:message) { MessageFactory.outgoing(server, domain: nil, credential: credential) } it "logs" do - service.call + processor.process expect(logger).to have_logged(/message has no domain/) end it "sets the message status to HardFail" do - service.call + processor.process expect(message.reload.status).to eq "HardFail" end it "creates a HardFail delivery" do - service.call + processor.process delivery = message.deliveries.last expect(delivery).to have_attributes(status: "HardFail", details: /Message's domain no longer exist/i) end it "removes the queued message" do - service.call + processor.process expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) end end @@ -151,23 +46,23 @@ RSpec.describe UnqueueMessageService do end it "logs" do - service.call + processor.process expect(logger).to have_logged(/message has no 'to' address/) end it "sets the message status to HardFail" do - service.call + processor.process expect(message.reload.status).to eq "HardFail" end it "creates a HardFail delivery" do - service.call + processor.process delivery = message.deliveries.last expect(delivery).to have_attributes(status: "HardFail", details: /Message doesn't have an RCPT to/i) end it "removes the queued message" do - service.call + processor.process expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) end end @@ -180,12 +75,12 @@ RSpec.describe UnqueueMessageService do end it "logs" do - service.call + processor.process expect(logger).to have_logged(/added tag: example-tag/) end it "adds the tag to the message object" do - service.call + processor.process expect(message.reload.tag).to eq("example-tag") end end @@ -197,7 +92,7 @@ RSpec.describe UnqueueMessageService do let(:queued_message) { create(:queued_message, :locked, message: message, manual: true) } it "does not hold the message" do - service.call + processor.process deliveries = message.deliveries.find { |d| d.status == "Held" } expect(deliveries).to be_nil end @@ -205,23 +100,23 @@ RSpec.describe UnqueueMessageService do context "when the message was not queued manually" do it "logs" do - service.call + processor.process expect(logger).to have_logged(/credential wants us to hold messages/) end it "sets the message status to Held" do - service.call + processor.process expect(message.reload.status).to eq "Held" end it "creates a Held delivery" do - service.call + processor.process delivery = message.deliveries.last expect(delivery).to have_attributes(status: "Held", details: /Credential is configured to hold all messages authenticated/i) end it "removes the queued message" do - service.call + processor.process expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) end end @@ -236,7 +131,7 @@ RSpec.describe UnqueueMessageService do let(:queued_message) { create(:queued_message, :locked, message: message, manual: true) } it "does not hold the message" do - service.call + processor.process deliveries = message.deliveries.find { |d| d.status == "Held" } expect(deliveries).to be_nil end @@ -244,23 +139,23 @@ RSpec.describe UnqueueMessageService do context "when the message was not queued manually" do it "logs" do - service.call + processor.process expect(logger).to have_logged(/recipient is on the suppression list/) end it "sets the message status to Held" do - service.call + processor.process expect(message.reload.status).to eq "Held" end it "creates a Held delivery" do - service.call + processor.process delivery = message.deliveries.last expect(delivery).to have_attributes(status: "Held", details: /Recipient \(#{message.rcpt_to}\) is on the suppression list/i) end it "removes the queued message" do - service.call + processor.process expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) end end @@ -273,7 +168,7 @@ RSpec.describe UnqueueMessageService do allow(mocked_parser).to receive(:tracked_links).and_return(0) allow(mocked_parser).to receive(:tracked_images).and_return(0) expect(Postal::MessageParser).to receive(:new).with(kind_of(Postal::MessageDB::Message)).and_return(mocked_parser) - service.call + processor.process reloaded_message = message.reload expect(reloaded_message.parsed).to eq 1 expect(reloaded_message.tracked_links).to eq 0 @@ -285,7 +180,7 @@ RSpec.describe UnqueueMessageService do let(:server) { create(:server, outbound_spam_threshold: 5.0) } it "logs" do - service.call + processor.process expect(logger).to have_logged(/inspecting message/) expect(logger).to have_logged(/message inspected successfully/) end @@ -293,7 +188,7 @@ RSpec.describe UnqueueMessageService do it "inspects the message" do inspection_result = double("Result", spam_score: 1.0, threat: false, threat_message: nil, spam_checks: []) expect(Postal::MessageInspection).to receive(:scan).and_return(inspection_result) - service.call + processor.process end context "when the message spam score is higher than the threshold" do @@ -303,28 +198,28 @@ RSpec.describe UnqueueMessageService do end it "logs" do - service.call + processor.process expect(logger).to have_logged(/message is spam/) end it "sets the spam boolean on the message" do - service.call + processor.process expect(message.reload.spam).to be true end it "sets the message status to HardFail" do - service.call + processor.process expect(message.reload.status).to eq "HardFail" end it "creates a HardFail delivery" do - service.call + processor.process delivery = message.deliveries.last expect(delivery).to have_attributes(status: "HardFail", details: /Message is likely spam. Threshold is 5.0 and the message scored 6.0/i) end it "removes the queued message" do - service.call + processor.process expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) end end @@ -333,7 +228,7 @@ RSpec.describe UnqueueMessageService do context "when the server does not have a outbound spam threshold configured" do it "does not inspect the message" do expect(Postal::MessageInspection).to_not receive(:scan) - service.call + processor.process end end @@ -345,24 +240,24 @@ RSpec.describe UnqueueMessageService do end it "does not another one" do - service.call + processor.process expect(message.reload.headers["x-postal-msgid"]).to eq ["existing-id"] end it "does not add dkim headers" do - service.call + processor.process expect(message.reload.headers["dkim-signature"]).to be_nil end end context "when the message does not have a x-postal-msgid header" do it "adds it" do - service.call + processor.process expect(message.reload.headers["x-postal-msgid"]).to match [match(/[a-zA-Z0-9]{12}/)] end it "adds a dkim header" do - service.call + processor.process expect(message.reload.headers["dkim-signature"]).to match [match(/\Av=1; a=rsa-sha256/)] end end @@ -375,27 +270,27 @@ RSpec.describe UnqueueMessageService do end it "updates the time the limit was exceeded" do - expect { service.call }.to change { server.reload.send_limit_exceeded_at }.from(nil).to(kind_of(Time)) + expect { processor.process }.to change { server.reload.send_limit_exceeded_at }.from(nil).to(kind_of(Time)) end it "logs" do - service.call + processor.process expect(logger).to have_logged(/server send limit has been exceeded/) end it "sets the message status to Held" do - service.call + processor.process expect(message.reload.status).to eq "Held" end it "creates a Held delivery" do - service.call + processor.process delivery = message.deliveries.last expect(delivery).to have_attributes(status: "Held", details: /Message held because send limit \(5\) has been reached/i) end it "removes the queued message" do - service.call + processor.process expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) end end @@ -408,11 +303,11 @@ RSpec.describe UnqueueMessageService do end it "updates the time the limit was being approached" do - expect { service.call }.to change { server.reload.send_limit_approaching_at }.from(nil).to(kind_of(Time)) + expect { processor.process }.to change { server.reload.send_limit_approaching_at }.from(nil).to(kind_of(Time)) end it "does not set the exceeded time" do - expect { service.call }.to_not change { server.reload.send_limit_exceeded_at } # rubocop:disable Lint/AmbiguousBlockAssociation + expect { processor.process }.to_not change { server.reload.send_limit_exceeded_at } # rubocop:disable Lint/AmbiguousBlockAssociation end end @@ -420,7 +315,7 @@ RSpec.describe UnqueueMessageService do let(:server) { create(:server, :exceeded_send_limit, send_limit: 10) } it "clears the approaching and exceeded limits" do - service.call + processor.process server.reload expect(server.send_limit_approaching_at).to be_nil expect(server.send_limit_exceeded_at).to be_nil @@ -434,7 +329,7 @@ RSpec.describe UnqueueMessageService do let(:queued_message) { create(:queued_message, :locked, message: message, manual: true) } it "does not hold the message" do - service.call + processor.process deliveries = message.deliveries.find { |d| d.status == "Held" } expect(deliveries).to be_nil end @@ -442,47 +337,65 @@ RSpec.describe UnqueueMessageService do context "when the message was not queued manually" do it "logs" do - service.call + processor.process expect(logger).to have_logged(/server is in development mode/) end it "sets the message status to Held" do - service.call + processor.process expect(message.reload.status).to eq "Held" end it "creates a Held delivery" do - service.call + processor.process delivery = message.deliveries.last expect(delivery).to have_attributes(status: "Held", details: /Server is in development mode/i) end it "removes the queued message" do - service.call + processor.process expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) end end end context "when there are no other impediments" do + let(:send_result) do + Postal::SendResult.new do |r| + r.type = "Sent" + end + end + + before do + mocked_sender = double("SMTPSender") + allow(mocked_sender).to receive(:send_message).and_return(send_result) + allow(state).to receive(:sender_for).and_return(mocked_sender) + end + it "increments the live stats" do - expect { service.call }.to change { server.message_db.live_stats.total(60) }.from(0).to(1) + expect { processor.process }.to change { server.message_db.live_stats.total(60) }.from(0).to(1) end context "when there is an IP address assigned to the queued message" do let(:ip) { create(:ip_address) } let(:queued_message) { create(:queued_message, :locked, message: message, ip_address: ip) } - it "sends the message to the SMTP sender with the IP" do - service.call - expect(Postal::SMTPSender).to have_received(:new).with(message.recipient_domain, ip) + it "gets a sender from the state and sends the message to it" do + mocked_sender = double("SMTPSender") + expect(mocked_sender).to receive(:send_message).with(queued_message.message).and_return(send_result) + expect(state).to receive(:sender_for).with(Postal::SMTPSender, message.recipient_domain, ip).and_return(mocked_sender) + + processor.process end end context "when there is no IP address assigned to the queued message" do - it "sends the message to the SMTP sender without an IP" do - service.call - expect(Postal::SMTPSender).to have_received(:new).with(message.recipient_domain, nil) + it "gets a sender from the state and sends the message to it" do + mocked_sender = double("SMTPSender") + expect(mocked_sender).to receive(:send_message).with(queued_message.message).and_return(send_result) + expect(state).to receive(:sender_for).with(Postal::SMTPSender, message.recipient_domain, nil).and_return(mocked_sender) + + processor.process end end @@ -493,7 +406,7 @@ RSpec.describe UnqueueMessageService do context "when the recipient has got no hard fails in the last 24 hours" do it "does not add to the suppression list" do - service.call + processor.process expect(server.message_db.suppression_list.all_with_pagination(1)[:total]).to eq 0 end end @@ -508,12 +421,12 @@ RSpec.describe UnqueueMessageService do end it "logs" do - service.call + processor.process expect(logger).to have_logged(/added #{message.rcpt_to} to suppression list because 2 hard fails in 24 hours/i) end it "adds the recipient to the suppression list" do - service.call + processor.process entry = server.message_db.suppression_list.get(:recipient, message.rcpt_to) expect(entry).to match hash_including( "address" => message.rcpt_to, @@ -532,24 +445,25 @@ RSpec.describe UnqueueMessageService do end it "logs" do - service.call + processor.process expect(logger).to have_logged(/removed #{message.rcpt_to} from suppression list/) end it "removes them from the suppression list" do - service.call + processor.process expect(server.message_db.suppression_list.get(:recipient, message.rcpt_to)).to be_nil end - it "adds the details to the result details" do - service.call - expect(send_result.details).to include("Recipient removed from suppression list") + it "adds the details to the delivery details" do + processor.process + delivery = message.deliveries.last + expect(delivery.details).to include("Recipient removed from suppression list") end end it "creates a delivery with the appropriate details" do send_result.details = "Sent successfully to mx.example.com" - service.call + processor.process delivery = message.deliveries.last expect(delivery).to have_attributes(status: "Sent", details: "Sent successfully to mx.example.com") end @@ -561,19 +475,19 @@ RSpec.describe UnqueueMessageService do end it "logs" do - service.call + processor.process expect(logger).to have_logged(/message requeued for trying later/) end it "sets the message status to SoftFail" do - service.call + processor.process expect(message.reload.status).to eq "SoftFail" end it "updates the retry time on the queued message" do Timecop.freeze do retry_time = 5.minutes.from_now.change(usec: 0) - service.call + processor.process expect(queued_message.reload.retry_after).to eq retry_time end end @@ -581,20 +495,48 @@ RSpec.describe UnqueueMessageService do context "if the message should not be retried" do it "logs" do - service.call + processor.process expect(logger).to have_logged(/message processing complete/) end it "sets the message status to Sent" do - service.call + processor.process expect(message.reload.status).to eq "Sent" end it "removes the queued message" do - service.call + processor.process expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) end end end + + context "when an exception occurrs during processing" do + before do + smtp_sender_mock = double("SMTPSender") + allow(Postal::SMTPSender).to receive(:new).and_return(smtp_sender_mock) + allow(smtp_sender_mock).to receive(:start) + allow(smtp_sender_mock).to receive(:send_message) do + 1 / 0 + end + end + + it "logs" do + processor.process + expect(logger).to have_logged(/internal error: ZeroDivisionError/i) + end + + it "creates an Error delivery" do + processor.process + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Error", details: /internal error/i) + end + + it "marks the message for retrying later" do + processor.process + expect(queued_message.reload.retry_after).to be_present + end + end end + end diff --git a/spec/lib/message_dequeuer/single_message_processor_spec.rb b/spec/lib/message_dequeuer/single_message_processor_spec.rb new file mode 100644 index 0000000..fedcbca --- /dev/null +++ b/spec/lib/message_dequeuer/single_message_processor_spec.rb @@ -0,0 +1,134 @@ +# frozen_string_literal: true + +require "rails_helper" + +module MessageDequeuer + + RSpec.describe SingleMessageProcessor do + let(:server) { create(:server) } + let(:state) { State.new } + let(:logger) { TestLogger.new } + let(:route) { create(:route, server: server) } + let(:message) { MessageFactory.incoming(server, route: route) } + let(:queued_message) { create(:queued_message, :locked, message: message) } + + subject(:processor) { described_class.new(queued_message, logger: logger, state: state) } + + context "when the server is suspended" do + before do + allow(queued_message.server).to receive(:suspended?).and_return(true) + end + + it "logs" do + processor.process + expect(logger).to have_logged(/server is suspended/) + end + + it "sets the message status to Held" do + processor.process + expect(message.reload.status).to eq "Held" + end + + it "creates a Held delivery" do + processor.process + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "Held", details: /server has been suspended/i) + end + + it "removes the queued message" do + processor.process + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the number of attempts is more than the maximum" do + let(:queued_message) { create(:queued_message, :locked, message: message, attempts: Postal.config.general.maximum_delivery_attempts + 1) } + + it "logs" do + processor.process + expect(logger).to have_logged(/message has reached maximum number of attempts/) + end + + it "sends a bounce to the sender" do + expect(BounceMessage).to receive(:new).with(server, queued_message.message) + processor.process + end + + it "sets the message status to HardFail" do + processor.process + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + processor.process + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /maximum number of delivery attempts.*bounce sent to sender/i) + end + + it "removes the queued message" do + processor.process + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the message raw data has been removed" do + before do + message.raw_table = nil + message.save + end + + it "logs" do + processor.process + expect(logger).to have_logged(/raw message has been removed/) + end + + it "sets the message status to HardFail" do + processor.process + expect(message.reload.status).to eq "HardFail" + end + + it "creates a HardFail delivery" do + processor.process + delivery = message.deliveries.last + expect(delivery).to have_attributes(status: "HardFail", details: /Raw message has been removed/i) + end + + it "removes the queued message" do + processor.process + expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "when the message is incoming" do + it "calls the incoming message processor" do + expect(IncomingMessageProcessor).to receive(:new).with(queued_message, + logger: logger, + state: processor.state) + processor.process + end + + it "does not call the outgoing message processor" do + expect(OutgoingMessageProcessor).to_not receive(:process) + processor.process + end + end + + context "when the message is outgoing" do + let(:message) { MessageFactory.outgoing(server) } + + it "calls the outgoing message processor" do + expect(OutgoingMessageProcessor).to receive(:process).with(queued_message, + logger: logger, + state: processor.state) + + processor.process + end + + it "does not call the incoming message processor" do + expect(IncomingMessageProcessor).to_not receive(:process) + processor.process + end + end + end + +end diff --git a/spec/lib/message_dequeuer/state_spec.rb b/spec/lib/message_dequeuer/state_spec.rb new file mode 100644 index 0000000..3bcb608 --- /dev/null +++ b/spec/lib/message_dequeuer/state_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require "rails_helper" + +module MessageDequeuer + + RSpec.describe State do + subject(:state) { described_class.new } + + describe "#send_result" do + it "can be get and set" do + result = instance_double(Postal::SendResult) + state.send_result = result + expect(state.send_result).to be result + end + end + + describe "#sender_for" do + it "returns a instance of the given sender initialized with the args" do + sender = state.sender_for(Postal::HTTPSender, "1234") + expect(sender).to be_a Postal::HTTPSender + end + + it "returns a cached sender on subsequent calls" do + sender = state.sender_for(Postal::HTTPSender, "1234") + expect(state.sender_for(Postal::HTTPSender, "1234")).to be sender + end + end + + describe "#finished" do + it "calls finish on all cached senders" do + sender1 = state.sender_for(Postal::HTTPSender, "1234") + sender2 = state.sender_for(Postal::HTTPSender, "4444") + expect(sender1).to receive(:finish) + expect(sender2).to receive(:finish) + + state.finished + end + end + end + +end diff --git a/spec/lib/worker/jobs/process_queued_messages_job.rb b/spec/lib/worker/jobs/process_queued_messages_job_spec.rb similarity index 73% rename from spec/lib/worker/jobs/process_queued_messages_job.rb rename to spec/lib/worker/jobs/process_queued_messages_job_spec.rb index f2ce50a..46ed1ef 100644 --- a/spec/lib/worker/jobs/process_queued_messages_job.rb +++ b/spec/lib/worker/jobs/process_queued_messages_job_spec.rb @@ -7,18 +7,16 @@ module Worker RSpec.describe ProcessQueuedMessagesJob do subject(:job) { described_class.new(logger: Postal.logger) } - let(:mocked_service) { instance_double(UnqueueMessageService) } before do - allow(UnqueueMessageService).to receive(:new).and_return(mocked_service) - allow(mocked_service).to receive(:call).with(any_args) + allow(MessageDequeuer).to receive(:process) end describe "#call" do context "when there are no queued messages" do it "does nothing" do job.call - expect(UnqueueMessageService).to_not have_received(:new) + expect(MessageDequeuer).to_not have_received(:process) end end @@ -27,7 +25,7 @@ module Worker ip_address = create(:ip_address) queued_message = create(:queued_message, ip_address: ip_address) job.call - expect(UnqueueMessageService).to_not have_received(:new) + expect(MessageDequeuer).to_not have_received(:process) expect(queued_message.reload.locked?).to be false end end @@ -36,10 +34,9 @@ module Worker it "locks the message and calls the service" do queued_message = create(:queued_message, ip_address: nil, retry_after: nil) job.call - expect(UnqueueMessageService).to have_received(:new).with(logger: kind_of(Klogger::Logger), queued_message: queued_message) - expect(mocked_service).to have_received(:call) + expect(MessageDequeuer).to have_received(:process).with(queued_message, logger: kind_of(Klogger::Logger)) expect(queued_message.reload.locked?).to be true - expect(queued_message.locked_by).to eq Postal.locker_name + expect(queued_message.locked_by).to match(/\A#{Postal.locker_name} [a-f0-9]{16}\z/) expect(queued_message.locked_at).to be_within(1.second).of(Time.current) end end @@ -48,10 +45,9 @@ module Worker it "locks the message and calls the service" do queued_message = create(:queued_message, ip_address: nil, retry_after: 10.minutes.ago) job.call - expect(UnqueueMessageService).to have_received(:new).with(logger: kind_of(Klogger::Logger), queued_message: queued_message) - expect(mocked_service).to have_received(:call) + expect(MessageDequeuer).to have_received(:process).with(queued_message, logger: kind_of(Klogger::Logger)) expect(queued_message.reload.locked?).to be true - expect(queued_message.locked_by).to eq Postal.locker_name + expect(queued_message.locked_by).to match(/\A#{Postal.locker_name} [a-f0-9]{16}\z/) expect(queued_message.locked_at).to be_within(1.second).of(Time.current) end end @@ -60,7 +56,7 @@ module Worker it "does nothing" do queued_message = create(:queued_message, ip_address: nil, retry_after: 10.minutes.from_now) job.call - expect(UnqueueMessageService).to_not have_received(:new) + expect(MessageDequeuer).to_not have_received(:process) expect(queued_message.reload.locked?).to be false end end @@ -69,7 +65,7 @@ module Worker it "does nothing" do queued_message = create(:queued_message, :locked, ip_address: nil, retry_after: nil) job.call - expect(UnqueueMessageService).to_not have_received(:new) + expect(MessageDequeuer).to_not have_received(:process) expect(queued_message.reload.locked?).to be true end end @@ -78,7 +74,7 @@ module Worker it "does nothing" do queued_message = create(:queued_message, :locked, ip_address: nil, retry_after: 1.month.ago) job.call - expect(UnqueueMessageService).to_not have_received(:new) + expect(MessageDequeuer).to_not have_received(:process) expect(queued_message.reload.locked?).to be true end end @@ -89,10 +85,9 @@ module Worker allow(Socket).to receive(:ip_address_list).and_return([Addrinfo.new(["AF_INET", 1, "localhost.localdomain", "10.20.30.40"])]) queued_message = create(:queued_message, ip_address: ip_address) job.call - expect(UnqueueMessageService).to have_received(:new).with(logger: kind_of(Klogger::Logger), queued_message: queued_message) - expect(mocked_service).to have_received(:call) + expect(MessageDequeuer).to have_received(:process).with(queued_message, logger: kind_of(Klogger::Logger)) expect(queued_message.reload.locked?).to be true - expect(queued_message.locked_by).to eq Postal.locker_name + expect(queued_message.locked_by).to match(/\A#{Postal.locker_name} [a-f0-9]{16}\z/) expect(queued_message.locked_at).to be_within(1.second).of(Time.current) end end @@ -103,7 +98,7 @@ module Worker allow(Socket).to receive(:ip_address_list).and_return([Addrinfo.new(["AF_INET", 1, "localhost.localdomain", "10.20.30.40"])]) queued_message = create(:queued_message, ip_address: ip_address, retry_after: 1.month.from_now) job.call - expect(UnqueueMessageService).to_not have_received(:new) + expect(MessageDequeuer).to_not have_received(:process) expect(queued_message.reload.locked?).to be false end end diff --git a/spec/lib/worker/jobs/process_webhook_requests_job.rb b/spec/lib/worker/jobs/process_webhook_requests_job_spec.rb similarity index 72% rename from spec/lib/worker/jobs/process_webhook_requests_job.rb rename to spec/lib/worker/jobs/process_webhook_requests_job_spec.rb index 7624060..795b6a6 100644 --- a/spec/lib/worker/jobs/process_webhook_requests_job.rb +++ b/spec/lib/worker/jobs/process_webhook_requests_job_spec.rb @@ -8,8 +8,11 @@ module Worker RSpec.describe ProcessWebhookRequestsJob do subject(:job) { described_class.new(logger: Postal.logger) } + let(:mocked_service) { double("Service") } + before do - allow_any_instance_of(WebhookRequest).to receive(:deliver) + allow(WebhookDeliveryService).to receive(:new).and_return(mocked_service) + allow(mocked_service).to receive(:call).with(no_args) end context "when there are no requests to process" do @@ -21,16 +24,18 @@ module Worker context "when there is a unlocked request with no retry time" do it "delivers the request" do - create(:webhook_request) + request = create(:webhook_request) job.call + expect(WebhookDeliveryService).to have_received(:new).with(webhook_request: request) expect(job.work_completed?).to be true end end context "when there is an unlocked request with a retry time in the past" do it "delivers the request" do - create(:webhook_request, retry_after: 1.minute.ago) + request = create(:webhook_request, retry_after: 1.minute.ago) job.call + expect(WebhookDeliveryService).to have_received(:new).with(webhook_request: request) expect(job.work_completed?).to be true end end diff --git a/spec/services/unqueue_message_service/incoming_messages_spec.rb b/spec/services/unqueue_message_service/incoming_messages_spec.rb deleted file mode 100644 index e9fe122..0000000 --- a/spec/services/unqueue_message_service/incoming_messages_spec.rb +++ /dev/null @@ -1,743 +0,0 @@ -# frozen_string_literal: true - -require "rails_helper" - -RSpec.describe UnqueueMessageService do - let(:server) { create(:server) } - let(:logger) { TestLogger.new } - let(:queued_message) { create(:queued_message, server: server) } - subject(:service) { described_class.new(queued_message: queued_message, logger: logger) } - - # We're going to, for now, just stop the SMTP sender from doing anything here because - # we don't want to leak out of this test in to the real world. - before do - smtp_sender_mock = double("SMTPSender") - allow(Postal::SMTPSender).to receive(:new).and_return(smtp_sender_mock) - allow(smtp_sender_mock).to receive(:start) - allow(smtp_sender_mock).to receive(:finish) - allow(smtp_sender_mock).to receive(:send_message) do - puts "SMTP SENDING DETECTED!" - end - end - - describe "#call" do - context "for an incoming message" do - let(:route) { create(:route, server: server) } - let(:message) { MessageFactory.incoming(server, route: route) } - let(:queued_message) { create(:queued_message, :locked, message: message) } - - context "when the server is suspended" do - before do - allow(queued_message.server).to receive(:suspended?).and_return(true) - end - - it "logs" do - service.call - expect(logger).to have_logged(/server is suspended/) - end - - it "sets the message status to Held" do - service.call - expect(message.reload.status).to eq "Held" - end - - it "creates a Held delivery" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "Held", details: /server has been suspended/i) - end - - it "removes the queued message" do - service.call - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "when the number of attempts is more than the maximum" do - let(:queued_message) { create(:queued_message, :locked, message: message, attempts: Postal.config.general.maximum_delivery_attempts + 1) } - - it "logs" do - service.call - expect(logger).to have_logged(/message has reached maximum number of attempts/) - end - - it "sends a bounce to the sender" do - expect(BounceMessage).to receive(:new).with(server, queued_message.message) - service.call - end - - it "sets the message status to HardFail" do - service.call - expect(message.reload.status).to eq "HardFail" - end - - it "creates a HardFail delivery" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "HardFail", details: /maximum number of delivery attempts.*bounce sent to sender/i) - end - - it "removes the queued message" do - service.call - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "when the message raw data has been removed" do - before do - message.raw_table = nil - message.save - end - - it "logs" do - service.call - expect(logger).to have_logged(/raw message has been removed/) - end - - it "sets the message status to HardFail" do - service.call - expect(message.reload.status).to eq "HardFail" - end - - it "creates a HardFail delivery" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "HardFail", details: /Raw message has been removed/i) - end - - it "removes the queued message" do - service.call - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "when the message is a bounce for an existing message" do - let(:existing_message) { MessageFactory.outgoing(server) } - - let(:message) do - MessageFactory.incoming(server) do |msg, mail| - msg.bounce = true - mail["X-Postal-MsgID"] = existing_message.token - end - end - - it "logs" do - service.call - expect(logger).to have_logged(/message is a bounce/) - end - - it "adds the original message as the bounce ID for the received message" do - service.call - expect(message.reload.bounce_for_id).to eq existing_message.id - end - - it "sets the received message status to Processed" do - service.call - expect(message.reload.status).to eq "Processed" - end - - it "creates a Processed delivery on the received message" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "Processed", details: /This has been detected as a bounce message for /i) - end - - it "sets the existing message status to Bounced" do - service.call - expect(existing_message.reload.status).to eq "Bounced" - end - - it "creates a Bounced delivery on the original message" do - service.call - delivery = existing_message.deliveries.last - expect(delivery).to have_attributes(status: "Bounced", details: /received a bounce message for this e-mail. See for/i) - end - - it "triggers a MessageBounced webhook event" do - expect(WebhookRequest).to receive(:trigger).with(server, "MessageBounced", { - original_message: kind_of(Hash), - bounce: kind_of(Hash) - }) - service.call - end - - it "removes the queued message" do - service.call - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "when the message was a bounce but there's no return path for it" do - let(:message) do - MessageFactory.incoming(server) do |msg| - msg.bounce = true - end - end - - it "logs" do - service.call - expect(logger).to have_logged(/no source messages found, hard failing/) - end - - it "sets the message status to HardFail" do - service.call - expect(message.reload.status).to eq "HardFail" - end - - it "creates a HardFail delivery" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "HardFail", details: /was a bounce but we couldn't link it with any outgoing message/i) - end - - it "removes the queued message" do - service.call - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "when the message is not a bounce" do - it "increments the stats for the server" do - expect { service.call }.to change { server.message_db.live_stats.total(5) }.by(1) - end - - it "inspects the message and adds headers" do - expect { service.call }.to change { message.reload.inspected }.from(false).to(true) - new_message = message.reload - expect(new_message.headers).to match hash_including( - "x-postal-spam" => ["no"], - "x-postal-spam-threshold" => ["5.0"], - "x-postal-threat" => ["no"] - ) - end - - it "marks the message as spam if the spam score is higher than the server threshold" do - inspection_result = double("Result", spam_score: server.spam_threshold + 1, threat: false, threat_message: nil, spam_checks: []) - allow(Postal::MessageInspection).to receive(:scan).and_return(inspection_result) - service.call - expect(message.reload.spam).to be true - end - end - - context "when the message has a spam score greater than the server's spam failure threshold" do - before do - inspection_result = double("Result", spam_score: 100, threat: false, threat_message: nil, spam_checks: []) - allow(Postal::MessageInspection).to receive(:scan).and_return(inspection_result) - end - - it "logs" do - service.call - expect(logger).to have_logged(/message has a spam score higher than the server's maxmimum/) - end - - it "sets the message status to HardFail" do - service.call - expect(message.reload.status).to eq "HardFail" - end - - it "creates a HardFail delivery" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "HardFail", details: /spam score is higher than the failure threshold for this server/i) - end - - it "removes the queued message" do - service.call - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "when the server mode is Development and the message was not manually queued" do - before do - server.update!(mode: "Development") - end - - after do - server.update!(mode: "Live") - end - - it "logs" do - service.call - expect(logger).to have_logged(/server is in development mode/) - end - - it "sets the message status to Held" do - service.call - expect(message.reload.status).to eq "Held" - end - - it "creates a Held delivery" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "Held", details: /server is in development mode/i) - end - - it "removes the queued message" do - service.call - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "when there is no route for the incoming message" do - let(:route) { nil } - - it "logs" do - service.call - expect(logger).to have_logged(/no route and\/or endpoint available for processing/i) - end - - it "sets the message status to HardFail" do - service.call - expect(message.reload.status).to eq "HardFail" - end - - it "creates a HardFail delivery" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "HardFail", details: /does not have a route and\/or endpoint available/i) - end - - it "removes the queued message" do - service.call - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "when the route's spam mode is Quarantine, the message is spam and not manually queued" do - let(:route) { create(:route, server: server, spam_mode: "Quarantine") } - - before do - inspection_result = double("Result", spam_score: server.spam_threshold + 1, threat: false, threat_message: nil, spam_checks: []) - allow(Postal::MessageInspection).to receive(:scan).and_return(inspection_result) - end - - it "logs" do - service.call - expect(logger).to have_logged(/message is spam and route says to quarantine spam message/i) - end - - it "sets the message status to Held" do - service.call - expect(message.reload.status).to eq "Held" - end - - it "creates a Held delivery" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "Held", details: /message placed into quarantine/i) - end - - it "removes the queued message" do - service.call - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "when the route's spam mode is Fail, the message is spam and not manually queued" do - let(:route) { create(:route, server: server, spam_mode: "Fail") } - - before do - inspection_result = double("Result", spam_score: server.spam_threshold + 1, threat: false, threat_message: nil, spam_checks: []) - allow(Postal::MessageInspection).to receive(:scan).and_return(inspection_result) - end - - it "logs" do - service.call - expect(logger).to have_logged(/message is spam and route says to fail spam message/i) - end - - it "sets the message status to HardFail" do - service.call - expect(message.reload.status).to eq "HardFail" - end - - it "creates a HardFail delivery" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "HardFail", details: /message is spam and the route specifies it should be failed/i) - end - - it "removes the queued message" do - service.call - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "when the route's mode is Accept" do - it "logs" do - service.call - expect(logger).to have_logged(/route says to accept without endpoint/i) - end - - it "sets the message status to Processed" do - service.call - expect(message.reload.status).to eq "Processed" - end - - it "creates a Processed delivery" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "Processed", details: /message has been accepted but not sent to any endpoints/i) - end - - it "removes the queued message" do - service.call - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "when the route's mode is Hold" do - let(:route) { create(:route, server: server, mode: "Hold") } - - context "when the message was queued manually" do - let(:queued_message) { create(:queued_message, :locked, server: server, message: message, manual: true) } - - it "logs" do - service.call - expect(logger).to have_logged(/route says to hold and message was queued manually/i) - end - - it "sets the message status to Processed" do - service.call - expect(message.reload.status).to eq "Processed" - end - - it "creates a Processed delivery" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "Processed", details: /message has been processed/i) - end - - it "removes the queued message" do - service.call - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "when the message was not queued manually" do - let(:queued_message) { create(:queued_message, :locked, server: server, message: message, manual: false) } - - it "logs" do - service.call - expect(logger).to have_logged(/route says to hold, marking as held/i) - end - - it "sets the message status to Held" do - service.call - expect(message.reload.status).to eq "Held" - end - - it "creates a Held delivery" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "Held", details: /message has been accepted but not sent to any endpoints/i) - end - - it "removes the queued message" do - service.call - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - end - - context "when the route's mode is Bounce" do - let(:route) { create(:route, server: server, mode: "Bounce") } - - it "logs" do - service.call - expect(logger).to have_logged(/route says to bounce/i) - end - - it "sends a bounce" do - expect(BounceMessage).to receive(:new).with(server, queued_message.message) - service.call - end - - it "sets the message status to HardFail" do - service.call - expect(message.reload.status).to eq "HardFail" - end - - it "creates a HardFail delivery" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "HardFail", details: /message has been bounced because/i) - end - - it "removes the queued message" do - service.call - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "when the route's mode is Reject" do - let(:route) { create(:route, server: server, mode: "Reject") } - - it "logs" do - service.call - expect(logger).to have_logged(/route says to bounce/i) - end - - it "sends a bounce" do - expect(BounceMessage).to receive(:new).with(server, queued_message.message) - service.call - end - - it "sets the message status to HardFail" do - service.call - expect(message.reload.status).to eq "HardFail" - end - - it "creates a HardFail delivery" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "HardFail", details: /message has been bounced because/i) - end - - it "removes the queued message" do - service.call - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "when the route's endpoint is an HTTP endpoint" do - let(:endpoint) { create(:http_endpoint, server: server) } - let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) } - - it "sends the message to the HTTPSender" do - http_sender_double = double("HTTPSender") - expect(Postal::HTTPSender).to receive(:new).with(endpoint).and_return(http_sender_double) - expect(http_sender_double).to receive(:start).with(no_args) - expect(http_sender_double).to receive(:finish).with(no_args) - expect(http_sender_double).to receive(:send_message).with(queued_message.message).and_return(Postal::SendResult.new) - service.call - end - end - - context "when the route's endpoint is an SMTP endpoint" do - let(:endpoint) { create(:smtp_endpoint, server: server) } - let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) } - - it "sends the message to the SMTPSender" do - smtp_sender_double = double("SMTPSender") - expect(smtp_sender_double).to receive(:start).with(no_args) - expect(smtp_sender_double).to receive(:finish).with(no_args) - expect(smtp_sender_double).to receive(:send_message).with(queued_message.message).and_return(Postal::SendResult.new) - expect(Postal::SMTPSender).to receive(:new).with(message.recipient_domain, nil, { servers: [endpoint] }).and_return(smtp_sender_double) - service.call - end - end - - context "when the route's endpoint is an Address endpoint" do - let(:endpoint) { create(:address_endpoint, server: server) } - let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) } - - it "sends the message to the SMTPSender" do - smtp_sender_double = double("SMTPSender") - expect(smtp_sender_double).to receive(:start).with(no_args) - expect(smtp_sender_double).to receive(:finish).with(no_args) - expect(smtp_sender_double).to receive(:send_message).with(queued_message.message).and_return(Postal::SendResult.new) - expect(Postal::SMTPSender).to receive(:new).with(endpoint.domain, nil, { force_rcpt_to: endpoint.address }).and_return(smtp_sender_double) - service.call - end - end - - context "when the route's endpoint is an unknown endpoint" do - let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: create(:webhook, server: server)) } - - it "logs" do - service.call - expect(logger).to have_logged(/invalid endpoint for route/i) - end - - it "sets the message status to HardFail" do - service.call - expect(message.reload.status).to eq "HardFail" - end - - it "creates a HardFail delivery" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "HardFail", details: /invalid endpoint for route/i) - end - - it "removes the queued message" do - service.call - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "when the message has been sent to a sender" do - let(:endpoint) { create(:smtp_endpoint, server: server) } - let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) } - - let(:send_result) do - Postal::SendResult.new do |result| - result.type = "Sent" - result.details = "Sent successfully" - end - end - - before do - smtp_sender_mock = double("SMTPSender") - allow(Postal::SMTPSender).to receive(:new).and_return(smtp_sender_mock) - allow(smtp_sender_mock).to receive(:start) - allow(smtp_sender_mock).to receive(:finish) - allow(smtp_sender_mock).to receive(:send_message).and_return(send_result) - end - - context "when the sender returns a HardFail and bounces are suppressed" do - before do - send_result.type = "HardFail" - send_result.suppress_bounce = true - end - - it "logs" do - service.call - expect(logger).to have_logged(/suppressing bounce message after hard fail/) - end - - it "does not send a bounce" do - allow(BounceMessage).to receive(:new) - service.call - expect(BounceMessage).to_not have_received(:new) - end - end - - context "when the sender returns a HardFail and bounces should be sent" do - before do - send_result.type = "HardFail" - send_result.details = "Failed to send message" - end - - it "logs" do - service.call - expect(logger).to have_logged(/sending a bounce because message hard failed/) - end - - it "sends a bounce" do - expect(BounceMessage).to receive(:new).with(server, queued_message.message) - service.call - end - - it "sets the message status to HardFail" do - service.call - expect(message.reload.status).to eq "HardFail" - end - - it "creates a delivery with the details and a suffix about the bounce message" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "HardFail", details: /Failed to send message. Sent bounce message to sender \(see message \)/i) - end - end - - it "creates a delivery with the result from the sender" do - send_result.output = "some output here" - send_result.secure = true - send_result.log_id = "12345" - send_result.time = 2.32 - - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "Sent", - details: "Sent successfully", - output: "some output here", - sent_with_ssl: true, - log_id: "12345", - time: 2.32) - end - - context "when the sender wants to retry" do - before do - send_result.type = "SoftFail" - send_result.retry = true - end - - it "logs" do - service.call - expect(logger).to have_logged(/message requeued for trying later, at/i) - end - - it "sets the message status to SoftFail" do - service.call - expect(message.reload.status).to eq "SoftFail" - end - - it "updates the queued message with a new retry time" do - Timecop.freeze do - retry_time = 5.minutes.from_now.change(usec: 0) - service.call - expect(queued_message.reload.retry_after).to eq retry_time - end - end - - it "allocates a new IP address to send the message from and updates the queued message" do - expect(queued_message).to receive(:allocate_ip_address) - service.call - end - - it "does not remove the queued message" do - service.call - expect(queued_message.reload).to be_present - end - end - - context "when the sender does not want a retry" do - it "logs" do - service.call - expect(logger).to have_logged(/message processing completed/i) - end - - it "sets the message status to Sent" do - service.call - expect(message.reload.status).to eq "Sent" - end - - it "marks the endpoint as used" do - route.endpoint.update!(last_used_at: nil) - Timecop.freeze do - expect { service.call }.to change { route.endpoint.reload.last_used_at.to_i }.from(0).to(Time.now.to_i) - end - end - - it "removes the queued message" do - service.call - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - end - - context "when an exception occurrs during processing" do - let(:endpoint) { create(:smtp_endpoint, server: server) } - let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) } - - before do - smtp_sender_mock = double("SMTPSender") - allow(Postal::SMTPSender).to receive(:new).and_return(smtp_sender_mock) - allow(smtp_sender_mock).to receive(:start) - allow(smtp_sender_mock).to receive(:finish) - allow(smtp_sender_mock).to receive(:send_message) do - 1 / 0 - end - end - - it "logs" do - service.call - expect(logger).to have_logged(/internal error: ZeroDivisionError/i) - end - - it "creates an Error delivery" do - service.call - delivery = message.deliveries.last - expect(delivery).to have_attributes(status: "Error", details: /internal error/i) - end - - it "marks the message for retrying later" do - service.call - expect(queued_message.reload.retry_after).to be_present - end - end - end - end -end diff --git a/spec/services/unqueue_message_service_spec.rb b/spec/services/unqueue_message_service_spec.rb deleted file mode 100644 index 39e7027..0000000 --- a/spec/services/unqueue_message_service_spec.rb +++ /dev/null @@ -1,99 +0,0 @@ -# frozen_string_literal: true - -require "rails_helper" - -RSpec.describe UnqueueMessageService do - let(:server) { create(:server) } - let(:logger) { TestLogger.new } - let(:queued_message) { create(:queued_message, server: server) } - subject(:service) { described_class.new(queued_message: queued_message, logger: logger) } - - describe "#call" do - context "when the backend message does not exist" do - it "deletes the queued message" do - service.call - expect(logger).to have_logged(/unqueue because backend message has been removed/) - expect { queued_message.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "when the message is not ready for processing" do - let(:message) { MessageFactory.outgoing(server) } - let(:queued_message) { create(:queued_message, :retry_in_future, message: message) } - - it "does not do anything" do - service.call - expect(logger).to have_logged(/skipping because message isn't ready for processing/) - end - end - - context "when there are other messages to batch with this one" do - let(:domain) { create(:domain, server: server) } - let(:message) { MessageFactory.outgoing(server, domain: domain) } - let(:queued_message) { create(:queued_message, :locked, message: message) } - let(:send_result) { Postal::SendResult.new } - - before do - smtp_sender_mock = double("SMTPSender") - allow(Postal::SMTPSender).to receive(:new).and_return(smtp_sender_mock) - allow(smtp_sender_mock).to receive(:start) - allow(smtp_sender_mock).to receive(:finish) - allow(smtp_sender_mock).to receive(:send_message).and_return(send_result) - end - - before do - # Create 2 extra messages which are similar to the original - @message2 = MessageFactory.outgoing(server, domain: domain) - @queued_message2 = create(:queued_message, message: @message2) - @message3 = MessageFactory.outgoing(server, domain: domain) - @queued_message3 = create(:queued_message, message: @message3) - end - - it "logs" do - service.call - expect(logger).to have_logged(/found 2 associated messages/) - end - - it "sends processes each message" do - allow(service).to receive(:process_message).and_call_original - service.call - expect(service).to have_received(:process_message).with(queued_message) - expect(service).to have_received(:process_message).with(@queued_message2) - expect(service).to have_received(:process_message).with(@queued_message3) - end - - context "when there is a connect error" do - before do - send_result.type = "SoftFail" - send_result.connect_error = true - send_result.details = "Connection Error" - send_result.retry = true - end - - it "uses the same result for subsequent messages" do - service.call - expect(Postal::SMTPSender).to have_received(:new).once - expect(message.reload.status).to eq "SoftFail" - expect(@message2.reload.status).to eq "SoftFail" - expect(@message3.reload.status).to eq "SoftFail" - end - end - - context "when the backend message of a sub-message has been removed" do - before do - @message2.delete - end - - it "logs" do - service.call - expect(logger).to have_logged(/unqueueing because backend message has been removed/) - end - - it "removes the queued message for that message" do - service.call - expect { @queued_message2.reload }.to raise_error(ActiveRecord::RecordNotFound) - end - end - end - end -end diff --git a/spec/util/message_dequeuer_spec.rb b/spec/util/message_dequeuer_spec.rb new file mode 100644 index 0000000..45496ed --- /dev/null +++ b/spec/util/message_dequeuer_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require "rails_helper" + +RSpec.describe MessageDequeuer do + describe ".process" do + it "calls the initial process with the given message and logger" do + message = create(:queued_message) + logger = TestLogger.new + + mock = double("InitialProcessor") + expect(mock).to receive(:process).with(no_args) + expect(MessageDequeuer::InitialProcessor).to receive(:new).with(message, logger: logger).and_return(mock) + + described_class.process(message, logger: logger) + end + end +end From 93fc120f442f1bce46040a708a041e8ba7885b42 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Thu, 22 Feb 2024 22:27:52 +0000 Subject: [PATCH 22/56] refactor: move worker from lib/worker to app/lib/worker --- {lib => app/lib}/worker/jobs/base_job.rb | 0 {lib => app/lib}/worker/jobs/process_queued_messages_job.rb | 0 {lib => app/lib}/worker/jobs/process_webhook_requests_job.rb | 0 {lib => app/lib}/worker/process.rb | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename {lib => app/lib}/worker/jobs/base_job.rb (100%) rename {lib => app/lib}/worker/jobs/process_queued_messages_job.rb (100%) rename {lib => app/lib}/worker/jobs/process_webhook_requests_job.rb (100%) rename {lib => app/lib}/worker/process.rb (100%) diff --git a/lib/worker/jobs/base_job.rb b/app/lib/worker/jobs/base_job.rb similarity index 100% rename from lib/worker/jobs/base_job.rb rename to app/lib/worker/jobs/base_job.rb diff --git a/lib/worker/jobs/process_queued_messages_job.rb b/app/lib/worker/jobs/process_queued_messages_job.rb similarity index 100% rename from lib/worker/jobs/process_queued_messages_job.rb rename to app/lib/worker/jobs/process_queued_messages_job.rb diff --git a/lib/worker/jobs/process_webhook_requests_job.rb b/app/lib/worker/jobs/process_webhook_requests_job.rb similarity index 100% rename from lib/worker/jobs/process_webhook_requests_job.rb rename to app/lib/worker/jobs/process_webhook_requests_job.rb diff --git a/lib/worker/process.rb b/app/lib/worker/process.rb similarity index 100% rename from lib/worker/process.rb rename to app/lib/worker/process.rb From eb246bb4e70b42879846c52f84e482d7ffd52084 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Thu, 22 Feb 2024 22:28:46 +0000 Subject: [PATCH 23/56] refactor: move app/util/* to app/lib/ --- app/{util => lib}/dkim_header.rb | 0 app/{util => lib}/dns_resolver.rb | 0 app/{util => lib}/message_dequeuer.rb | 0 app/{util => lib}/query_string.rb | 0 spec/{util => lib}/dkim_header_spec.rb | 0 spec/{util => lib}/dns_resolver_spec.rb | 0 spec/{util => lib}/message_dequeuer_spec.rb | 0 spec/{util => lib}/query_string_spec.rb | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename app/{util => lib}/dkim_header.rb (100%) rename app/{util => lib}/dns_resolver.rb (100%) rename app/{util => lib}/message_dequeuer.rb (100%) rename app/{util => lib}/query_string.rb (100%) rename spec/{util => lib}/dkim_header_spec.rb (100%) rename spec/{util => lib}/dns_resolver_spec.rb (100%) rename spec/{util => lib}/message_dequeuer_spec.rb (100%) rename spec/{util => lib}/query_string_spec.rb (100%) diff --git a/app/util/dkim_header.rb b/app/lib/dkim_header.rb similarity index 100% rename from app/util/dkim_header.rb rename to app/lib/dkim_header.rb diff --git a/app/util/dns_resolver.rb b/app/lib/dns_resolver.rb similarity index 100% rename from app/util/dns_resolver.rb rename to app/lib/dns_resolver.rb diff --git a/app/util/message_dequeuer.rb b/app/lib/message_dequeuer.rb similarity index 100% rename from app/util/message_dequeuer.rb rename to app/lib/message_dequeuer.rb diff --git a/app/util/query_string.rb b/app/lib/query_string.rb similarity index 100% rename from app/util/query_string.rb rename to app/lib/query_string.rb diff --git a/spec/util/dkim_header_spec.rb b/spec/lib/dkim_header_spec.rb similarity index 100% rename from spec/util/dkim_header_spec.rb rename to spec/lib/dkim_header_spec.rb diff --git a/spec/util/dns_resolver_spec.rb b/spec/lib/dns_resolver_spec.rb similarity index 100% rename from spec/util/dns_resolver_spec.rb rename to spec/lib/dns_resolver_spec.rb diff --git a/spec/util/message_dequeuer_spec.rb b/spec/lib/message_dequeuer_spec.rb similarity index 100% rename from spec/util/message_dequeuer_spec.rb rename to spec/lib/message_dequeuer_spec.rb diff --git a/spec/util/query_string_spec.rb b/spec/lib/query_string_spec.rb similarity index 100% rename from spec/util/query_string_spec.rb rename to spec/lib/query_string_spec.rb From 73a55a5053b871cd0ca923aace208617342c5f55 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Thu, 22 Feb 2024 22:33:30 +0000 Subject: [PATCH 24/56] refactor: move senders in to app/senders/ --- app/lib/dns_resolver.rb | 2 + .../incoming_message_processor.rb | 6 +- .../outgoing_message_processor.rb | 2 +- app/senders/base_sender.rb | 14 + app/senders/http_sender.rb | 134 ++++++++ app/senders/send_result.rb | 20 ++ app/senders/smtp_sender.rb | 291 +++++++++++++++++ lib/postal/http_sender.rb | 136 -------- lib/postal/send_result.rb | 22 -- lib/postal/sender.rb | 16 - lib/postal/smtp_sender.rb | 295 ------------------ .../incoming_message_processor_spec.rb | 18 +- .../outgoing_message_processor_spec.rb | 8 +- spec/lib/message_dequeuer/state_spec.rb | 14 +- 14 files changed, 485 insertions(+), 493 deletions(-) create mode 100644 app/senders/base_sender.rb create mode 100644 app/senders/http_sender.rb create mode 100644 app/senders/send_result.rb create mode 100644 app/senders/smtp_sender.rb delete mode 100644 lib/postal/http_sender.rb delete mode 100644 lib/postal/send_result.rb delete mode 100644 lib/postal/sender.rb delete mode 100644 lib/postal/smtp_sender.rb diff --git a/app/lib/dns_resolver.rb b/app/lib/dns_resolver.rb index 0526584..366ddb1 100644 --- a/app/lib/dns_resolver.rb +++ b/app/lib/dns_resolver.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require "resolv" + class DNSResolver attr_reader :nameservers diff --git a/app/lib/message_dequeuer/incoming_message_processor.rb b/app/lib/message_dequeuer/incoming_message_processor.rb index d42a1dc..0d1881b 100644 --- a/app/lib/message_dequeuer/incoming_message_processor.rb +++ b/app/lib/message_dequeuer/incoming_message_processor.rb @@ -163,11 +163,11 @@ module MessageDequeuer case queued_message.message.endpoint when SMTPEndpoint - sender = @state.sender_for(Postal::SMTPSender, queued_message.message.recipient_domain, nil, servers: [queued_message.message.endpoint]) + sender = @state.sender_for(SMTPSender, queued_message.message.recipient_domain, nil, servers: [queued_message.message.endpoint]) when HTTPEndpoint - sender = @state.sender_for(Postal::HTTPSender, queued_message.message.endpoint) + sender = @state.sender_for(HTTPSender, queued_message.message.endpoint) when AddressEndpoint - sender = @state.sender_for(Postal::SMTPSender, queued_message.message.endpoint.domain, nil, force_rcpt_to: queued_message.message.endpoint.address) + sender = @state.sender_for(SMTPSender, queued_message.message.endpoint.domain, nil, force_rcpt_to: queued_message.message.endpoint.address) else log "invalid endpoint for route (#{queued_message.message.endpoint_type})" create_delivery "HardFail", details: "Invalid endpoint for route." diff --git a/app/lib/message_dequeuer/outgoing_message_processor.rb b/app/lib/message_dequeuer/outgoing_message_processor.rb index af27c8f..e930b33 100644 --- a/app/lib/message_dequeuer/outgoing_message_processor.rb +++ b/app/lib/message_dequeuer/outgoing_message_processor.rb @@ -135,7 +135,7 @@ module MessageDequeuer @result = @state.send_result return if @result - sender = @state.sender_for(Postal::SMTPSender, + sender = @state.sender_for(SMTPSender, queued_message.message.recipient_domain, queued_message.ip_address) diff --git a/app/senders/base_sender.rb b/app/senders/base_sender.rb new file mode 100644 index 0000000..a009e9f --- /dev/null +++ b/app/senders/base_sender.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class BaseSender + + def start + end + + def send_message(message) + end + + def finish + end + +end diff --git a/app/senders/http_sender.rb b/app/senders/http_sender.rb new file mode 100644 index 0000000..af931e9 --- /dev/null +++ b/app/senders/http_sender.rb @@ -0,0 +1,134 @@ +# frozen_string_literal: true + +class HTTPSender < BaseSender + + def initialize(endpoint, options = {}) + super() + @endpoint = endpoint + @options = options + @log_id = Nifty::Utils::RandomString.generate(length: 8).upcase + end + + def send_message(message) + start_time = Time.now + result = SendResult.new + result.log_id = @log_id + + request_options = {} + request_options[:sign] = true + request_options[:timeout] = @endpoint.timeout || 5 + case @endpoint.encoding + when "BodyAsJSON" + request_options[:json] = parameters(message, flat: false).to_json + when "FormData" + request_options[:params] = parameters(message, flat: true) + end + + log "Sending request to #{@endpoint.url}" + response = Postal::HTTP.post(@endpoint.url, request_options) + result.secure = !!response[:secure] # rubocop:disable Style/DoubleNegation + result.details = "Received a #{response[:code]} from #{@endpoint.url}" + log " -> Received: #{response[:code]}" + if response[:body] + log " -> Body: #{response[:body][0, 255]}" + result.output = response[:body].to_s[0, 500].strip + end + if response[:code] >= 200 && response[:code] < 300 + # This is considered a success + result.type = "Sent" + elsif response[:code] >= 500 && response[:code] < 600 + # This is temporary. They might fix their server so it should soft fail. + result.type = "SoftFail" + result.retry = true + elsif response[:code].negative? + # Connection/SSL etc... errors + result.type = "SoftFail" + result.retry = true + result.connect_error = true + elsif response[:code] == 429 + # Rate limit exceeded, treat as a hard fail and don't send bounces + result.type = "HardFail" + result.suppress_bounce = true + else + # This is permanent. Any other error isn't cool with us. + result.type = "HardFail" + end + result.time = (Time.now - start_time).to_f.round(2) + result + end + + private + + def log(text) + Postal.logger.info text, id: @log_id, component: "http-sender" + end + + def parameters(message, options = {}) + case @endpoint.format + when "Hash" + hash = { + id: message.id, + rcpt_to: message.rcpt_to, + mail_from: message.mail_from, + token: message.token, + subject: message.subject, + message_id: message.message_id, + timestamp: message.timestamp.to_f, + size: message.size, + spam_status: message.spam_status, + bounce: message.bounce, + received_with_ssl: message.received_with_ssl, + to: message.headers["to"]&.last, + cc: message.headers["cc"]&.last, + from: message.headers["from"]&.last, + date: message.headers["date"]&.last, + in_reply_to: message.headers["in-reply-to"]&.last, + references: message.headers["references"]&.last, + html_body: message.html_body, + attachment_quantity: message.attachments.size, + auto_submitted: message.headers["auto-submitted"]&.last, + reply_to: message.headers["reply-to"] + } + + if @endpoint.strip_replies + hash[:plain_body], hash[:replies_from_plain_body] = Postal::ReplySeparator.separate(message.plain_body) + else + hash[:plain_body] = message.plain_body + end + + if @endpoint.include_attachments? + if options[:flat] + message.attachments.each_with_index do |a, i| + hash["attachments[#{i}][filename]"] = a.filename + hash["attachments[#{i}][content_type]"] = a.content_type + hash["attachments[#{i}][size]"] = a.body.to_s.bytesize.to_s + hash["attachments[#{i}][data]"] = Base64.encode64(a.body.to_s) + end + else + hash[:attachments] = message.attachments.map do |a| + { + filename: a.filename, + content_type: a.mime_type, + size: a.body.to_s.bytesize, + data: Base64.encode64(a.body.to_s) + } + end + end + end + + hash + when "RawMessage" + { + id: message.id, + rcpt_to: message.rcpt_to, + mail_from: message.mail_from, + message: Base64.encode64(message.raw_message), + base64: true, + size: message.size.to_i + } + else + {} + end + end + +end diff --git a/app/senders/send_result.rb b/app/senders/send_result.rb new file mode 100644 index 0000000..c8a6353 --- /dev/null +++ b/app/senders/send_result.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class SendResult + + attr_accessor :type + attr_accessor :details + attr_accessor :retry + attr_accessor :output + attr_accessor :secure + attr_accessor :connect_error + attr_accessor :log_id + attr_accessor :time + attr_accessor :suppress_bounce + + def initialize + @details = "" + yield self if block_given? + end + +end diff --git a/app/senders/smtp_sender.rb b/app/senders/smtp_sender.rb new file mode 100644 index 0000000..bb6c124 --- /dev/null +++ b/app/senders/smtp_sender.rb @@ -0,0 +1,291 @@ +# frozen_string_literal: true + +class SMTPSender < BaseSender + + def initialize(domain, source_ip_address, options = {}) + super() + @domain = domain + @source_ip_address = source_ip_address + @options = options + @smtp_client = nil + @connection_errors = [] + @hostnames = [] + @log_id = Nifty::Utils::RandomString.generate(length: 8).upcase + end + + def start + servers.each do |server| + if server.is_a?(SMTPEndpoint) + hostname = server.hostname + port = server.port || 25 + ssl_mode = server.ssl_mode + elsif server.is_a?(Hash) + hostname = server[:hostname] + port = server[:port] || 25 + ssl_mode = server[:ssl_mode] || "Auto" + else + hostname = server + port = 25 + ssl_mode = "Auto" + end + + @hostnames << hostname + [:aaaa, :a].each do |ip_type| + if @source_ip_address && @source_ip_address.ipv6.blank? && ip_type == :aaaa + # Don't try to use IPv6 if the IP address we're sending from doesn't support it. + next + end + + begin + @remote_ip = lookup_ip_address(ip_type, hostname) + if @remote_ip.nil? + if ip_type == :a + # As we can't resolve the last IP, we'll put this + @connection_errors << "Could not resolve #{hostname}" + end + next + end + + smtp_client = Net::SMTP.new(@remote_ip, port) + smtp_client.open_timeout = Postal.config.smtp_client.open_timeout + smtp_client.read_timeout = Postal.config.smtp_client.read_timeout + smtp_client.tls_hostname = hostname + + if @source_ip_address + # Set the source IP as appropriate + smtp_client.source_address = ip_type == :aaaa ? @source_ip_address.ipv6 : @source_ip_address.ipv4 + end + + case ssl_mode + when "Auto" + smtp_client.enable_starttls_auto(self.class.ssl_context_without_verify) + when "STARTTLS" + smtp_client.enable_starttls(self.class.ssl_context_with_verify) + when "TLS" + smtp_client.enable_tls(self.class.ssl_context_with_verify) + else + smtp_client.disable_starttls + smtp_client.disable_tls + end + + smtp_client.start(@source_ip_address ? @source_ip_address.hostname : self.class.default_helo_hostname) + log "Connected to #{@remote_ip}:#{port} (#{hostname})" + rescue StandardError => e + if e.is_a?(OpenSSL::SSL::SSLError) && ssl_mode == "Auto" + log "SSL error (#{e.message}), retrying without SSL" + ssl_mode = nil + retry + end + + log "Cannot connect to #{@remote_ip}:#{port} (#{hostname}) (#{e.class}: #{e.message})" + @connection_errors << e.message unless @connection_errors.include?(e.message) + begin + smtp_client.finish + rescue StandardError + nil + end + smtp_client = nil + end + + if smtp_client + @smtp_client = smtp_client + return true + end + end + end + + @connection_errors + end + + def reconnect + log "Reconnecting" + begin + @smtp_client&.finish + rescue StandardError + nil + end + start + end + + def safe_rset + # Something went wrong sending the last email. Reset the connection if possible, else disconnect. + + @smtp_client.rset + rescue StandardError + # Don't reconnect, this would be rather rude if we don't have any more emails to send. + begin + @smtp_client.finish + rescue StandardError + nil + end + end + + def send_message(message, force_rcpt_to = nil) + start_time = Time.now + result = SendResult.new + result.log_id = @log_id + if @smtp_client && !@smtp_client.started? + # For some reason we had an SMTP connection but it's no longer connected. + # Make a new one. + start + end + + if @smtp_client + result.secure = @smtp_client.secure_socket? + end + + begin + if message.bounce + mail_from = "" + elsif message.domain.return_path_status == "OK" + mail_from = "#{message.server.token}@#{message.domain.return_path_domain}" + else + mail_from = "#{message.server.token}@#{Postal.config.dns.return_path}" + end + if Postal.config.general.use_resent_sender_header + raw_message = "Resent-Sender: #{mail_from}\r\n" + message.raw_message + else + raw_message = message.raw_message + end + tries = 0 + begin + if @smtp_client.nil? + log "-> No SMTP server available for #{@domain}" + log "-> Hostnames: #{@hostnames.inspect}" + log "-> Errors: #{@connection_errors.inspect}" + result.type = "SoftFail" + result.retry = true + result.details = "No SMTP servers were available for #{@domain}. Tried #{@hostnames.to_sentence}" + result.output = @connection_errors.join(", ") + result.connect_error = true + return result + else + @smtp_client.rset_errors + rcpt_to = force_rcpt_to || @options[:force_rcpt_to] || message.rcpt_to + log "Sending message #{message.server.id}::#{message.id} to #{rcpt_to}" + smtp_result = @smtp_client.send_message(raw_message, mail_from, [rcpt_to]) + end + rescue Errno::ECONNRESET, Errno::EPIPE, OpenSSL::SSL::SSLError + raise unless (tries += 1) < 2 + + reconnect + retry + end + result.type = "Sent" + result.details = "Message for #{rcpt_to} accepted by #{destination_host_description}" + if @smtp_client.source_address + result.details += " (from #{@smtp_client.source_address})" + end + result.output = smtp_result.string + log "Message sent ##{message.id} to #{destination_host_description} for #{rcpt_to}" + rescue Net::SMTPServerBusy, Net::SMTPAuthenticationError, Net::SMTPSyntaxError, Net::SMTPUnknownError, Net::ReadTimeout => e + log "#{e.class}: #{e.message}" + result.type = "SoftFail" + result.retry = true + result.details = "Temporary SMTP delivery error when sending to #{destination_host_description}" + result.output = e.message + if e.to_s =~ /(\d+) seconds/ + result.retry = ::Regexp.last_match(1).to_i + 10 + elsif e.to_s =~ /(\d+) minutes/ + result.retry = (::Regexp.last_match(1).to_i * 60) + 10 + end + + safe_rset + rescue Net::SMTPFatalError => e + log "#{e.class}: #{e.message}" + result.type = "HardFail" + result.details = "Permanent SMTP delivery error when sending to #{destination_host_description}" + result.output = e.message + safe_rset + rescue StandardError => e + log "#{e.class}: #{e.message}" + if defined?(Sentry) + Sentry.capture_exception(e, extra: { log_id: @log_id, server_id: message.server.id, message_id: message.id }) + end + result.type = "SoftFail" + result.retry = true + result.details = "An error occurred while sending the message to #{destination_host_description}" + result.output = e.message + safe_rset + end + + result.time = (Time.now - start_time).to_f.round(2) + result + end + + def finish + log "Finishing up" + @smtp_client&.finish + end + + private + + def servers + @options[:servers] || self.class.relay_hosts || @servers ||= begin + mx_servers = DNSResolver.local.mx(@domain).map(&:last) + if mx_servers.empty? + mx_servers = [@domain] # This will be resolved to an A or AAAA record later + end + mx_servers + end + end + + def log(text) + Postal.logger.info text, id: @log_id, component: "smtp-sender" + end + + def destination_host_description + "#{@hostnames.last} (#{@remote_ip})" + end + + def lookup_ip_address(type, hostname) + records = [] + case type + when :a + records = DNSResolver.local.a(hostname) + when :aaaa + records = DNSResolver.local.aaaa(hostname) + end + records.first&.to_s&.downcase + end + + class << self + + def ssl_context_with_verify + @ssl_context_with_verify ||= begin + c = OpenSSL::SSL::SSLContext.new + c.verify_mode = OpenSSL::SSL::VERIFY_PEER + c.cert_store = OpenSSL::X509::Store.new + c.cert_store.set_default_paths + c + end + end + + def ssl_context_without_verify + @ssl_context_without_verify ||= begin + c = OpenSSL::SSL::SSLContext.new + c.verify_mode = OpenSSL::SSL::VERIFY_NONE + c + end + end + + def default_helo_hostname + Postal.config.dns.helo_hostname || Postal.config.dns.smtp_server_hostname || "localhost" + end + + def relay_hosts + hosts = Postal.config.smtp_relays.map do |relay| + next unless relay.hostname.present? + + { + hostname: relay.hostname, + port: relay.port, + ssl_mode: relay.ssl_mode + } + end.compact + hosts.empty? ? nil : hosts + end + + end + +end diff --git a/lib/postal/http_sender.rb b/lib/postal/http_sender.rb deleted file mode 100644 index 69011dd..0000000 --- a/lib/postal/http_sender.rb +++ /dev/null @@ -1,136 +0,0 @@ -# frozen_string_literal: true - -module Postal - class HTTPSender < Sender - - def initialize(endpoint, options = {}) - super() - @endpoint = endpoint - @options = options - @log_id = Nifty::Utils::RandomString.generate(length: 8).upcase - end - - def send_message(message) - start_time = Time.now - result = SendResult.new - result.log_id = @log_id - - request_options = {} - request_options[:sign] = true - request_options[:timeout] = @endpoint.timeout || 5 - case @endpoint.encoding - when "BodyAsJSON" - request_options[:json] = parameters(message, flat: false).to_json - when "FormData" - request_options[:params] = parameters(message, flat: true) - end - - log "Sending request to #{@endpoint.url}" - response = Postal::HTTP.post(@endpoint.url, request_options) - result.secure = !!response[:secure] # rubocop:disable Style/DoubleNegation - result.details = "Received a #{response[:code]} from #{@endpoint.url}" - log " -> Received: #{response[:code]}" - if response[:body] - log " -> Body: #{response[:body][0, 255]}" - result.output = response[:body].to_s[0, 500].strip - end - if response[:code] >= 200 && response[:code] < 300 - # This is considered a success - result.type = "Sent" - elsif response[:code] >= 500 && response[:code] < 600 - # This is temporary. They might fix their server so it should soft fail. - result.type = "SoftFail" - result.retry = true - elsif response[:code].negative? - # Connection/SSL etc... errors - result.type = "SoftFail" - result.retry = true - result.connect_error = true - elsif response[:code] == 429 - # Rate limit exceeded, treat as a hard fail and don't send bounces - result.type = "HardFail" - result.suppress_bounce = true - else - # This is permanent. Any other error isn't cool with us. - result.type = "HardFail" - end - result.time = (Time.now - start_time).to_f.round(2) - result - end - - private - - def log(text) - Postal.logger.info text, id: @log_id, component: "http-sender" - end - - def parameters(message, options = {}) - case @endpoint.format - when "Hash" - hash = { - id: message.id, - rcpt_to: message.rcpt_to, - mail_from: message.mail_from, - token: message.token, - subject: message.subject, - message_id: message.message_id, - timestamp: message.timestamp.to_f, - size: message.size, - spam_status: message.spam_status, - bounce: message.bounce, - received_with_ssl: message.received_with_ssl, - to: message.headers["to"]&.last, - cc: message.headers["cc"]&.last, - from: message.headers["from"]&.last, - date: message.headers["date"]&.last, - in_reply_to: message.headers["in-reply-to"]&.last, - references: message.headers["references"]&.last, - html_body: message.html_body, - attachment_quantity: message.attachments.size, - auto_submitted: message.headers["auto-submitted"]&.last, - reply_to: message.headers["reply-to"] - } - - if @endpoint.strip_replies - hash[:plain_body], hash[:replies_from_plain_body] = Postal::ReplySeparator.separate(message.plain_body) - else - hash[:plain_body] = message.plain_body - end - - if @endpoint.include_attachments? - if options[:flat] - message.attachments.each_with_index do |a, i| - hash["attachments[#{i}][filename]"] = a.filename - hash["attachments[#{i}][content_type]"] = a.content_type - hash["attachments[#{i}][size]"] = a.body.to_s.bytesize.to_s - hash["attachments[#{i}][data]"] = Base64.encode64(a.body.to_s) - end - else - hash[:attachments] = message.attachments.map do |a| - { - filename: a.filename, - content_type: a.mime_type, - size: a.body.to_s.bytesize, - data: Base64.encode64(a.body.to_s) - } - end - end - end - - hash - when "RawMessage" - { - id: message.id, - rcpt_to: message.rcpt_to, - mail_from: message.mail_from, - message: Base64.encode64(message.raw_message), - base64: true, - size: message.size.to_i - } - else - {} - end - end - - end -end diff --git a/lib/postal/send_result.rb b/lib/postal/send_result.rb deleted file mode 100644 index b0505ba..0000000 --- a/lib/postal/send_result.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Postal - class SendResult - - attr_accessor :type - attr_accessor :details - attr_accessor :retry - attr_accessor :output - attr_accessor :secure - attr_accessor :connect_error - attr_accessor :log_id - attr_accessor :time - attr_accessor :suppress_bounce - - def initialize - @details = "" - yield self if block_given? - end - - end -end diff --git a/lib/postal/sender.rb b/lib/postal/sender.rb deleted file mode 100644 index ff5cd49..0000000 --- a/lib/postal/sender.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -module Postal - class Sender - - def start - end - - def send_message(message) - end - - def finish - end - - end -end diff --git a/lib/postal/smtp_sender.rb b/lib/postal/smtp_sender.rb deleted file mode 100644 index 2857c30..0000000 --- a/lib/postal/smtp_sender.rb +++ /dev/null @@ -1,295 +0,0 @@ -# frozen_string_literal: true - -require "resolv" - -module Postal - class SMTPSender < Sender - - def initialize(domain, source_ip_address, options = {}) - super() - @domain = domain - @source_ip_address = source_ip_address - @options = options - @smtp_client = nil - @connection_errors = [] - @hostnames = [] - @log_id = Nifty::Utils::RandomString.generate(length: 8).upcase - end - - def start - servers.each do |server| - if server.is_a?(SMTPEndpoint) - hostname = server.hostname - port = server.port || 25 - ssl_mode = server.ssl_mode - elsif server.is_a?(Hash) - hostname = server[:hostname] - port = server[:port] || 25 - ssl_mode = server[:ssl_mode] || "Auto" - else - hostname = server - port = 25 - ssl_mode = "Auto" - end - - @hostnames << hostname - [:aaaa, :a].each do |ip_type| - if @source_ip_address && @source_ip_address.ipv6.blank? && ip_type == :aaaa - # Don't try to use IPv6 if the IP address we're sending from doesn't support it. - next - end - - begin - @remote_ip = lookup_ip_address(ip_type, hostname) - if @remote_ip.nil? - if ip_type == :a - # As we can't resolve the last IP, we'll put this - @connection_errors << "Could not resolve #{hostname}" - end - next - end - - smtp_client = Net::SMTP.new(@remote_ip, port) - smtp_client.open_timeout = Postal.config.smtp_client.open_timeout - smtp_client.read_timeout = Postal.config.smtp_client.read_timeout - smtp_client.tls_hostname = hostname - - if @source_ip_address - # Set the source IP as appropriate - smtp_client.source_address = ip_type == :aaaa ? @source_ip_address.ipv6 : @source_ip_address.ipv4 - end - - case ssl_mode - when "Auto" - smtp_client.enable_starttls_auto(self.class.ssl_context_without_verify) - when "STARTTLS" - smtp_client.enable_starttls(self.class.ssl_context_with_verify) - when "TLS" - smtp_client.enable_tls(self.class.ssl_context_with_verify) - else - smtp_client.disable_starttls - smtp_client.disable_tls - end - - smtp_client.start(@source_ip_address ? @source_ip_address.hostname : self.class.default_helo_hostname) - log "Connected to #{@remote_ip}:#{port} (#{hostname})" - rescue StandardError => e - if e.is_a?(OpenSSL::SSL::SSLError) && ssl_mode == "Auto" - log "SSL error (#{e.message}), retrying without SSL" - ssl_mode = nil - retry - end - - log "Cannot connect to #{@remote_ip}:#{port} (#{hostname}) (#{e.class}: #{e.message})" - @connection_errors << e.message unless @connection_errors.include?(e.message) - begin - smtp_client.finish - rescue StandardError - nil - end - smtp_client = nil - end - - if smtp_client - @smtp_client = smtp_client - return true - end - end - end - - @connection_errors - end - - def reconnect - log "Reconnecting" - begin - @smtp_client&.finish - rescue StandardError - nil - end - start - end - - def safe_rset - # Something went wrong sending the last email. Reset the connection if possible, else disconnect. - - @smtp_client.rset - rescue StandardError - # Don't reconnect, this would be rather rude if we don't have any more emails to send. - begin - @smtp_client.finish - rescue StandardError - nil - end - end - - def send_message(message, force_rcpt_to = nil) - start_time = Time.now - result = SendResult.new - result.log_id = @log_id - if @smtp_client && !@smtp_client.started? - # For some reason we had an SMTP connection but it's no longer connected. - # Make a new one. - start - end - - if @smtp_client - result.secure = @smtp_client.secure_socket? - end - - begin - if message.bounce - mail_from = "" - elsif message.domain.return_path_status == "OK" - mail_from = "#{message.server.token}@#{message.domain.return_path_domain}" - else - mail_from = "#{message.server.token}@#{Postal.config.dns.return_path}" - end - if Postal.config.general.use_resent_sender_header - raw_message = "Resent-Sender: #{mail_from}\r\n" + message.raw_message - else - raw_message = message.raw_message - end - tries = 0 - begin - if @smtp_client.nil? - log "-> No SMTP server available for #{@domain}" - log "-> Hostnames: #{@hostnames.inspect}" - log "-> Errors: #{@connection_errors.inspect}" - result.type = "SoftFail" - result.retry = true - result.details = "No SMTP servers were available for #{@domain}. Tried #{@hostnames.to_sentence}" - result.output = @connection_errors.join(", ") - result.connect_error = true - return result - else - @smtp_client.rset_errors - rcpt_to = force_rcpt_to || @options[:force_rcpt_to] || message.rcpt_to - log "Sending message #{message.server.id}::#{message.id} to #{rcpt_to}" - smtp_result = @smtp_client.send_message(raw_message, mail_from, [rcpt_to]) - end - rescue Errno::ECONNRESET, Errno::EPIPE, OpenSSL::SSL::SSLError - raise unless (tries += 1) < 2 - - reconnect - retry - end - result.type = "Sent" - result.details = "Message for #{rcpt_to} accepted by #{destination_host_description}" - if @smtp_client.source_address - result.details += " (from #{@smtp_client.source_address})" - end - result.output = smtp_result.string - log "Message sent ##{message.id} to #{destination_host_description} for #{rcpt_to}" - rescue Net::SMTPServerBusy, Net::SMTPAuthenticationError, Net::SMTPSyntaxError, Net::SMTPUnknownError, Net::ReadTimeout => e - log "#{e.class}: #{e.message}" - result.type = "SoftFail" - result.retry = true - result.details = "Temporary SMTP delivery error when sending to #{destination_host_description}" - result.output = e.message - if e.to_s =~ /(\d+) seconds/ - result.retry = ::Regexp.last_match(1).to_i + 10 - elsif e.to_s =~ /(\d+) minutes/ - result.retry = (::Regexp.last_match(1).to_i * 60) + 10 - end - - safe_rset - rescue Net::SMTPFatalError => e - log "#{e.class}: #{e.message}" - result.type = "HardFail" - result.details = "Permanent SMTP delivery error when sending to #{destination_host_description}" - result.output = e.message - safe_rset - rescue StandardError => e - log "#{e.class}: #{e.message}" - if defined?(Sentry) - Sentry.capture_exception(e, extra: { log_id: @log_id, server_id: message.server.id, message_id: message.id }) - end - result.type = "SoftFail" - result.retry = true - result.details = "An error occurred while sending the message to #{destination_host_description}" - result.output = e.message - safe_rset - end - - result.time = (Time.now - start_time).to_f.round(2) - result - end - - def finish - log "Finishing up" - @smtp_client&.finish - end - - private - - def servers - @options[:servers] || self.class.relay_hosts || @servers ||= begin - mx_servers = DNSResolver.local.mx(@domain).map(&:last) - if mx_servers.empty? - mx_servers = [@domain] # This will be resolved to an A or AAAA record later - end - mx_servers - end - end - - def log(text) - Postal.logger.info text, id: @log_id, component: "smtp-sender" - end - - def destination_host_description - "#{@hostnames.last} (#{@remote_ip})" - end - - def lookup_ip_address(type, hostname) - records = [] - case type - when :a - records = DNSResolver.local.a(hostname) - when :aaaa - records = DNSResolver.local.aaaa(hostname) - end - records.first&.to_s&.downcase - end - - class << self - - def ssl_context_with_verify - @ssl_context_with_verify ||= begin - c = OpenSSL::SSL::SSLContext.new - c.verify_mode = OpenSSL::SSL::VERIFY_PEER - c.cert_store = OpenSSL::X509::Store.new - c.cert_store.set_default_paths - c - end - end - - def ssl_context_without_verify - @ssl_context_without_verify ||= begin - c = OpenSSL::SSL::SSLContext.new - c.verify_mode = OpenSSL::SSL::VERIFY_NONE - c - end - end - - def default_helo_hostname - Postal.config.dns.helo_hostname || Postal.config.dns.smtp_server_hostname || "localhost" - end - - def relay_hosts - hosts = Postal.config.smtp_relays.map do |relay| - next unless relay.hostname.present? - - { - hostname: relay.hostname, - port: relay.port, - ssl_mode: relay.ssl_mode - } - end.compact - hosts.empty? ? nil : hosts - end - - end - - end -end diff --git a/spec/lib/message_dequeuer/incoming_message_processor_spec.rb b/spec/lib/message_dequeuer/incoming_message_processor_spec.rb index 5ac111a..9cd5f81 100644 --- a/spec/lib/message_dequeuer/incoming_message_processor_spec.rb +++ b/spec/lib/message_dequeuer/incoming_message_processor_spec.rb @@ -409,8 +409,8 @@ module MessageDequeuer it "gets a sender from the state and sends the message to it" do http_sender_double = double("HTTPSender") - expect(http_sender_double).to receive(:send_message).with(queued_message.message).and_return(Postal::SendResult.new) - expect(state).to receive(:sender_for).with(Postal::HTTPSender, endpoint).and_return(http_sender_double) + expect(http_sender_double).to receive(:send_message).with(queued_message.message).and_return(SendResult.new) + expect(state).to receive(:sender_for).with(HTTPSender, endpoint).and_return(http_sender_double) processor.process end end @@ -421,8 +421,8 @@ module MessageDequeuer it "gets a sender from the state and sends the message to it" do smtp_sender_double = double("SMTPSender") - expect(smtp_sender_double).to receive(:send_message).with(queued_message.message).and_return(Postal::SendResult.new) - expect(state).to receive(:sender_for).with(Postal::SMTPSender, message.recipient_domain, nil, { servers: [endpoint] }).and_return(smtp_sender_double) + expect(smtp_sender_double).to receive(:send_message).with(queued_message.message).and_return(SendResult.new) + expect(state).to receive(:sender_for).with(SMTPSender, message.recipient_domain, nil, { servers: [endpoint] }).and_return(smtp_sender_double) processor.process end end @@ -433,8 +433,8 @@ module MessageDequeuer it "gets a sender from the state and sends the message to it" do smtp_sender_double = double("SMTPSender") - expect(smtp_sender_double).to receive(:send_message).with(queued_message.message).and_return(Postal::SendResult.new) - expect(state).to receive(:sender_for).with(Postal::SMTPSender, endpoint.domain, nil, { force_rcpt_to: endpoint.address }).and_return(smtp_sender_double) + expect(smtp_sender_double).to receive(:send_message).with(queued_message.message).and_return(SendResult.new) + expect(state).to receive(:sender_for).with(SMTPSender, endpoint.domain, nil, { force_rcpt_to: endpoint.address }).and_return(smtp_sender_double) processor.process end end @@ -469,7 +469,7 @@ module MessageDequeuer let(:route) { create(:route, server: server, mode: "Endpoint", endpoint: endpoint) } let(:send_result) do - Postal::SendResult.new do |result| + SendResult.new do |result| result.type = "Sent" result.details = "Sent successfully" end @@ -477,7 +477,7 @@ module MessageDequeuer before do smtp_sender_mock = double("SMTPSender") - allow(Postal::SMTPSender).to receive(:new).and_return(smtp_sender_mock) + allow(SMTPSender).to receive(:new).and_return(smtp_sender_mock) allow(smtp_sender_mock).to receive(:start) allow(smtp_sender_mock).to receive(:finish) allow(smtp_sender_mock).to receive(:send_message).and_return(send_result) @@ -611,7 +611,7 @@ module MessageDequeuer before do smtp_sender_mock = double("SMTPSender") - allow(Postal::SMTPSender).to receive(:new).and_return(smtp_sender_mock) + allow(SMTPSender).to receive(:new).and_return(smtp_sender_mock) allow(smtp_sender_mock).to receive(:start) allow(smtp_sender_mock).to receive(:finish) allow(smtp_sender_mock).to receive(:send_message) do diff --git a/spec/lib/message_dequeuer/outgoing_message_processor_spec.rb b/spec/lib/message_dequeuer/outgoing_message_processor_spec.rb index a455175..e070a17 100644 --- a/spec/lib/message_dequeuer/outgoing_message_processor_spec.rb +++ b/spec/lib/message_dequeuer/outgoing_message_processor_spec.rb @@ -361,7 +361,7 @@ module MessageDequeuer context "when there are no other impediments" do let(:send_result) do - Postal::SendResult.new do |r| + SendResult.new do |r| r.type = "Sent" end end @@ -383,7 +383,7 @@ module MessageDequeuer it "gets a sender from the state and sends the message to it" do mocked_sender = double("SMTPSender") expect(mocked_sender).to receive(:send_message).with(queued_message.message).and_return(send_result) - expect(state).to receive(:sender_for).with(Postal::SMTPSender, message.recipient_domain, ip).and_return(mocked_sender) + expect(state).to receive(:sender_for).with(SMTPSender, message.recipient_domain, ip).and_return(mocked_sender) processor.process end @@ -393,7 +393,7 @@ module MessageDequeuer it "gets a sender from the state and sends the message to it" do mocked_sender = double("SMTPSender") expect(mocked_sender).to receive(:send_message).with(queued_message.message).and_return(send_result) - expect(state).to receive(:sender_for).with(Postal::SMTPSender, message.recipient_domain, nil).and_return(mocked_sender) + expect(state).to receive(:sender_for).with(SMTPSender, message.recipient_domain, nil).and_return(mocked_sender) processor.process end @@ -514,7 +514,7 @@ module MessageDequeuer context "when an exception occurrs during processing" do before do smtp_sender_mock = double("SMTPSender") - allow(Postal::SMTPSender).to receive(:new).and_return(smtp_sender_mock) + allow(SMTPSender).to receive(:new).and_return(smtp_sender_mock) allow(smtp_sender_mock).to receive(:start) allow(smtp_sender_mock).to receive(:send_message) do 1 / 0 diff --git a/spec/lib/message_dequeuer/state_spec.rb b/spec/lib/message_dequeuer/state_spec.rb index 3bcb608..49ca96b 100644 --- a/spec/lib/message_dequeuer/state_spec.rb +++ b/spec/lib/message_dequeuer/state_spec.rb @@ -9,7 +9,7 @@ module MessageDequeuer describe "#send_result" do it "can be get and set" do - result = instance_double(Postal::SendResult) + result = instance_double(SendResult) state.send_result = result expect(state.send_result).to be result end @@ -17,20 +17,20 @@ module MessageDequeuer describe "#sender_for" do it "returns a instance of the given sender initialized with the args" do - sender = state.sender_for(Postal::HTTPSender, "1234") - expect(sender).to be_a Postal::HTTPSender + sender = state.sender_for(HTTPSender, "1234") + expect(sender).to be_a HTTPSender end it "returns a cached sender on subsequent calls" do - sender = state.sender_for(Postal::HTTPSender, "1234") - expect(state.sender_for(Postal::HTTPSender, "1234")).to be sender + sender = state.sender_for(HTTPSender, "1234") + expect(state.sender_for(HTTPSender, "1234")).to be sender end end describe "#finished" do it "calls finish on all cached senders" do - sender1 = state.sender_for(Postal::HTTPSender, "1234") - sender2 = state.sender_for(Postal::HTTPSender, "4444") + sender1 = state.sender_for(HTTPSender, "1234") + sender2 = state.sender_for(HTTPSender, "4444") expect(sender1).to receive(:finish) expect(sender2).to receive(:finish) From 321ab95936ec626b7574bbc78072309c02a8f0ad Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Thu, 22 Feb 2024 22:36:04 +0000 Subject: [PATCH 25/56] refactor: move lib/postal/smtp_server to app/lib/smtp_server --- app/lib/smtp_server/client.rb | 507 +++++++++++++++++ app/lib/smtp_server/server.rb | 272 ++++++++++ lib/postal/smtp_server.rb | 6 - lib/postal/smtp_server/client.rb | 510 ------------------ lib/postal/smtp_server/server.rb | 274 ---------- script/smtp_server.rb | 2 +- .../postal/smtp_server/client/auth_spec.rb | 122 ----- .../postal/smtp_server/client/data_spec.rb | 89 --- .../smtp_server/client/finished_spec.rb | 208 ------- .../postal/smtp_server/client/helo_spec.rb | 38 -- .../smtp_server/client/mail_from_spec.rb | 35 -- .../postal/smtp_server/client/rcpt_to_spec.rb | 172 ------ spec/lib/postal/smtp_server/client_spec.rb | 14 - spec/lib/smtp_server/client/auth_spec.rb | 120 +++++ spec/lib/smtp_server/client/data_spec.rb | 87 +++ spec/lib/smtp_server/client/finished_spec.rb | 206 +++++++ spec/lib/smtp_server/client/helo_spec.rb | 36 ++ spec/lib/smtp_server/client/mail_from_spec.rb | 33 ++ spec/lib/smtp_server/client/rcpt_to_spec.rb | 170 ++++++ spec/lib/smtp_server/client_spec.rb | 12 + 20 files changed, 1444 insertions(+), 1469 deletions(-) create mode 100644 app/lib/smtp_server/client.rb create mode 100644 app/lib/smtp_server/server.rb delete mode 100644 lib/postal/smtp_server.rb delete mode 100644 lib/postal/smtp_server/client.rb delete mode 100644 lib/postal/smtp_server/server.rb delete mode 100644 spec/lib/postal/smtp_server/client/auth_spec.rb delete mode 100644 spec/lib/postal/smtp_server/client/data_spec.rb delete mode 100644 spec/lib/postal/smtp_server/client/finished_spec.rb delete mode 100644 spec/lib/postal/smtp_server/client/helo_spec.rb delete mode 100644 spec/lib/postal/smtp_server/client/mail_from_spec.rb delete mode 100644 spec/lib/postal/smtp_server/client/rcpt_to_spec.rb delete mode 100644 spec/lib/postal/smtp_server/client_spec.rb create mode 100644 spec/lib/smtp_server/client/auth_spec.rb create mode 100644 spec/lib/smtp_server/client/data_spec.rb create mode 100644 spec/lib/smtp_server/client/finished_spec.rb create mode 100644 spec/lib/smtp_server/client/helo_spec.rb create mode 100644 spec/lib/smtp_server/client/mail_from_spec.rb create mode 100644 spec/lib/smtp_server/client/rcpt_to_spec.rb create mode 100644 spec/lib/smtp_server/client_spec.rb diff --git a/app/lib/smtp_server/client.rb b/app/lib/smtp_server/client.rb new file mode 100644 index 0000000..3040f7f --- /dev/null +++ b/app/lib/smtp_server/client.rb @@ -0,0 +1,507 @@ +# frozen_string_literal: true + +require "nifty/utils/random_string" + +module SMTPServer + class Client + + CRAM_MD5_DIGEST = OpenSSL::Digest.new("md5") + LOG_REDACTION_STRING = "[redacted]" + + attr_reader :logging_enabled + attr_reader :credential + attr_reader :ip_address + attr_reader :recipients + attr_reader :headers + attr_reader :state + attr_reader :helo_name + + def initialize(ip_address) + @logging_enabled = true + @ip_address = ip_address + if @ip_address + check_ip_address + @state = :welcome + else + @state = :preauth + end + transaction_reset + end + + def check_ip_address + return unless @ip_address && Postal.config.smtp_server.log_exclude_ips && @ip_address =~ Regexp.new(Postal.config.smtp_server.log_exclude_ips) + + @logging_enabled = false + end + + def transaction_reset + @recipients = [] + @mail_from = nil + @data = nil + @headers = nil + end + + def id + @id ||= Nifty::Utils::RandomString.generate(length: 6).upcase + end + + def handle(data) + Postal.logger.tagged(id: id) do + if @state == :preauth + return proxy(data) + end + + log "\e[32m<= #{sanitize_input_for_log(data.strip)}\e[0m" + if @proc + @proc.call(data) + + else + handle_command(data) + end + end + end + + def finished? + @finished || false + end + + def start_tls? + @start_tls || false + end + + attr_writer :start_tls + + def handle_command(data) + case data + when /^QUIT/i then quit + when /^STARTTLS/i then starttls + when /^EHLO/i then ehlo(data) + when /^HELO/i then helo(data) + when /^RSET/i then rset + when /^NOOP/i then noop + when /^AUTH PLAIN/i then auth_plain(data) + when /^AUTH LOGIN/i then auth_login(data) + when /^AUTH CRAM-MD5/i then auth_cram_md5(data) + when /^MAIL FROM/i then mail_from(data) + when /^RCPT TO/i then rcpt_to(data) + when /^DATA/i then data(data) + else + "502 Invalid/unsupported command" + end + end + + def log(text) + return false unless @logging_enabled + + Postal.logger.debug(text, id: id) + end + + private + + def proxy(data) + if m = data.match(/\APROXY (.+) (.+) (.+) (.+) (.+)\z/) + @ip_address = m[2] + check_ip_address + @state = :welcome + log "\e[35m Client identified as #{@ip_address}\e[0m" + "220 #{Postal.config.dns.smtp_server_hostname} ESMTP Postal/#{id}" + else + @finished = true + "502 Proxy Error" + end + end + + def quit + @finished = true + "221 Closing Connection" + end + + def starttls + if Postal.config.smtp_server.tls_enabled? + @start_tls = true + @tls = true + "220 Ready to start TLS" + else + "502 TLS not available" + end + end + + def ehlo(data) + @helo_name = data.strip.split(" ", 2)[1] + transaction_reset + @state = :welcomed + [ + "250-My capabilities are", + Postal.config.smtp_server.tls_enabled? && !@tls ? "250-STARTTLS" : nil, + "250 AUTH CRAM-MD5 PLAIN LOGIN" + ].compact + end + + def helo(data) + @helo_name = data.strip.split(" ", 2)[1] + transaction_reset + @state = :welcomed + "250 #{Postal.config.dns.smtp_server_hostname}" + end + + def rset + transaction_reset + @state = :welcomed + "250 OK" + end + + def noop + "250 OK" + end + + def auth_plain(data) + handler = proc do |idata| + @proc = nil + idata = Base64.decode64(idata) + parts = idata.split("\0") + username = parts[-2] + password = parts[-1] + unless username && password + next "535 Authenticated failed - protocol error" + end + + authenticate(password) + end + + data = data.gsub(/AUTH PLAIN ?/i, "") + if data.strip == "" + @proc = handler + @password_expected_next = true + "334" + else + handler.call(data) + end + end + + def auth_login(data) + password_handler = proc do |idata| + @proc = nil + password = Base64.decode64(idata) + authenticate(password) + end + + username_handler = proc do + @proc = password_handler + @password_expected_next = true + "334 UGFzc3dvcmQ6" # "Password:" + end + + data = data.gsub(/AUTH LOGIN ?/i, "") + if data.strip == "" + @proc = username_handler + "334 VXNlcm5hbWU6" # "Username:" + else + username_handler.call(nil) + end + end + + def authenticate(password) + if @credential = Credential.where(type: "SMTP", key: password).first + @credential.use + "235 Granted for #{@credential.server.organization.permalink}/#{@credential.server.permalink}" + else + log "\e[33m WARN: AUTH failure for #{@ip_address}\e[0m" + "535 Invalid credential" + end + end + + def auth_cram_md5(data) + challenge = Digest::SHA1.hexdigest(Time.now.to_i.to_s + rand(100_000).to_s) + challenge = "<#{challenge[0, 20]}@#{Postal.config.dns.smtp_server_hostname}>" + + handler = proc do |idata| + @proc = nil + username, password = Base64.decode64(idata).split(" ", 2).map { |a| a.chomp } + org_permlink, server_permalink = username.split(/[\/_]/, 2) + server = ::Server.includes(:organization).where(organizations: { permalink: org_permlink }, permalink: server_permalink).first + if server.nil? + log "\e[33m WARN: AUTH failure for #{@ip_address}\e[0m" + next "535 Denied" + end + + grant = nil + server.credentials.where(type: "SMTP").each do |credential| + correct_response = OpenSSL::HMAC.hexdigest(CRAM_MD5_DIGEST, credential.key, challenge) + next unless password == correct_response + + @credential = credential + @credential.use + grant = "235 Granted for #{credential.server.organization.permalink}/#{credential.server.permalink}" + break + end + + if grant.nil? + log "\e[33m WARN: AUTH failure for #{@ip_address}\e[0m" + next "535 Denied" + end + + grant + end + + @proc = handler + "334 " + Base64.encode64(challenge).gsub(/[\r\n]/, "") + end + + def mail_from(data) + unless in_state(:welcomed, :mail_from_received) + return "503 EHLO/HELO first please" + end + + @state = :mail_from_received + transaction_reset + if data =~ /AUTH=/ + # Discard AUTH= parameter and anything that follows. + # We don't need this parameter as we don't trust any client to set it + mail_from_line = data.sub(/ *AUTH=.*/, "") + else + mail_from_line = data + end + @mail_from = mail_from_line.gsub(/MAIL FROM\s*:\s*/i, "").gsub(/.*.*/, "").strip + "250 OK" + end + + def rcpt_to(data) + unless in_state(:mail_from_received, :rcpt_to_received) + return "503 EHLO/HELO and MAIL FROM first please" + end + + rcpt_to = data.gsub(/RCPT TO\s*:\s*/i, "").gsub(/.*.*/, "").strip + + if rcpt_to.blank? + return "501 RCPT TO should not be empty" + end + + uname, domain = rcpt_to.split("@", 2) + + if domain.blank? + return "501 Invalid RCPT TO" + end + + uname, tag = uname.split("+", 2) + + if domain == Postal.config.dns.return_path || domain =~ /\A#{Regexp.escape(Postal.config.dns.custom_return_path_prefix)}\./ + # This is a return path + @state = :rcpt_to_received + if server = ::Server.where(token: uname).first + if server.suspended? + "535 Mail server has been suspended" + else + log "Added bounce on server #{server.id}" + @recipients << [:bounce, rcpt_to, server] + "250 OK" + end + else + "550 Invalid server token" + end + + elsif domain == Postal.config.dns.route_domain + # This is an email direct to a route. This isn't actually supported yet. + @state = :rcpt_to_received + if route = Route.where(token: uname).first + if route.server.suspended? + "535 Mail server has been suspended" + elsif route.mode == "Reject" + "550 Route does not accept incoming messages" + else + log "Added route #{route.id} to recipients (tag: #{tag.inspect})" + actual_rcpt_to = "#{route.name}#{tag ? "+#{tag}" : ''}@#{route.domain.name}" + @recipients << [:route, actual_rcpt_to, route.server, { route: route }] + "250 OK" + end + else + "550 Invalid route token" + end + + elsif @credential + # This is outgoing mail for an authenticated user + @state = :rcpt_to_received + if @credential.server.suspended? + "535 Mail server has been suspended" + else + log "Added external address '#{rcpt_to}'" + @recipients << [:credential, rcpt_to, @credential.server] + "250 OK" + end + + elsif uname && domain && route = Route.find_by_name_and_domain(uname, domain) + # This is incoming mail for a route + @state = :rcpt_to_received + if route.server.suspended? + "535 Mail server has been suspended" + elsif route.mode == "Reject" + "550 Route does not accept incoming messages" + else + log "Added route #{route.id} to recipients (tag: #{tag.inspect})" + @recipients << [:route, rcpt_to, route.server, { route: route }] + "250 OK" + end + + else + # User is trying to relay but is not authenticated. Try to authenticate by IP address + @credential = Credential.where(type: "SMTP-IP").all.sort_by { |c| c.ipaddr&.prefix || 0 }.reverse.find do |credential| + credential.ipaddr.include?(@ip_address) || (credential.ipaddr.ipv4? && credential.ipaddr.ipv4_mapped.include?(@ip_address)) + end + + if @credential + # Retry with credential + @credential.use + rcpt_to(data) + else + "530 Authentication required" + end + end + end + + def data(_data) + unless in_state(:rcpt_to_received) + return "503 HELO/EHLO, MAIL FROM and RCPT TO before sending data" + end + + @data = String.new.force_encoding("BINARY") + @headers = {} + @receiving_headers = true + + received_header = Postal::ReceivedHeader.generate(@credential&.server, @helo_name, @ip_address, :smtp) + .force_encoding("BINARY") + + @data << "Received: #{received_header}\r\n" + @headers["received"] = [received_header] + + handler = proc do |idata| + if idata == "." + @logging_enabled = true + @proc = nil + finished + else + idata = idata.to_s.sub(/\A\.\./, ".") + + if @credential&.server&.log_smtp_data? + # We want to log if enabled + else + log "Not logging further message data." + @logging_enabled = false + end + + if @receiving_headers + if idata.blank? + @receiving_headers = false + elsif idata.to_s =~ /^\s/ + # This is a continuation of a header + if @header_key && @headers[@header_key.downcase] && @headers[@header_key.downcase].last + @headers[@header_key.downcase].last << idata.to_s + end + else + @header_key, value = idata.split(/:\s*/, 2) + @headers[@header_key.downcase] ||= [] + @headers[@header_key.downcase] << value + end + end + @data << idata + @data << "\r\n" + nil + end + end + + @proc = handler + "354 Go ahead" + end + + def finished + if @data.bytesize > Postal.config.smtp_server.max_message_size.megabytes.to_i + transaction_reset + @state = :welcomed + return format("552 Message too large (maximum size %dMB)", Postal.config.smtp_server.max_message_size) + end + + if @headers["received"].grep(/by #{Postal.config.dns.smtp_server_hostname}/).count > 4 + transaction_reset + @state = :welcomed + return "550 Loop detected" + end + + authenticated_domain = nil + if @credential + authenticated_domain = @credential.server.find_authenticated_domain_from_headers(@headers) + if authenticated_domain.nil? + transaction_reset + @state = :welcomed + return "530 From/Sender name is not valid" + end + end + + @recipients.each do |recipient| + type, rcpt_to, server, options = recipient + + case type + when :credential + # Outgoing messages are just inserted + message = server.message_db.new_message + message.rcpt_to = rcpt_to + message.mail_from = @mail_from + message.raw_message = @data + message.received_with_ssl = @tls + message.scope = "outgoing" + message.domain_id = authenticated_domain&.id + message.credential_id = @credential.id + message.save + + when :bounce + if rp_route = server.routes.where(name: "__returnpath__").first + # If there's a return path route, we can use this to create the message + rp_route.create_messages do |msg| + msg.rcpt_to = rcpt_to + msg.mail_from = @mail_from + msg.raw_message = @data + msg.received_with_ssl = @tls + msg.bounce = 1 + end + else + # There's no return path route, we just need to insert the mesage + # without going through the route. + message = server.message_db.new_message + message.rcpt_to = rcpt_to + message.mail_from = @mail_from + message.raw_message = @data + message.received_with_ssl = @tls + message.scope = "incoming" + message.bounce = 1 + message.save + end + when :route + options[:route].create_messages do |message| + message.rcpt_to = rcpt_to + message.mail_from = @mail_from + message.raw_message = @data + message.received_with_ssl = @tls + end + end + end + transaction_reset + @state = :welcomed + "250 OK" + end + + def in_state(*states) + states.include?(@state) + end + + def sanitize_input_for_log(data) + if @password_expected_next + @password_expected_next = false + if data =~ /\A[a-z0-9]{3,}=*\z/i + return LOG_REDACTION_STRING + end + end + + data = data.dup + data.gsub!(/(.*AUTH \w+) (.*)\z/i) { "#{::Regexp.last_match(1)} #{LOG_REDACTION_STRING}" } + data + end + + end +end diff --git a/app/lib/smtp_server/server.rb b/app/lib/smtp_server/server.rb new file mode 100644 index 0000000..e51fadd --- /dev/null +++ b/app/lib/smtp_server/server.rb @@ -0,0 +1,272 @@ +# frozen_string_literal: true + +require "ipaddr" +require "nio" + +module SMTPServer + class Server + + def initialize(options = {}) + @options = options + @options[:debug] ||= false + prepare_environment + end + + def run + logger.tagged(component: "smtp-server") do + listen + run_event_loop + end + end + + private + + def prepare_environment + $\ = "\r\n" + BasicSocket.do_not_reverse_lookup = true + + trap("TERM") do + $stdout.puts "Received TERM signal, shutting down." + unlisten + end + + trap("INT") do + $stdout.puts "Received INT signal, shutting down." + unlisten + end + end + + def ssl_context + @ssl_context ||= begin + ssl_context = OpenSSL::SSL::SSLContext.new + ssl_context.cert = Postal.smtp_certificates[0] + ssl_context.extra_chain_cert = Postal.smtp_certificates[1..] + ssl_context.key = Postal.smtp_private_key + ssl_context.ssl_version = Postal.config.smtp_server.ssl_version if Postal.config.smtp_server.ssl_version + ssl_context.ciphers = Postal.config.smtp_server.tls_ciphers if Postal.config.smtp_server.tls_ciphers + ssl_context + end + end + + def listen + @server = TCPServer.open(Postal.config.smtp_server.bind_address, Postal.config.smtp_server.port) + @server.autoclose = false + @server.close_on_exec = false + if defined?(Socket::SOL_SOCKET) && defined?(Socket::SO_KEEPALIVE) + @server.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true) + end + if defined?(Socket::SOL_TCP) && defined?(Socket::TCP_KEEPIDLE) && defined?(Socket::TCP_KEEPINTVL) && defined?(Socket::TCP_KEEPCNT) + @server.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPIDLE, 50) + @server.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPINTVL, 10) + @server.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPCNT, 5) + end + logger.info "Listening on #{Postal.config.smtp_server.bind_address}:#{Postal.config.smtp_server.port}" + end + + def unlisten + # Instruct the nio loop to unlisten and wake it + @unlisten = true + @io_selector.wakeup + end + + def run_event_loop + # Set up an instance of nio4r to monitor for connections and data + @io_selector = NIO::Selector.new + # Register the SMTP listener + @io_selector.register(@server, :r) + # Create a hash to contain a buffer for each client. + buffers = Hash.new { |h, k| h[k] = String.new.force_encoding("BINARY") } + loop do + # Wait for an event to occur + @io_selector.select do |monitor| + # Get the IO from the nio monitor + io = monitor.io + # Is this event an incoming connection? + if io.is_a?(TCPServer) + begin + # Accept the connection + new_io = io.accept + if Postal.config.smtp_server.proxy_protocol + # If we are using the haproxy proxy protocol, we will be sent the + # client's IP later. Delay the welcome process. + client = Client.new(nil) + if Postal.config.smtp_server.log_connect + logger.debug "[#{client.id}] \e[35m Connection opened from #{new_io.remote_address.ip_address}\e[0m" + end + else + # We're not using the proxy protocol so we already know the client's IP + client = Client.new(new_io.remote_address.ip_address) + if Postal.config.smtp_server.log_connect + logger.debug "[#{client.id}] \e[35m Connection opened from #{new_io.remote_address.ip_address}\e[0m" + end + # We know who the client is, welcome them. + client.log "\e[35m Client identified as #{new_io.remote_address.ip_address}\e[0m" + new_io.print("220 #{Postal.config.dns.smtp_server_hostname} ESMTP Postal/#{client.id}") + end + # Register the client and its socket with nio4r + monitor = @io_selector.register(new_io, :r) + monitor.value = client + rescue StandardError => e + # If something goes wrong, log as appropriate and disconnect the client + if defined?(Sentry) + Sentry.capture_exception(e, extra: { log_id: begin + client.id + rescue StandardError + nil + end }) + end + logger.error "An error occurred while accepting a new client." + logger.error "#{e.class}: #{e.message}" + e.backtrace.each do |line| + logger.error line + end + begin + new_io.close + rescue StandardError + nil + end + end + else + # This event is not an incoming connection so it must be data from a client + begin + # Get the client from the nio monitor + client = monitor.value + # For now we assume the connection isn't closed + eof = false + # Is the client negotiating a TLS handshake? + if client.start_tls? + begin + # Can we accept the TLS connection at this time? + io.accept_nonblock + # We were able to accept the connection, the client is no longer handshaking + client.start_tls = false + rescue IO::WaitReadable, IO::WaitWritable => e + # Could not accept without blocking + # We will try again later + next + rescue OpenSSL::SSL::SSLError => e + client.log "SSL Negotiation Failed: #{e.message}" + eof = true + end + else + # The client is not negotiating a TLS handshake at this time + begin + # Read 10kiB of data at a time from the socket. + buffers[io] << io.readpartial(10_240) + + # There is an extra step for SSL sockets + if io.is_a?(OpenSSL::SSL::SSLSocket) + buffers[io] << io.readpartial(10_240) while io.pending.positive? + end + rescue EOFError, Errno::ECONNRESET, Errno::ETIMEDOUT + # Client went away + eof = true + end + + # Normalize all \r\n and \n to \r\n, but ignore only \r. + # A \r\n may be split in 2 buffers (\n in one buffer and \r in the other) + buffers[io] = buffers[io].gsub(/\r/, "").encode(buffers[io].encoding, crlf_newline: true) + + # We line buffer, so look to see if we have received a newline + # and keep doing so until all buffered lines have been processed. + while buffers[io].index("\r\n") + # Extract the line + line, buffers[io] = buffers[io].split("\r\n", 2) + + # Send the received line to the client object for processing + result = client.handle(line) + # If the client object returned some data, write it back to the client + next if result.nil? + + result = [result] unless result.is_a?(Array) + result.compact.each do |iline| + client.log "\e[34m=> #{iline.strip}\e[0m" + begin + io.write(iline.to_s + "\r\n") + io.flush + rescue Errno::ECONNRESET + # Client disconnected before we could write response + eof = true + end + end + end + + # Did the client request STARTTLS? + if !eof && client.start_tls? + # Deregister the unencrypted IO + @io_selector.deregister(io) + buffers.delete(io) + io = OpenSSL::SSL::SSLSocket.new(io, ssl_context) + # Close the underlying IO when the TLS socket is closed + io.sync_close = true + # Register the new TLS socket with nio + monitor = @io_selector.register(io, :r) + monitor.value = client + end + end + + # Has the client requested we close the connection? + if client.finished? || eof + client.log "\e[35m Connection closed\e[0m" + # Deregister the socket and close it + @io_selector.deregister(io) + buffers.delete(io) + io.close + # If we have no more clients or listeners left, exit the process + if @io_selector.empty? + Process.exit(0) + end + end + rescue StandardError => e + # Something went wrong, log as appropriate + client_id = client ? client.id : "------" + if defined?(Sentry) + Sentry.capture_exception(e, extra: { log_id: begin + client.id + rescue StandardError + nil + end }) + end + logger.error "[#{client_id}] An error occurred while processing data from a client." + logger.error "[#{client_id}] #{e.class}: #{e.message}" + e.backtrace.each do |iline| + logger.error "[#{client_id}] #{iline}" + end + # Close all IO and forget this client + begin + @io_selector.deregister(io) + rescue StandardError + nil + end + buffers.delete(io) + begin + io.close + rescue StandardError + nil + end + if @io_selector.empty? + Process.exit(0) + end + end + end + end + # If unlisten has been called, stop listening + next unless @unlisten + + @io_selector.deregister(@server) + @server.close + # If there's nothing left to do, shut down the process + if @io_selector.empty? + Process.exit(0) + end + # Clear the request + @unlisten = false + end + end + + def logger + Postal.logger + end + + end +end diff --git a/lib/postal/smtp_server.rb b/lib/postal/smtp_server.rb deleted file mode 100644 index 154ae62..0000000 --- a/lib/postal/smtp_server.rb +++ /dev/null @@ -1,6 +0,0 @@ -# frozen_string_literal: true - -module Postal - module SMTPServer - end -end diff --git a/lib/postal/smtp_server/client.rb b/lib/postal/smtp_server/client.rb deleted file mode 100644 index 6aedcfc..0000000 --- a/lib/postal/smtp_server/client.rb +++ /dev/null @@ -1,510 +0,0 @@ -# frozen_string_literal: true - -require "resolv" -require "nifty/utils/random_string" - -module Postal - module SMTPServer - class Client - - CRAM_MD5_DIGEST = OpenSSL::Digest.new("md5") - LOG_REDACTION_STRING = "[redacted]" - - attr_reader :logging_enabled - attr_reader :credential - attr_reader :ip_address - attr_reader :recipients - attr_reader :headers - attr_reader :state - attr_reader :helo_name - - def initialize(ip_address) - @logging_enabled = true - @ip_address = ip_address - if @ip_address - check_ip_address - @state = :welcome - else - @state = :preauth - end - transaction_reset - end - - def check_ip_address - return unless @ip_address && Postal.config.smtp_server.log_exclude_ips && @ip_address =~ Regexp.new(Postal.config.smtp_server.log_exclude_ips) - - @logging_enabled = false - end - - def transaction_reset - @recipients = [] - @mail_from = nil - @data = nil - @headers = nil - end - - def id - @id ||= Nifty::Utils::RandomString.generate(length: 6).upcase - end - - def handle(data) - Postal.logger.tagged(id: id) do - if @state == :preauth - return proxy(data) - end - - log "\e[32m<= #{sanitize_input_for_log(data.strip)}\e[0m" - if @proc - @proc.call(data) - - else - handle_command(data) - end - end - end - - def finished? - @finished || false - end - - def start_tls? - @start_tls || false - end - - attr_writer :start_tls - - def handle_command(data) - case data - when /^QUIT/i then quit - when /^STARTTLS/i then starttls - when /^EHLO/i then ehlo(data) - when /^HELO/i then helo(data) - when /^RSET/i then rset - when /^NOOP/i then noop - when /^AUTH PLAIN/i then auth_plain(data) - when /^AUTH LOGIN/i then auth_login(data) - when /^AUTH CRAM-MD5/i then auth_cram_md5(data) - when /^MAIL FROM/i then mail_from(data) - when /^RCPT TO/i then rcpt_to(data) - when /^DATA/i then data(data) - else - "502 Invalid/unsupported command" - end - end - - def log(text) - return false unless @logging_enabled - - Postal.logger.debug(text, id: id) - end - - private - - def proxy(data) - if m = data.match(/\APROXY (.+) (.+) (.+) (.+) (.+)\z/) - @ip_address = m[2] - check_ip_address - @state = :welcome - log "\e[35m Client identified as #{@ip_address}\e[0m" - "220 #{Postal.config.dns.smtp_server_hostname} ESMTP Postal/#{id}" - else - @finished = true - "502 Proxy Error" - end - end - - def quit - @finished = true - "221 Closing Connection" - end - - def starttls - if Postal.config.smtp_server.tls_enabled? - @start_tls = true - @tls = true - "220 Ready to start TLS" - else - "502 TLS not available" - end - end - - def ehlo(data) - @helo_name = data.strip.split(" ", 2)[1] - transaction_reset - @state = :welcomed - [ - "250-My capabilities are", - Postal.config.smtp_server.tls_enabled? && !@tls ? "250-STARTTLS" : nil, - "250 AUTH CRAM-MD5 PLAIN LOGIN" - ].compact - end - - def helo(data) - @helo_name = data.strip.split(" ", 2)[1] - transaction_reset - @state = :welcomed - "250 #{Postal.config.dns.smtp_server_hostname}" - end - - def rset - transaction_reset - @state = :welcomed - "250 OK" - end - - def noop - "250 OK" - end - - def auth_plain(data) - handler = proc do |idata| - @proc = nil - idata = Base64.decode64(idata) - parts = idata.split("\0") - username = parts[-2] - password = parts[-1] - unless username && password - next "535 Authenticated failed - protocol error" - end - - authenticate(password) - end - - data = data.gsub(/AUTH PLAIN ?/i, "") - if data.strip == "" - @proc = handler - @password_expected_next = true - "334" - else - handler.call(data) - end - end - - def auth_login(data) - password_handler = proc do |idata| - @proc = nil - password = Base64.decode64(idata) - authenticate(password) - end - - username_handler = proc do - @proc = password_handler - @password_expected_next = true - "334 UGFzc3dvcmQ6" # "Password:" - end - - data = data.gsub(/AUTH LOGIN ?/i, "") - if data.strip == "" - @proc = username_handler - "334 VXNlcm5hbWU6" # "Username:" - else - username_handler.call(nil) - end - end - - def authenticate(password) - if @credential = Credential.where(type: "SMTP", key: password).first - @credential.use - "235 Granted for #{@credential.server.organization.permalink}/#{@credential.server.permalink}" - else - log "\e[33m WARN: AUTH failure for #{@ip_address}\e[0m" - "535 Invalid credential" - end - end - - def auth_cram_md5(data) - challenge = Digest::SHA1.hexdigest(Time.now.to_i.to_s + rand(100_000).to_s) - challenge = "<#{challenge[0, 20]}@#{Postal.config.dns.smtp_server_hostname}>" - - handler = proc do |idata| - @proc = nil - username, password = Base64.decode64(idata).split(" ", 2).map { |a| a.chomp } - org_permlink, server_permalink = username.split(/[\/_]/, 2) - server = ::Server.includes(:organization).where(organizations: { permalink: org_permlink }, permalink: server_permalink).first - if server.nil? - log "\e[33m WARN: AUTH failure for #{@ip_address}\e[0m" - next "535 Denied" - end - - grant = nil - server.credentials.where(type: "SMTP").each do |credential| - correct_response = OpenSSL::HMAC.hexdigest(CRAM_MD5_DIGEST, credential.key, challenge) - next unless password == correct_response - - @credential = credential - @credential.use - grant = "235 Granted for #{credential.server.organization.permalink}/#{credential.server.permalink}" - break - end - - if grant.nil? - log "\e[33m WARN: AUTH failure for #{@ip_address}\e[0m" - next "535 Denied" - end - - grant - end - - @proc = handler - "334 " + Base64.encode64(challenge).gsub(/[\r\n]/, "") - end - - def mail_from(data) - unless in_state(:welcomed, :mail_from_received) - return "503 EHLO/HELO first please" - end - - @state = :mail_from_received - transaction_reset - if data =~ /AUTH=/ - # Discard AUTH= parameter and anything that follows. - # We don't need this parameter as we don't trust any client to set it - mail_from_line = data.sub(/ *AUTH=.*/, "") - else - mail_from_line = data - end - @mail_from = mail_from_line.gsub(/MAIL FROM\s*:\s*/i, "").gsub(/.*.*/, "").strip - "250 OK" - end - - def rcpt_to(data) - unless in_state(:mail_from_received, :rcpt_to_received) - return "503 EHLO/HELO and MAIL FROM first please" - end - - rcpt_to = data.gsub(/RCPT TO\s*:\s*/i, "").gsub(/.*.*/, "").strip - - if rcpt_to.blank? - return "501 RCPT TO should not be empty" - end - - uname, domain = rcpt_to.split("@", 2) - - if domain.blank? - return "501 Invalid RCPT TO" - end - - uname, tag = uname.split("+", 2) - - if domain == Postal.config.dns.return_path || domain =~ /\A#{Regexp.escape(Postal.config.dns.custom_return_path_prefix)}\./ - # This is a return path - @state = :rcpt_to_received - if server = ::Server.where(token: uname).first - if server.suspended? - "535 Mail server has been suspended" - else - log "Added bounce on server #{server.id}" - @recipients << [:bounce, rcpt_to, server] - "250 OK" - end - else - "550 Invalid server token" - end - - elsif domain == Postal.config.dns.route_domain - # This is an email direct to a route. This isn't actually supported yet. - @state = :rcpt_to_received - if route = Route.where(token: uname).first - if route.server.suspended? - "535 Mail server has been suspended" - elsif route.mode == "Reject" - "550 Route does not accept incoming messages" - else - log "Added route #{route.id} to recipients (tag: #{tag.inspect})" - actual_rcpt_to = "#{route.name}#{tag ? "+#{tag}" : ''}@#{route.domain.name}" - @recipients << [:route, actual_rcpt_to, route.server, { route: route }] - "250 OK" - end - else - "550 Invalid route token" - end - - elsif @credential - # This is outgoing mail for an authenticated user - @state = :rcpt_to_received - if @credential.server.suspended? - "535 Mail server has been suspended" - else - log "Added external address '#{rcpt_to}'" - @recipients << [:credential, rcpt_to, @credential.server] - "250 OK" - end - - elsif uname && domain && route = Route.find_by_name_and_domain(uname, domain) - # This is incoming mail for a route - @state = :rcpt_to_received - if route.server.suspended? - "535 Mail server has been suspended" - elsif route.mode == "Reject" - "550 Route does not accept incoming messages" - else - log "Added route #{route.id} to recipients (tag: #{tag.inspect})" - @recipients << [:route, rcpt_to, route.server, { route: route }] - "250 OK" - end - - else - # User is trying to relay but is not authenticated. Try to authenticate by IP address - @credential = Credential.where(type: "SMTP-IP").all.sort_by { |c| c.ipaddr&.prefix || 0 }.reverse.find do |credential| - credential.ipaddr.include?(@ip_address) || (credential.ipaddr.ipv4? && credential.ipaddr.ipv4_mapped.include?(@ip_address)) - end - - if @credential - # Retry with credential - @credential.use - rcpt_to(data) - else - "530 Authentication required" - end - end - end - - def data(_data) - unless in_state(:rcpt_to_received) - return "503 HELO/EHLO, MAIL FROM and RCPT TO before sending data" - end - - @data = String.new.force_encoding("BINARY") - @headers = {} - @receiving_headers = true - - received_header = Postal::ReceivedHeader.generate(@credential&.server, @helo_name, @ip_address, :smtp) - .force_encoding("BINARY") - - @data << "Received: #{received_header}\r\n" - @headers["received"] = [received_header] - - handler = proc do |idata| - if idata == "." - @logging_enabled = true - @proc = nil - finished - else - idata = idata.to_s.sub(/\A\.\./, ".") - - if @credential&.server&.log_smtp_data? - # We want to log if enabled - else - log "Not logging further message data." - @logging_enabled = false - end - - if @receiving_headers - if idata.blank? - @receiving_headers = false - elsif idata.to_s =~ /^\s/ - # This is a continuation of a header - if @header_key && @headers[@header_key.downcase] && @headers[@header_key.downcase].last - @headers[@header_key.downcase].last << idata.to_s - end - else - @header_key, value = idata.split(/:\s*/, 2) - @headers[@header_key.downcase] ||= [] - @headers[@header_key.downcase] << value - end - end - @data << idata - @data << "\r\n" - nil - end - end - - @proc = handler - "354 Go ahead" - end - - def finished - if @data.bytesize > Postal.config.smtp_server.max_message_size.megabytes.to_i - transaction_reset - @state = :welcomed - return format("552 Message too large (maximum size %dMB)", Postal.config.smtp_server.max_message_size) - end - - if @headers["received"].grep(/by #{Postal.config.dns.smtp_server_hostname}/).count > 4 - transaction_reset - @state = :welcomed - return "550 Loop detected" - end - - authenticated_domain = nil - if @credential - authenticated_domain = @credential.server.find_authenticated_domain_from_headers(@headers) - if authenticated_domain.nil? - transaction_reset - @state = :welcomed - return "530 From/Sender name is not valid" - end - end - - @recipients.each do |recipient| - type, rcpt_to, server, options = recipient - - case type - when :credential - # Outgoing messages are just inserted - message = server.message_db.new_message - message.rcpt_to = rcpt_to - message.mail_from = @mail_from - message.raw_message = @data - message.received_with_ssl = @tls - message.scope = "outgoing" - message.domain_id = authenticated_domain&.id - message.credential_id = @credential.id - message.save - - when :bounce - if rp_route = server.routes.where(name: "__returnpath__").first - # If there's a return path route, we can use this to create the message - rp_route.create_messages do |msg| - msg.rcpt_to = rcpt_to - msg.mail_from = @mail_from - msg.raw_message = @data - msg.received_with_ssl = @tls - msg.bounce = 1 - end - else - # There's no return path route, we just need to insert the mesage - # without going through the route. - message = server.message_db.new_message - message.rcpt_to = rcpt_to - message.mail_from = @mail_from - message.raw_message = @data - message.received_with_ssl = @tls - message.scope = "incoming" - message.bounce = 1 - message.save - end - when :route - options[:route].create_messages do |message| - message.rcpt_to = rcpt_to - message.mail_from = @mail_from - message.raw_message = @data - message.received_with_ssl = @tls - end - end - end - transaction_reset - @state = :welcomed - "250 OK" - end - - def in_state(*states) - states.include?(@state) - end - - def sanitize_input_for_log(data) - if @password_expected_next - @password_expected_next = false - if data =~ /\A[a-z0-9]{3,}=*\z/i - return LOG_REDACTION_STRING - end - end - - data = data.dup - data.gsub!(/(.*AUTH \w+) (.*)\z/i) { "#{::Regexp.last_match(1)} #{LOG_REDACTION_STRING}" } - data - end - - end - end -end diff --git a/lib/postal/smtp_server/server.rb b/lib/postal/smtp_server/server.rb deleted file mode 100644 index 3a110a0..0000000 --- a/lib/postal/smtp_server/server.rb +++ /dev/null @@ -1,274 +0,0 @@ -# frozen_string_literal: true - -require "ipaddr" -require "nio" - -module Postal - module SMTPServer - class Server - - def initialize(options = {}) - @options = options - @options[:debug] ||= false - prepare_environment - end - - def run - logger.tagged(component: "smtp-server") do - listen - run_event_loop - end - end - - private - - def prepare_environment - $\ = "\r\n" - BasicSocket.do_not_reverse_lookup = true - - trap("TERM") do - $stdout.puts "Received TERM signal, shutting down." - unlisten - end - - trap("INT") do - $stdout.puts "Received INT signal, shutting down." - unlisten - end - end - - def ssl_context - @ssl_context ||= begin - ssl_context = OpenSSL::SSL::SSLContext.new - ssl_context.cert = Postal.smtp_certificates[0] - ssl_context.extra_chain_cert = Postal.smtp_certificates[1..] - ssl_context.key = Postal.smtp_private_key - ssl_context.ssl_version = Postal.config.smtp_server.ssl_version if Postal.config.smtp_server.ssl_version - ssl_context.ciphers = Postal.config.smtp_server.tls_ciphers if Postal.config.smtp_server.tls_ciphers - ssl_context - end - end - - def listen - @server = TCPServer.open(Postal.config.smtp_server.bind_address, Postal.config.smtp_server.port) - @server.autoclose = false - @server.close_on_exec = false - if defined?(Socket::SOL_SOCKET) && defined?(Socket::SO_KEEPALIVE) - @server.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true) - end - if defined?(Socket::SOL_TCP) && defined?(Socket::TCP_KEEPIDLE) && defined?(Socket::TCP_KEEPINTVL) && defined?(Socket::TCP_KEEPCNT) - @server.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPIDLE, 50) - @server.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPINTVL, 10) - @server.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPCNT, 5) - end - logger.info "Listening on #{Postal.config.smtp_server.bind_address}:#{Postal.config.smtp_server.port}" - end - - def unlisten - # Instruct the nio loop to unlisten and wake it - @unlisten = true - @io_selector.wakeup - end - - def run_event_loop - # Set up an instance of nio4r to monitor for connections and data - @io_selector = NIO::Selector.new - # Register the SMTP listener - @io_selector.register(@server, :r) - # Create a hash to contain a buffer for each client. - buffers = Hash.new { |h, k| h[k] = String.new.force_encoding("BINARY") } - loop do - # Wait for an event to occur - @io_selector.select do |monitor| - # Get the IO from the nio monitor - io = monitor.io - # Is this event an incoming connection? - if io.is_a?(TCPServer) - begin - # Accept the connection - new_io = io.accept - if Postal.config.smtp_server.proxy_protocol - # If we are using the haproxy proxy protocol, we will be sent the - # client's IP later. Delay the welcome process. - client = Client.new(nil) - if Postal.config.smtp_server.log_connect - logger.debug "[#{client.id}] \e[35m Connection opened from #{new_io.remote_address.ip_address}\e[0m" - end - else - # We're not using the proxy protocol so we already know the client's IP - client = Client.new(new_io.remote_address.ip_address) - if Postal.config.smtp_server.log_connect - logger.debug "[#{client.id}] \e[35m Connection opened from #{new_io.remote_address.ip_address}\e[0m" - end - # We know who the client is, welcome them. - client.log "\e[35m Client identified as #{new_io.remote_address.ip_address}\e[0m" - new_io.print("220 #{Postal.config.dns.smtp_server_hostname} ESMTP Postal/#{client.id}") - end - # Register the client and its socket with nio4r - monitor = @io_selector.register(new_io, :r) - monitor.value = client - rescue StandardError => e - # If something goes wrong, log as appropriate and disconnect the client - if defined?(Sentry) - Sentry.capture_exception(e, extra: { log_id: begin - client.id - rescue StandardError - nil - end }) - end - logger.error "An error occurred while accepting a new client." - logger.error "#{e.class}: #{e.message}" - e.backtrace.each do |line| - logger.error line - end - begin - new_io.close - rescue StandardError - nil - end - end - else - # This event is not an incoming connection so it must be data from a client - begin - # Get the client from the nio monitor - client = monitor.value - # For now we assume the connection isn't closed - eof = false - # Is the client negotiating a TLS handshake? - if client.start_tls? - begin - # Can we accept the TLS connection at this time? - io.accept_nonblock - # We were able to accept the connection, the client is no longer handshaking - client.start_tls = false - rescue IO::WaitReadable, IO::WaitWritable => e - # Could not accept without blocking - # We will try again later - next - rescue OpenSSL::SSL::SSLError => e - client.log "SSL Negotiation Failed: #{e.message}" - eof = true - end - else - # The client is not negotiating a TLS handshake at this time - begin - # Read 10kiB of data at a time from the socket. - buffers[io] << io.readpartial(10_240) - - # There is an extra step for SSL sockets - if io.is_a?(OpenSSL::SSL::SSLSocket) - buffers[io] << io.readpartial(10_240) while io.pending.positive? - end - rescue EOFError, Errno::ECONNRESET, Errno::ETIMEDOUT - # Client went away - eof = true - end - - # Normalize all \r\n and \n to \r\n, but ignore only \r. - # A \r\n may be split in 2 buffers (\n in one buffer and \r in the other) - buffers[io] = buffers[io].gsub(/\r/, "").encode(buffers[io].encoding, crlf_newline: true) - - # We line buffer, so look to see if we have received a newline - # and keep doing so until all buffered lines have been processed. - while buffers[io].index("\r\n") - # Extract the line - line, buffers[io] = buffers[io].split("\r\n", 2) - - # Send the received line to the client object for processing - result = client.handle(line) - # If the client object returned some data, write it back to the client - next if result.nil? - - result = [result] unless result.is_a?(Array) - result.compact.each do |iline| - client.log "\e[34m=> #{iline.strip}\e[0m" - begin - io.write(iline.to_s + "\r\n") - io.flush - rescue Errno::ECONNRESET - # Client disconnected before we could write response - eof = true - end - end - end - - # Did the client request STARTTLS? - if !eof && client.start_tls? - # Deregister the unencrypted IO - @io_selector.deregister(io) - buffers.delete(io) - io = OpenSSL::SSL::SSLSocket.new(io, ssl_context) - # Close the underlying IO when the TLS socket is closed - io.sync_close = true - # Register the new TLS socket with nio - monitor = @io_selector.register(io, :r) - monitor.value = client - end - end - - # Has the client requested we close the connection? - if client.finished? || eof - client.log "\e[35m Connection closed\e[0m" - # Deregister the socket and close it - @io_selector.deregister(io) - buffers.delete(io) - io.close - # If we have no more clients or listeners left, exit the process - if @io_selector.empty? - Process.exit(0) - end - end - rescue StandardError => e - # Something went wrong, log as appropriate - client_id = client ? client.id : "------" - if defined?(Sentry) - Sentry.capture_exception(e, extra: { log_id: begin - client.id - rescue StandardError - nil - end }) - end - logger.error "[#{client_id}] An error occurred while processing data from a client." - logger.error "[#{client_id}] #{e.class}: #{e.message}" - e.backtrace.each do |iline| - logger.error "[#{client_id}] #{iline}" - end - # Close all IO and forget this client - begin - @io_selector.deregister(io) - rescue StandardError - nil - end - buffers.delete(io) - begin - io.close - rescue StandardError - nil - end - if @io_selector.empty? - Process.exit(0) - end - end - end - end - # If unlisten has been called, stop listening - next unless @unlisten - - @io_selector.deregister(@server) - @server.close - # If there's nothing left to do, shut down the process - if @io_selector.empty? - Process.exit(0) - end - # Clear the request - @unlisten = false - end - end - - def logger - Postal.logger - end - - end - end -end diff --git a/script/smtp_server.rb b/script/smtp_server.rb index f9ea52f..69211ef 100644 --- a/script/smtp_server.rb +++ b/script/smtp_server.rb @@ -1,4 +1,4 @@ # frozen_string_literal: true require_relative "../config/environment" -Postal::SMTPServer::Server.new(debug: true).run +SMTPServer::Server.new(debug: true).run diff --git a/spec/lib/postal/smtp_server/client/auth_spec.rb b/spec/lib/postal/smtp_server/client/auth_spec.rb deleted file mode 100644 index bdb271d..0000000 --- a/spec/lib/postal/smtp_server/client/auth_spec.rb +++ /dev/null @@ -1,122 +0,0 @@ -# frozen_string_literal: true - -require "rails_helper" - -module Postal - module SMTPServer - - describe Client do - let(:ip_address) { "1.2.3.4" } - subject(:client) { described_class.new(ip_address) } - - before do - client.handle("HELO test.example.com") - end - - describe "AUTH PLAIN" do - context "when no credentials are provided on the initial data" do - it "returns a 334" do - expect(client.handle("AUTH PLAIN")).to eq("334") - end - - it "accepts the username and password from the next input" do - client.handle("AUTH PLAIN") - credential = create(:credential, type: "SMTP") - expect(client.handle(credential.to_smtp_plain)).to match(/235 Granted for/) - end - end - - context "when valid credentials are provided on one line" do - it "authenticates and returns a response" do - credential = create(:credential, type: "SMTP") - expect(client.handle("AUTH PLAIN #{credential.to_smtp_plain}")).to match(/235 Granted for/) - expect(client.credential).to eq credential - end - end - - context "when invalid credentials are provided" do - it "returns an error and resets the state" do - base64 = Base64.encode64("user\0pass") - expect(client.handle("AUTH PLAIN #{base64}")).to eq("535 Invalid credential") - expect(client.state).to eq :welcomed - end - end - - context "when username or password is missing" do - it "returns an error and resets the state" do - base64 = Base64.encode64("pass") - expect(client.handle("AUTH PLAIN #{base64}")).to eq("535 Authenticated failed - protocol error") - expect(client.state).to eq :welcomed - end - end - end - - describe "AUTH LOGIN" do - context "when no username is provided on the first line" do - it "requests the username" do - expect(client.handle("AUTH LOGIN")).to eq("334 VXNlcm5hbWU6") - end - end - - context "when a username is provided on the first line" do - it "requests a password" do - username = Base64.encode64("xx") - expect(client.handle("AUTH LOGIN #{username}")).to eq("334 UGFzc3dvcmQ6") - end - - it "authenticates and returns a response" do - credential = create(:credential, type: "SMTP") - username = Base64.encode64("xx") - password = Base64.encode64(credential.key) - expect(client.handle("AUTH LOGIN #{username}")).to eq("334 UGFzc3dvcmQ6") - expect(client.handle(password)).to match(/235 Granted for/) - expect(client.credential).to eq credential - end - end - - context "when invalid credentials are provided" do - it "returns an error and resets the state" do - username = Base64.encode64("xx") - password = Base64.encode64("xx") - expect(client.handle("AUTH LOGIN #{username}")).to eq("334 UGFzc3dvcmQ6") - expect(client.handle(password)).to eq("535 Invalid credential") - expect(client.state).to eq :welcomed - end - end - end - - describe "AUTH CRAM-MD5" do - context "when valid credentials are provided" do - it "authenticates and returns a response" do - credential = create(:credential, type: "SMTP") - result = client.handle("AUTH CRAM-MD5") - expect(result).to match(/\A334 [A-Za-z0-9=]+\z/) - challenge = Base64.decode64(result.split[1]) - password = OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new("md5"), credential.key, challenge) - base64 = Base64.encode64("#{credential.server.organization.permalink}/#{credential.server.permalink} #{password}") - expect(client.handle(base64)).to match(/235 Granted for/) - expect(client.credential).to eq credential - end - end - - context "when no org/server matches the provided username" do - it "returns an error" do - client.handle("AUTH CRAM-MD5") - base64 = Base64.encode64("org/server password") - expect(client.handle(base64)).to eq "535 Denied" - end - end - - context "when invalid credentials are provided" do - it "returns an error and resets the state" do - server = create(:server) - base64 = Base64.encode64("#{server.organization.permalink}/#{server.permalink} invalid-password") - client.handle("AUTH CRAM-MD5") - expect(client.handle(base64)).to eq("535 Denied") - end - end - end - end - - end -end diff --git a/spec/lib/postal/smtp_server/client/data_spec.rb b/spec/lib/postal/smtp_server/client/data_spec.rb deleted file mode 100644 index b4a9d8f..0000000 --- a/spec/lib/postal/smtp_server/client/data_spec.rb +++ /dev/null @@ -1,89 +0,0 @@ -# frozen_string_literal: true - -require "rails_helper" - -module Postal - module SMTPServer - - describe Client do - let(:ip_address) { "1.2.3.4" } - subject(:client) { described_class.new(ip_address) } - - describe "DATA" do - it "returns an error if no helo" do - expect(client.handle("DATA")).to eq "503 HELO/EHLO, MAIL FROM and RCPT TO before sending data" - end - - it "returns an error if no mail from" do - client.handle("HELO test.example.com") - expect(client.handle("DATA")).to eq "503 HELO/EHLO, MAIL FROM and RCPT TO before sending data" - end - - it "returns an error if no rcpt to" do - client.handle("HELO test.example.com") - client.handle("MAIL FROM: test@example.com") - expect(client.handle("DATA")).to eq "503 HELO/EHLO, MAIL FROM and RCPT TO before sending data" - end - - it "returns go ahead" do - route = create(:route) - client.handle("HELO test.example.com") - client.handle("MAIL FROM: test@test.com") - client.handle("RCPT TO: #{route.name}@#{route.domain.name}") - expect(client.handle("DATA")).to eq "354 Go ahead" - end - - it "adds a received header for itself" do - route = create(:route) - client.handle("HELO test.example.com") - client.handle("MAIL FROM: test@test.com") - client.handle("RCPT TO: #{route.name}@#{route.domain.name}") - Timecop.freeze do - client.handle("DATA") - expect(client.headers["received"]).to include "from test.example.com (1.2.3.4 [1.2.3.4]) by postal.example.com with SMTP; #{Time.now.utc.rfc2822}" - end - end - - describe "subsequent commands" do - let(:route) { create(:route) } - before do - client.handle("HELO test.example.com") - client.handle("MAIL FROM: test@test.com") - client.handle("RCPT TO: #{route.name}@#{route.domain.name}") - end - - it "logs headers" do - client.handle("DATA") - client.handle("Subject: Test") - client.handle("From: test@test.com") - client.handle("To: test1@example.com") - client.handle("To: test2@example.com") - client.handle("X-Something: abcdef1234") - expect(client.headers["subject"]).to eq ["Test"] - expect(client.headers["from"]).to eq ["test@test.com"] - expect(client.headers["to"]).to eq ["test1@example.com", "test2@example.com"] - expect(client.headers["x-something"]).to eq ["abcdef1234"] - end - - it "logs content" do - Timecop.freeze do - client.handle("DATA") - client.handle("Subject: Test") - client.handle("") - client.handle("This is some content for the message.") - client.handle("It will keep going.") - expect(client.instance_variable_get("@data")).to eq <<~DATA - Received: from test.example.com (1.2.3.4 [1.2.3.4]) by #{Postal.config.dns.smtp_server_hostname} with SMTP; #{Time.now.utc.rfc2822}\r - Subject: Test\r - \r - This is some content for the message.\r - It will keep going.\r - DATA - end - end - end - end - end - - end -end diff --git a/spec/lib/postal/smtp_server/client/finished_spec.rb b/spec/lib/postal/smtp_server/client/finished_spec.rb deleted file mode 100644 index aa6b48c..0000000 --- a/spec/lib/postal/smtp_server/client/finished_spec.rb +++ /dev/null @@ -1,208 +0,0 @@ -# frozen_string_literal: true - -require "rails_helper" - -module Postal - module SMTPServer - - describe Client do - let(:ip_address) { "1.2.3.4" } - let(:server) { create(:server) } - subject(:client) { described_class.new(ip_address) } - - let(:credential) { create(:credential, server: server, type: "SMTP") } - let(:auth_plain) { credential&.to_smtp_plain } - let(:mail_from) { "test@example.com" } - let(:rcpt_to) { "test@example.com" } - - before do - client.handle("HELO test.example.com") - client.handle("AUTH PLAIN #{auth_plain}") if auth_plain - client.handle("MAIL FROM: #{mail_from}") - client.handle("RCPT TO: #{rcpt_to}") - end - - describe "when finished sending data" do - context "when the data is larger than the maximum message size" do - it "returns an error and resets the state" do - allow(Postal.config.smtp_server).to receive(:max_message_size).and_return(1) - client.handle("DATA") - client.handle("a" * 1024 * 1024 * 10) - expect(client.handle(".")).to eq "552 Message too large (maximum size 1MB)" - end - end - - context "when a loop is detected" do - it "returns an error and resets the state" do - client.handle("DATA") - client.handle("Received: from example1.com by #{Postal.config.dns.smtp_server_hostname}") - client.handle("Received: from example2.com by #{Postal.config.dns.smtp_server_hostname}") - client.handle("Received: from example1.com by #{Postal.config.dns.smtp_server_hostname}") - client.handle("Received: from example2.com by #{Postal.config.dns.smtp_server_hostname}") - client.handle("Subject: Test") - client.handle("From: #{mail_from}") - client.handle("To: #{rcpt_to}") - client.handle("") - client.handle("This is a test message") - expect(client.handle(".")).to eq "550 Loop detected" - end - end - - context "when the email content is not suitable for the credential" do - it "returns an error and resets the state" do - client.handle("DATA") - client.handle("Subject: Test") - client.handle("From: invalid@krystal.uk") - client.handle("To: #{rcpt_to}") - client.handle("") - client.handle("This is a test message") - expect(client.handle(".")).to eq "530 From/Sender name is not valid" - end - end - - context "when sending an outgoing email" do - let(:domain) { create(:domain, owner: server) } - let(:mail_from) { "test@#{domain.name}" } - let(:auth_plain) { credential.to_smtp_plain } - - it "stores the message and resets the state" do - client.handle("DATA") - client.handle("Subject: Test") - client.handle("From: #{mail_from}") - client.handle("To: #{rcpt_to}") - client.handle("") - client.handle("This is a test message") - expect(client.handle(".")).to eq "250 OK" - queued_message = QueuedMessage.first - expect(queued_message).to have_attributes( - domain: "example.com", - server: server - ) - - expect(server.message(queued_message.message_id)).to have_attributes( - mail_from: mail_from, - rcpt_to: rcpt_to, - subject: "Test", - scope: "outgoing", - route_id: nil, - credential_id: credential.id, - raw_headers: kind_of(String), - raw_message: kind_of(String) - ) - end - end - - context "when sending a bounce message" do - let(:credential) { nil } - let(:rcpt_to) { "#{server.token}@#{Postal.config.dns.return_path}" } - - context "when there is a return path route" do - let(:domain) { create(:domain, owner: server) } - - before do - endpoint = create(:http_endpoint, server: server) - create(:route, domain: domain, server: server, name: "__returnpath__", mode: "Endpoint", endpoint: endpoint) - end - - it "stores the message for the return path route and resets the state" do - client.handle("DATA") - client.handle("Subject: Bounce: Test") - client.handle("From: #{mail_from}") - client.handle("To: #{rcpt_to}") - client.handle("") - client.handle("This is a test message") - expect(client.handle(".")).to eq "250 OK" - - queued_message = QueuedMessage.first - expect(queued_message).to have_attributes( - domain: Postal.config.dns.return_path, - server: server - ) - - expect(server.message(queued_message.message_id)).to have_attributes( - mail_from: mail_from, - rcpt_to: rcpt_to, - subject: "Bounce: Test", - scope: "incoming", - route_id: server.routes.first.id, - domain_id: domain.id, - credential_id: nil, - raw_headers: kind_of(String), - raw_message: kind_of(String), - bounce: true - ) - end - end - - context "when there is no return path route" do - it "stores the message normally and resets the state" do - client.handle("DATA") - client.handle("Subject: Bounce: Test") - client.handle("From: #{mail_from}") - client.handle("To: #{rcpt_to}") - client.handle("") - client.handle("This is a test message") - expect(client.handle(".")).to eq "250 OK" - - queued_message = QueuedMessage.first - expect(queued_message).to have_attributes( - domain: Postal.config.dns.return_path, - server: server - ) - - expect(server.message(queued_message.message_id)).to have_attributes( - mail_from: mail_from, - rcpt_to: rcpt_to, - subject: "Bounce: Test", - scope: "incoming", - route_id: nil, - domain_id: nil, - credential_id: nil, - raw_headers: kind_of(String), - raw_message: kind_of(String), - bounce: true - ) - end - end - end - - context "when receiving an incoming email" do - let(:domain) { create(:domain, owner: server) } - let(:route) { create(:route, server: server, domain: domain) } - - let(:credential) { nil } - let(:rcpt_to) { "#{route.name}@#{domain.name}" } - - it "stores the message and resets the state" do - client.handle("DATA") - client.handle("Subject: Test") - client.handle("From: #{mail_from}") - client.handle("To: #{rcpt_to}") - client.handle("") - client.handle("This is a test message") - expect(client.handle(".")).to eq "250 OK" - - queued_message = QueuedMessage.first - expect(queued_message).to have_attributes( - domain: domain.name, - server: server - ) - - expect(server.message(queued_message.message_id)).to have_attributes( - mail_from: mail_from, - rcpt_to: rcpt_to, - subject: "Test", - scope: "incoming", - route_id: route.id, - domain_id: domain.id, - credential_id: nil, - raw_headers: kind_of(String), - raw_message: kind_of(String) - ) - end - end - end - end - - end -end diff --git a/spec/lib/postal/smtp_server/client/helo_spec.rb b/spec/lib/postal/smtp_server/client/helo_spec.rb deleted file mode 100644 index 85d513b..0000000 --- a/spec/lib/postal/smtp_server/client/helo_spec.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -require "rails_helper" - -module Postal - module SMTPServer - - describe Client do - let(:ip_address) { "1.2.3.4" } - subject(:client) { described_class.new(ip_address) } - - describe "HELO" do - it "returns the hostname" do - expect(client.state).to eq :welcome - expect(client.handle("HELO: test.example.com")).to eq "250 #{Postal.config.dns.smtp_server_hostname}" - expect(client.state).to eq :welcomed - end - end - - describe "EHLO" do - it "returns the capabilities" do - expect(client.handle("EHLO test.example.com")).to eq ["250-My capabilities are", - "250 AUTH CRAM-MD5 PLAIN LOGIN"] - end - - context "when TLS is enabled" do - it "returns capabilities include starttls" do - allow(Postal.config.smtp_server).to receive(:tls_enabled?).and_return(true) - expect(client.handle("EHLO test.example.com")).to eq ["250-My capabilities are", - "250-STARTTLS", - "250 AUTH CRAM-MD5 PLAIN LOGIN"] - end - end - end - end - - end -end diff --git a/spec/lib/postal/smtp_server/client/mail_from_spec.rb b/spec/lib/postal/smtp_server/client/mail_from_spec.rb deleted file mode 100644 index b62de45..0000000 --- a/spec/lib/postal/smtp_server/client/mail_from_spec.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -require "rails_helper" - -module Postal - module SMTPServer - - describe Client do - let(:ip_address) { "1.2.3.4" } - subject(:client) { described_class.new(ip_address) } - - describe "MAIL FROM" do - it "returns an error if no HELO is provided" do - expect(client.handle("MAIL FROM: test@example.com")).to eq "503 EHLO/HELO first please" - expect(client.state).to eq :welcome - end - - it "resets the transaction when called" do - expect(client).to receive(:transaction_reset).and_call_original.at_least(3).times - client.handle("HELO test.example.com") - client.handle("MAIL FROM: test@example.com") - client.handle("MAIL FROM: test2@example.com") - end - - it "sets the mail from address" do - client.handle("HELO test.example.com") - expect(client.handle("MAIL FROM: test@example.com")).to eq "250 OK" - expect(client.state).to eq :mail_from_received - expect(client.instance_variable_get("@mail_from")).to eq "test@example.com" - end - end - end - - end -end diff --git a/spec/lib/postal/smtp_server/client/rcpt_to_spec.rb b/spec/lib/postal/smtp_server/client/rcpt_to_spec.rb deleted file mode 100644 index ee35abb..0000000 --- a/spec/lib/postal/smtp_server/client/rcpt_to_spec.rb +++ /dev/null @@ -1,172 +0,0 @@ -# frozen_string_literal: true - -require "rails_helper" - -module Postal - module SMTPServer - - describe Client do - let(:ip_address) { "1.2.3.4" } - subject(:client) { described_class.new(ip_address) } - - describe "RCPT TO" do - let(:helo) { "test.example.com" } - let(:mail_from) { "test@example.com" } - - before do - client.handle("HELO #{helo}") - client.handle("MAIL FROM: #{mail_from}") if mail_from - end - - context "when MAIL FROM has not been sent" do - let(:mail_from) { nil } - - it "returns an error if RCPT TO is sent before MAIL FROM" do - expect(client.handle("RCPT TO: no-route-here@internal.com")).to eq "503 EHLO/HELO and MAIL FROM first please" - expect(client.state).to eq :welcomed - end - end - - it "returns an error if RCPT TO is not valid" do - expect(client.handle("RCPT TO: blah")).to eq "501 Invalid RCPT TO" - end - - it "returns an error if RCPT TO is empty" do - expect(client.handle("RCPT TO: ")).to eq "501 RCPT TO should not be empty" - end - - context "when the RCPT TO address is the system return path host" do - it "returns an error if the server does not exist" do - expect(client.handle("RCPT TO: nothing@#{Postal.config.dns.return_path}")).to eq "550 Invalid server token" - end - - it "returns an error if the server is suspended" do - server = create(:server, :suspended) - expect(client.handle("RCPT TO: #{server.token}@#{Postal.config.dns.return_path}")) - .to eq "535 Mail server has been suspended" - end - - it "adds a recipient if all OK" do - server = create(:server) - address = "#{server.token}@#{Postal.config.dns.return_path}" - expect(client.handle("RCPT TO: #{address}")).to eq "250 OK" - expect(client.recipients).to eq [[:bounce, address, server]] - expect(client.state).to eq :rcpt_to_received - end - end - - context "when the RCPT TO address is on a host using the return path prefix" do - it "returns an error if the server does not exist" do - address = "nothing@#{Postal.config.dns.custom_return_path_prefix}.example.com" - expect(client.handle("RCPT TO: #{address}")).to eq "550 Invalid server token" - end - - it "returns an error if the server is suspended" do - server = create(:server, :suspended) - address = "#{server.token}@#{Postal.config.dns.custom_return_path_prefix}.example.com" - expect(client.handle("RCPT TO: #{address}")).to eq "535 Mail server has been suspended" - end - - it "adds a recipient if all OK" do - server = create(:server) - address = "#{server.token}@#{Postal.config.dns.custom_return_path_prefix}.example.com" - expect(client.handle("RCPT TO: #{address}")).to eq "250 OK" - expect(client.recipients).to eq [[:bounce, address, server]] - expect(client.state).to eq :rcpt_to_received - end - end - - context "when the RCPT TO address is within the route domain" do - it "returns an error if the route token is invalid" do - address = "nothing@#{Postal.config.dns.route_domain}" - expect(client.handle("RCPT TO: #{address}")).to eq "550 Invalid route token" - end - - it "returns an error if the server is suspended" do - server = create(:server, :suspended) - route = create(:route, server: server) - address = "#{route.token}@#{Postal.config.dns.route_domain}" - expect(client.handle("RCPT TO: #{address}")).to eq "535 Mail server has been suspended" - end - - it "returns an error if the route is set to Reject mail" do - server = create(:server) - route = create(:route, server: server, mode: "Reject") - address = "#{route.token}@#{Postal.config.dns.route_domain}" - expect(client.handle("RCPT TO: #{address}")).to eq "550 Route does not accept incoming messages" - end - - it "adds a recipient if all OK" do - server = create(:server) - route = create(:route, server: server) - address = "#{route.token}+tag1@#{Postal.config.dns.route_domain}" - expect(client.handle("RCPT TO: #{address}")).to eq "250 OK" - expect(client.recipients).to eq [[:route, "#{route.name}+tag1@#{route.domain.name}", server, { route: route }]] - expect(client.state).to eq :rcpt_to_received - end - end - - context "when authenticated and the RCPT TO address is provided" do - it "returns an error if the server is suspended" do - server = create(:server, :suspended) - credential = create(:credential, server: server, type: "SMTP") - expect(client.handle("AUTH PLAIN #{credential.to_smtp_plain}")).to match(/235 Granted for /) - expect(client.handle("RCPT TO: outgoing@example.com")).to eq "535 Mail server has been suspended" - end - - it "adds a recipient if all OK" do - server = create(:server) - credential = create(:credential, server: server, type: "SMTP") - expect(client.handle("AUTH PLAIN #{credential.to_smtp_plain}")).to match(/235 Granted for /) - expect(client.handle("RCPT TO: outgoing@example.com")).to eq "250 OK" - expect(client.recipients).to eq [[:credential, "outgoing@example.com", server]] - expect(client.state).to eq :rcpt_to_received - end - end - - context "when not authenticated and the RCPT TO address is a route" do - it "returns an error if the server is suspended" do - server = create(:server, :suspended) - route = create(:route, server: server) - address = "#{route.name}@#{route.domain.name}" - expect(client.handle("RCPT TO: #{address}")).to eq "535 Mail server has been suspended" - end - - it "returns an error if the route is set to Reject mail" do - server = create(:server) - route = create(:route, server: server, mode: "Reject") - address = "#{route.name}@#{route.domain.name}" - expect(client.handle("RCPT TO: #{address}")).to eq "550 Route does not accept incoming messages" - end - - it "adds a recipient if all OK" do - server = create(:server) - route = create(:route, server: server) - address = "#{route.name}@#{route.domain.name}" - expect(client.handle("RCPT TO: #{address}")).to eq "250 OK" - expect(client.recipients).to eq [[:route, address, server, { route: route }]] - expect(client.state).to eq :rcpt_to_received - end - end - - context "when not authenticated and RCPT TO does not match a route" do - it "returns an error" do - expect(client.handle("RCPT TO: nothing@nothing.com")).to eq "530 Authentication required" - end - - context "when the connecting IP has an credential" do - it "adds a recipient" do - server = create(:server) - create(:credential, server: server, type: "SMTP-IP", key: "1.0.0.0/8") - address = "test@example.com" - expect(client.handle("RCPT TO: #{address}")).to eq "250 OK" - expect(client.recipients).to eq [[:credential, address, server]] - expect(client.state).to eq :rcpt_to_received - end - end - end - end - end - - end -end diff --git a/spec/lib/postal/smtp_server/client_spec.rb b/spec/lib/postal/smtp_server/client_spec.rb deleted file mode 100644 index 0c9261b..0000000 --- a/spec/lib/postal/smtp_server/client_spec.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -require "rails_helper" - -module Postal - module SMTPServer - - describe Client do - let(:ip_address) { "1.2.3.4" } - subject(:client) { described_class.new(ip_address) } - end - - end -end diff --git a/spec/lib/smtp_server/client/auth_spec.rb b/spec/lib/smtp_server/client/auth_spec.rb new file mode 100644 index 0000000..edf9e50 --- /dev/null +++ b/spec/lib/smtp_server/client/auth_spec.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +require "rails_helper" + +module SMTPServer + + describe Client do + let(:ip_address) { "1.2.3.4" } + subject(:client) { described_class.new(ip_address) } + + before do + client.handle("HELO test.example.com") + end + + describe "AUTH PLAIN" do + context "when no credentials are provided on the initial data" do + it "returns a 334" do + expect(client.handle("AUTH PLAIN")).to eq("334") + end + + it "accepts the username and password from the next input" do + client.handle("AUTH PLAIN") + credential = create(:credential, type: "SMTP") + expect(client.handle(credential.to_smtp_plain)).to match(/235 Granted for/) + end + end + + context "when valid credentials are provided on one line" do + it "authenticates and returns a response" do + credential = create(:credential, type: "SMTP") + expect(client.handle("AUTH PLAIN #{credential.to_smtp_plain}")).to match(/235 Granted for/) + expect(client.credential).to eq credential + end + end + + context "when invalid credentials are provided" do + it "returns an error and resets the state" do + base64 = Base64.encode64("user\0pass") + expect(client.handle("AUTH PLAIN #{base64}")).to eq("535 Invalid credential") + expect(client.state).to eq :welcomed + end + end + + context "when username or password is missing" do + it "returns an error and resets the state" do + base64 = Base64.encode64("pass") + expect(client.handle("AUTH PLAIN #{base64}")).to eq("535 Authenticated failed - protocol error") + expect(client.state).to eq :welcomed + end + end + end + + describe "AUTH LOGIN" do + context "when no username is provided on the first line" do + it "requests the username" do + expect(client.handle("AUTH LOGIN")).to eq("334 VXNlcm5hbWU6") + end + end + + context "when a username is provided on the first line" do + it "requests a password" do + username = Base64.encode64("xx") + expect(client.handle("AUTH LOGIN #{username}")).to eq("334 UGFzc3dvcmQ6") + end + + it "authenticates and returns a response" do + credential = create(:credential, type: "SMTP") + username = Base64.encode64("xx") + password = Base64.encode64(credential.key) + expect(client.handle("AUTH LOGIN #{username}")).to eq("334 UGFzc3dvcmQ6") + expect(client.handle(password)).to match(/235 Granted for/) + expect(client.credential).to eq credential + end + end + + context "when invalid credentials are provided" do + it "returns an error and resets the state" do + username = Base64.encode64("xx") + password = Base64.encode64("xx") + expect(client.handle("AUTH LOGIN #{username}")).to eq("334 UGFzc3dvcmQ6") + expect(client.handle(password)).to eq("535 Invalid credential") + expect(client.state).to eq :welcomed + end + end + end + + describe "AUTH CRAM-MD5" do + context "when valid credentials are provided" do + it "authenticates and returns a response" do + credential = create(:credential, type: "SMTP") + result = client.handle("AUTH CRAM-MD5") + expect(result).to match(/\A334 [A-Za-z0-9=]+\z/) + challenge = Base64.decode64(result.split[1]) + password = OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new("md5"), credential.key, challenge) + base64 = Base64.encode64("#{credential.server.organization.permalink}/#{credential.server.permalink} #{password}") + expect(client.handle(base64)).to match(/235 Granted for/) + expect(client.credential).to eq credential + end + end + + context "when no org/server matches the provided username" do + it "returns an error" do + client.handle("AUTH CRAM-MD5") + base64 = Base64.encode64("org/server password") + expect(client.handle(base64)).to eq "535 Denied" + end + end + + context "when invalid credentials are provided" do + it "returns an error and resets the state" do + server = create(:server) + base64 = Base64.encode64("#{server.organization.permalink}/#{server.permalink} invalid-password") + client.handle("AUTH CRAM-MD5") + expect(client.handle(base64)).to eq("535 Denied") + end + end + end + end + +end diff --git a/spec/lib/smtp_server/client/data_spec.rb b/spec/lib/smtp_server/client/data_spec.rb new file mode 100644 index 0000000..66ae280 --- /dev/null +++ b/spec/lib/smtp_server/client/data_spec.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +require "rails_helper" + +module SMTPServer + + describe Client do + let(:ip_address) { "1.2.3.4" } + subject(:client) { described_class.new(ip_address) } + + describe "DATA" do + it "returns an error if no helo" do + expect(client.handle("DATA")).to eq "503 HELO/EHLO, MAIL FROM and RCPT TO before sending data" + end + + it "returns an error if no mail from" do + client.handle("HELO test.example.com") + expect(client.handle("DATA")).to eq "503 HELO/EHLO, MAIL FROM and RCPT TO before sending data" + end + + it "returns an error if no rcpt to" do + client.handle("HELO test.example.com") + client.handle("MAIL FROM: test@example.com") + expect(client.handle("DATA")).to eq "503 HELO/EHLO, MAIL FROM and RCPT TO before sending data" + end + + it "returns go ahead" do + route = create(:route) + client.handle("HELO test.example.com") + client.handle("MAIL FROM: test@test.com") + client.handle("RCPT TO: #{route.name}@#{route.domain.name}") + expect(client.handle("DATA")).to eq "354 Go ahead" + end + + it "adds a received header for itself" do + route = create(:route) + client.handle("HELO test.example.com") + client.handle("MAIL FROM: test@test.com") + client.handle("RCPT TO: #{route.name}@#{route.domain.name}") + Timecop.freeze do + client.handle("DATA") + expect(client.headers["received"]).to include "from test.example.com (1.2.3.4 [1.2.3.4]) by postal.example.com with SMTP; #{Time.now.utc.rfc2822}" + end + end + + describe "subsequent commands" do + let(:route) { create(:route) } + before do + client.handle("HELO test.example.com") + client.handle("MAIL FROM: test@test.com") + client.handle("RCPT TO: #{route.name}@#{route.domain.name}") + end + + it "logs headers" do + client.handle("DATA") + client.handle("Subject: Test") + client.handle("From: test@test.com") + client.handle("To: test1@example.com") + client.handle("To: test2@example.com") + client.handle("X-Something: abcdef1234") + expect(client.headers["subject"]).to eq ["Test"] + expect(client.headers["from"]).to eq ["test@test.com"] + expect(client.headers["to"]).to eq ["test1@example.com", "test2@example.com"] + expect(client.headers["x-something"]).to eq ["abcdef1234"] + end + + it "logs content" do + Timecop.freeze do + client.handle("DATA") + client.handle("Subject: Test") + client.handle("") + client.handle("This is some content for the message.") + client.handle("It will keep going.") + expect(client.instance_variable_get("@data")).to eq <<~DATA + Received: from test.example.com (1.2.3.4 [1.2.3.4]) by #{Postal.config.dns.smtp_server_hostname} with SMTP; #{Time.now.utc.rfc2822}\r + Subject: Test\r + \r + This is some content for the message.\r + It will keep going.\r + DATA + end + end + end + end + end + +end diff --git a/spec/lib/smtp_server/client/finished_spec.rb b/spec/lib/smtp_server/client/finished_spec.rb new file mode 100644 index 0000000..5985fad --- /dev/null +++ b/spec/lib/smtp_server/client/finished_spec.rb @@ -0,0 +1,206 @@ +# frozen_string_literal: true + +require "rails_helper" + +module SMTPServer + + describe Client do + let(:ip_address) { "1.2.3.4" } + let(:server) { create(:server) } + subject(:client) { described_class.new(ip_address) } + + let(:credential) { create(:credential, server: server, type: "SMTP") } + let(:auth_plain) { credential&.to_smtp_plain } + let(:mail_from) { "test@example.com" } + let(:rcpt_to) { "test@example.com" } + + before do + client.handle("HELO test.example.com") + client.handle("AUTH PLAIN #{auth_plain}") if auth_plain + client.handle("MAIL FROM: #{mail_from}") + client.handle("RCPT TO: #{rcpt_to}") + end + + describe "when finished sending data" do + context "when the data is larger than the maximum message size" do + it "returns an error and resets the state" do + allow(Postal.config.smtp_server).to receive(:max_message_size).and_return(1) + client.handle("DATA") + client.handle("a" * 1024 * 1024 * 10) + expect(client.handle(".")).to eq "552 Message too large (maximum size 1MB)" + end + end + + context "when a loop is detected" do + it "returns an error and resets the state" do + client.handle("DATA") + client.handle("Received: from example1.com by #{Postal.config.dns.smtp_server_hostname}") + client.handle("Received: from example2.com by #{Postal.config.dns.smtp_server_hostname}") + client.handle("Received: from example1.com by #{Postal.config.dns.smtp_server_hostname}") + client.handle("Received: from example2.com by #{Postal.config.dns.smtp_server_hostname}") + client.handle("Subject: Test") + client.handle("From: #{mail_from}") + client.handle("To: #{rcpt_to}") + client.handle("") + client.handle("This is a test message") + expect(client.handle(".")).to eq "550 Loop detected" + end + end + + context "when the email content is not suitable for the credential" do + it "returns an error and resets the state" do + client.handle("DATA") + client.handle("Subject: Test") + client.handle("From: invalid@krystal.uk") + client.handle("To: #{rcpt_to}") + client.handle("") + client.handle("This is a test message") + expect(client.handle(".")).to eq "530 From/Sender name is not valid" + end + end + + context "when sending an outgoing email" do + let(:domain) { create(:domain, owner: server) } + let(:mail_from) { "test@#{domain.name}" } + let(:auth_plain) { credential.to_smtp_plain } + + it "stores the message and resets the state" do + client.handle("DATA") + client.handle("Subject: Test") + client.handle("From: #{mail_from}") + client.handle("To: #{rcpt_to}") + client.handle("") + client.handle("This is a test message") + expect(client.handle(".")).to eq "250 OK" + queued_message = QueuedMessage.first + expect(queued_message).to have_attributes( + domain: "example.com", + server: server + ) + + expect(server.message(queued_message.message_id)).to have_attributes( + mail_from: mail_from, + rcpt_to: rcpt_to, + subject: "Test", + scope: "outgoing", + route_id: nil, + credential_id: credential.id, + raw_headers: kind_of(String), + raw_message: kind_of(String) + ) + end + end + + context "when sending a bounce message" do + let(:credential) { nil } + let(:rcpt_to) { "#{server.token}@#{Postal.config.dns.return_path}" } + + context "when there is a return path route" do + let(:domain) { create(:domain, owner: server) } + + before do + endpoint = create(:http_endpoint, server: server) + create(:route, domain: domain, server: server, name: "__returnpath__", mode: "Endpoint", endpoint: endpoint) + end + + it "stores the message for the return path route and resets the state" do + client.handle("DATA") + client.handle("Subject: Bounce: Test") + client.handle("From: #{mail_from}") + client.handle("To: #{rcpt_to}") + client.handle("") + client.handle("This is a test message") + expect(client.handle(".")).to eq "250 OK" + + queued_message = QueuedMessage.first + expect(queued_message).to have_attributes( + domain: Postal.config.dns.return_path, + server: server + ) + + expect(server.message(queued_message.message_id)).to have_attributes( + mail_from: mail_from, + rcpt_to: rcpt_to, + subject: "Bounce: Test", + scope: "incoming", + route_id: server.routes.first.id, + domain_id: domain.id, + credential_id: nil, + raw_headers: kind_of(String), + raw_message: kind_of(String), + bounce: true + ) + end + end + + context "when there is no return path route" do + it "stores the message normally and resets the state" do + client.handle("DATA") + client.handle("Subject: Bounce: Test") + client.handle("From: #{mail_from}") + client.handle("To: #{rcpt_to}") + client.handle("") + client.handle("This is a test message") + expect(client.handle(".")).to eq "250 OK" + + queued_message = QueuedMessage.first + expect(queued_message).to have_attributes( + domain: Postal.config.dns.return_path, + server: server + ) + + expect(server.message(queued_message.message_id)).to have_attributes( + mail_from: mail_from, + rcpt_to: rcpt_to, + subject: "Bounce: Test", + scope: "incoming", + route_id: nil, + domain_id: nil, + credential_id: nil, + raw_headers: kind_of(String), + raw_message: kind_of(String), + bounce: true + ) + end + end + end + + context "when receiving an incoming email" do + let(:domain) { create(:domain, owner: server) } + let(:route) { create(:route, server: server, domain: domain) } + + let(:credential) { nil } + let(:rcpt_to) { "#{route.name}@#{domain.name}" } + + it "stores the message and resets the state" do + client.handle("DATA") + client.handle("Subject: Test") + client.handle("From: #{mail_from}") + client.handle("To: #{rcpt_to}") + client.handle("") + client.handle("This is a test message") + expect(client.handle(".")).to eq "250 OK" + + queued_message = QueuedMessage.first + expect(queued_message).to have_attributes( + domain: domain.name, + server: server + ) + + expect(server.message(queued_message.message_id)).to have_attributes( + mail_from: mail_from, + rcpt_to: rcpt_to, + subject: "Test", + scope: "incoming", + route_id: route.id, + domain_id: domain.id, + credential_id: nil, + raw_headers: kind_of(String), + raw_message: kind_of(String) + ) + end + end + end + end + +end diff --git a/spec/lib/smtp_server/client/helo_spec.rb b/spec/lib/smtp_server/client/helo_spec.rb new file mode 100644 index 0000000..7a39b8e --- /dev/null +++ b/spec/lib/smtp_server/client/helo_spec.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require "rails_helper" + +module SMTPServer + + describe Client do + let(:ip_address) { "1.2.3.4" } + subject(:client) { described_class.new(ip_address) } + + describe "HELO" do + it "returns the hostname" do + expect(client.state).to eq :welcome + expect(client.handle("HELO: test.example.com")).to eq "250 #{Postal.config.dns.smtp_server_hostname}" + expect(client.state).to eq :welcomed + end + end + + describe "EHLO" do + it "returns the capabilities" do + expect(client.handle("EHLO test.example.com")).to eq ["250-My capabilities are", + "250 AUTH CRAM-MD5 PLAIN LOGIN"] + end + + context "when TLS is enabled" do + it "returns capabilities include starttls" do + allow(Postal.config.smtp_server).to receive(:tls_enabled?).and_return(true) + expect(client.handle("EHLO test.example.com")).to eq ["250-My capabilities are", + "250-STARTTLS", + "250 AUTH CRAM-MD5 PLAIN LOGIN"] + end + end + end + end + +end diff --git a/spec/lib/smtp_server/client/mail_from_spec.rb b/spec/lib/smtp_server/client/mail_from_spec.rb new file mode 100644 index 0000000..e00aaa0 --- /dev/null +++ b/spec/lib/smtp_server/client/mail_from_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require "rails_helper" + +module SMTPServer + + describe Client do + let(:ip_address) { "1.2.3.4" } + subject(:client) { described_class.new(ip_address) } + + describe "MAIL FROM" do + it "returns an error if no HELO is provided" do + expect(client.handle("MAIL FROM: test@example.com")).to eq "503 EHLO/HELO first please" + expect(client.state).to eq :welcome + end + + it "resets the transaction when called" do + expect(client).to receive(:transaction_reset).and_call_original.at_least(3).times + client.handle("HELO test.example.com") + client.handle("MAIL FROM: test@example.com") + client.handle("MAIL FROM: test2@example.com") + end + + it "sets the mail from address" do + client.handle("HELO test.example.com") + expect(client.handle("MAIL FROM: test@example.com")).to eq "250 OK" + expect(client.state).to eq :mail_from_received + expect(client.instance_variable_get("@mail_from")).to eq "test@example.com" + end + end + end + +end diff --git a/spec/lib/smtp_server/client/rcpt_to_spec.rb b/spec/lib/smtp_server/client/rcpt_to_spec.rb new file mode 100644 index 0000000..7325486 --- /dev/null +++ b/spec/lib/smtp_server/client/rcpt_to_spec.rb @@ -0,0 +1,170 @@ +# frozen_string_literal: true + +require "rails_helper" + +module SMTPServer + + describe Client do + let(:ip_address) { "1.2.3.4" } + subject(:client) { described_class.new(ip_address) } + + describe "RCPT TO" do + let(:helo) { "test.example.com" } + let(:mail_from) { "test@example.com" } + + before do + client.handle("HELO #{helo}") + client.handle("MAIL FROM: #{mail_from}") if mail_from + end + + context "when MAIL FROM has not been sent" do + let(:mail_from) { nil } + + it "returns an error if RCPT TO is sent before MAIL FROM" do + expect(client.handle("RCPT TO: no-route-here@internal.com")).to eq "503 EHLO/HELO and MAIL FROM first please" + expect(client.state).to eq :welcomed + end + end + + it "returns an error if RCPT TO is not valid" do + expect(client.handle("RCPT TO: blah")).to eq "501 Invalid RCPT TO" + end + + it "returns an error if RCPT TO is empty" do + expect(client.handle("RCPT TO: ")).to eq "501 RCPT TO should not be empty" + end + + context "when the RCPT TO address is the system return path host" do + it "returns an error if the server does not exist" do + expect(client.handle("RCPT TO: nothing@#{Postal.config.dns.return_path}")).to eq "550 Invalid server token" + end + + it "returns an error if the server is suspended" do + server = create(:server, :suspended) + expect(client.handle("RCPT TO: #{server.token}@#{Postal.config.dns.return_path}")) + .to eq "535 Mail server has been suspended" + end + + it "adds a recipient if all OK" do + server = create(:server) + address = "#{server.token}@#{Postal.config.dns.return_path}" + expect(client.handle("RCPT TO: #{address}")).to eq "250 OK" + expect(client.recipients).to eq [[:bounce, address, server]] + expect(client.state).to eq :rcpt_to_received + end + end + + context "when the RCPT TO address is on a host using the return path prefix" do + it "returns an error if the server does not exist" do + address = "nothing@#{Postal.config.dns.custom_return_path_prefix}.example.com" + expect(client.handle("RCPT TO: #{address}")).to eq "550 Invalid server token" + end + + it "returns an error if the server is suspended" do + server = create(:server, :suspended) + address = "#{server.token}@#{Postal.config.dns.custom_return_path_prefix}.example.com" + expect(client.handle("RCPT TO: #{address}")).to eq "535 Mail server has been suspended" + end + + it "adds a recipient if all OK" do + server = create(:server) + address = "#{server.token}@#{Postal.config.dns.custom_return_path_prefix}.example.com" + expect(client.handle("RCPT TO: #{address}")).to eq "250 OK" + expect(client.recipients).to eq [[:bounce, address, server]] + expect(client.state).to eq :rcpt_to_received + end + end + + context "when the RCPT TO address is within the route domain" do + it "returns an error if the route token is invalid" do + address = "nothing@#{Postal.config.dns.route_domain}" + expect(client.handle("RCPT TO: #{address}")).to eq "550 Invalid route token" + end + + it "returns an error if the server is suspended" do + server = create(:server, :suspended) + route = create(:route, server: server) + address = "#{route.token}@#{Postal.config.dns.route_domain}" + expect(client.handle("RCPT TO: #{address}")).to eq "535 Mail server has been suspended" + end + + it "returns an error if the route is set to Reject mail" do + server = create(:server) + route = create(:route, server: server, mode: "Reject") + address = "#{route.token}@#{Postal.config.dns.route_domain}" + expect(client.handle("RCPT TO: #{address}")).to eq "550 Route does not accept incoming messages" + end + + it "adds a recipient if all OK" do + server = create(:server) + route = create(:route, server: server) + address = "#{route.token}+tag1@#{Postal.config.dns.route_domain}" + expect(client.handle("RCPT TO: #{address}")).to eq "250 OK" + expect(client.recipients).to eq [[:route, "#{route.name}+tag1@#{route.domain.name}", server, { route: route }]] + expect(client.state).to eq :rcpt_to_received + end + end + + context "when authenticated and the RCPT TO address is provided" do + it "returns an error if the server is suspended" do + server = create(:server, :suspended) + credential = create(:credential, server: server, type: "SMTP") + expect(client.handle("AUTH PLAIN #{credential.to_smtp_plain}")).to match(/235 Granted for /) + expect(client.handle("RCPT TO: outgoing@example.com")).to eq "535 Mail server has been suspended" + end + + it "adds a recipient if all OK" do + server = create(:server) + credential = create(:credential, server: server, type: "SMTP") + expect(client.handle("AUTH PLAIN #{credential.to_smtp_plain}")).to match(/235 Granted for /) + expect(client.handle("RCPT TO: outgoing@example.com")).to eq "250 OK" + expect(client.recipients).to eq [[:credential, "outgoing@example.com", server]] + expect(client.state).to eq :rcpt_to_received + end + end + + context "when not authenticated and the RCPT TO address is a route" do + it "returns an error if the server is suspended" do + server = create(:server, :suspended) + route = create(:route, server: server) + address = "#{route.name}@#{route.domain.name}" + expect(client.handle("RCPT TO: #{address}")).to eq "535 Mail server has been suspended" + end + + it "returns an error if the route is set to Reject mail" do + server = create(:server) + route = create(:route, server: server, mode: "Reject") + address = "#{route.name}@#{route.domain.name}" + expect(client.handle("RCPT TO: #{address}")).to eq "550 Route does not accept incoming messages" + end + + it "adds a recipient if all OK" do + server = create(:server) + route = create(:route, server: server) + address = "#{route.name}@#{route.domain.name}" + expect(client.handle("RCPT TO: #{address}")).to eq "250 OK" + expect(client.recipients).to eq [[:route, address, server, { route: route }]] + expect(client.state).to eq :rcpt_to_received + end + end + + context "when not authenticated and RCPT TO does not match a route" do + it "returns an error" do + expect(client.handle("RCPT TO: nothing@nothing.com")).to eq "530 Authentication required" + end + + context "when the connecting IP has an credential" do + it "adds a recipient" do + server = create(:server) + create(:credential, server: server, type: "SMTP-IP", key: "1.0.0.0/8") + address = "test@example.com" + expect(client.handle("RCPT TO: #{address}")).to eq "250 OK" + expect(client.recipients).to eq [[:credential, address, server]] + expect(client.state).to eq :rcpt_to_received + end + end + end + end + end + +end diff --git a/spec/lib/smtp_server/client_spec.rb b/spec/lib/smtp_server/client_spec.rb new file mode 100644 index 0000000..353c12e --- /dev/null +++ b/spec/lib/smtp_server/client_spec.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +require "rails_helper" + +module SMTPServer + + describe Client do + let(:ip_address) { "1.2.3.4" } + subject(:client) { described_class.new(ip_address) } + end + +end From 5cc9eb3df79e79fee77f1479a85a4886ab6295d7 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Thu, 22 Feb 2024 22:36:46 +0000 Subject: [PATCH 26/56] refactor: move lib/postal/reply_separator to app/lib/reply_separator --- {lib/postal => app/lib}/reply_separator.rb | 0 app/senders/http_sender.rb | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename {lib/postal => app/lib}/reply_separator.rb (100%) diff --git a/lib/postal/reply_separator.rb b/app/lib/reply_separator.rb similarity index 100% rename from lib/postal/reply_separator.rb rename to app/lib/reply_separator.rb diff --git a/app/senders/http_sender.rb b/app/senders/http_sender.rb index af931e9..4da8c7f 100644 --- a/app/senders/http_sender.rb +++ b/app/senders/http_sender.rb @@ -91,7 +91,7 @@ class HTTPSender < BaseSender } if @endpoint.strip_replies - hash[:plain_body], hash[:replies_from_plain_body] = Postal::ReplySeparator.separate(message.plain_body) + hash[:plain_body], hash[:replies_from_plain_body] = ReplySeparator.separate(message.plain_body) else hash[:plain_body] = message.plain_body end From e3bc9da253bf278b05e675148220fea5a27eb988 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Thu, 22 Feb 2024 22:38:17 +0000 Subject: [PATCH 27/56] refactor: move lib/postal/received_header to app/lib/received_header --- app/lib/received_header.rb | 30 +++++++++++++++++ app/lib/smtp_server/client.rb | 4 +-- app/models/incoming_message_prototype.rb | 2 +- app/models/outgoing_message_prototype.rb | 2 +- lib/postal/received_header.rb | 32 ------------------- spec/lib/{postal => }/received_header_spec.rb | 2 +- 6 files changed, 35 insertions(+), 37 deletions(-) create mode 100644 app/lib/received_header.rb delete mode 100644 lib/postal/received_header.rb rename spec/lib/{postal => }/received_header_spec.rb (98%) diff --git a/app/lib/received_header.rb b/app/lib/received_header.rb new file mode 100644 index 0000000..7d578f4 --- /dev/null +++ b/app/lib/received_header.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class ReceivedHeader + + OUR_HOSTNAMES = { + smtp: Postal.config.dns.smtp_server_hostname, + http: Postal.config.web.host + }.freeze + + class << self + + def generate(server, helo, ip_address, method) + our_hostname = OUR_HOSTNAMES[method] + if our_hostname.nil? + raise Error, "`method` is invalid (must be one of #{OUR_HOSTNAMES.join(', ')})" + end + + header = "by #{our_hostname} with #{method.to_s.upcase}; #{Time.now.utc.rfc2822}" + + if server.nil? || server.privacy_mode == false + hostname = DNSResolver.local.ip_to_hostname(ip_address) + header = "from #{helo} (#{hostname} [#{ip_address}]) #{header}" + end + + header + end + + end + +end diff --git a/app/lib/smtp_server/client.rb b/app/lib/smtp_server/client.rb index 3040f7f..c89aea2 100644 --- a/app/lib/smtp_server/client.rb +++ b/app/lib/smtp_server/client.rb @@ -366,8 +366,8 @@ module SMTPServer @headers = {} @receiving_headers = true - received_header = Postal::ReceivedHeader.generate(@credential&.server, @helo_name, @ip_address, :smtp) - .force_encoding("BINARY") + received_header = ReceivedHeader.generate(@credential&.server, @helo_name, @ip_address, :smtp) + .force_encoding("BINARY") @data << "Received: #{received_header}\r\n" @headers["received"] = [received_header] diff --git a/app/models/incoming_message_prototype.rb b/app/models/incoming_message_prototype.rb index 334df62..ee3c0ec 100644 --- a/app/models/incoming_message_prototype.rb +++ b/app/models/incoming_message_prototype.rb @@ -95,7 +95,7 @@ class IncomingMessagePrototype content: attachment[:data] } end - mail.header["Received"] = Postal::ReceivedHeader.generate(@server, @source_type, @ip, :http) + mail.header["Received"] = ReceivedHeader.generate(@server, @source_type, @ip, :http) mail.to_s end end diff --git a/app/models/outgoing_message_prototype.rb b/app/models/outgoing_message_prototype.rb index 7362f30..f36522a 100644 --- a/app/models/outgoing_message_prototype.rb +++ b/app/models/outgoing_message_prototype.rb @@ -177,7 +177,7 @@ class OutgoingMessagePrototype content: attachment[:data] } end - mail.header["Received"] = Postal::ReceivedHeader.generate(@server, @source_type, @ip, :http) + mail.header["Received"] = ReceivedHeader.generate(@server, @source_type, @ip, :http) mail.message_id = "<#{@message_id}>" mail.to_s end diff --git a/lib/postal/received_header.rb b/lib/postal/received_header.rb deleted file mode 100644 index b019d69..0000000 --- a/lib/postal/received_header.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module Postal - class ReceivedHeader - - OUR_HOSTNAMES = { - smtp: Postal.config.dns.smtp_server_hostname, - http: Postal.config.web.host - }.freeze - - class << self - - def generate(server, helo, ip_address, method) - our_hostname = OUR_HOSTNAMES[method] - if our_hostname.nil? - raise Error, "`method` is invalid (must be one of #{OUR_HOSTNAMES.join(', ')})" - end - - header = "by #{our_hostname} with #{method.to_s.upcase}; #{Time.now.utc.rfc2822}" - - if server.nil? || server.privacy_mode == false - hostname = DNSResolver.local.ip_to_hostname(ip_address) - header = "from #{helo} (#{hostname} [#{ip_address}]) #{header}" - end - - header - end - - end - - end -end diff --git a/spec/lib/postal/received_header_spec.rb b/spec/lib/received_header_spec.rb similarity index 98% rename from spec/lib/postal/received_header_spec.rb rename to spec/lib/received_header_spec.rb index 0a9c6bb..199f027 100644 --- a/spec/lib/postal/received_header_spec.rb +++ b/spec/lib/received_header_spec.rb @@ -2,7 +2,7 @@ require "rails_helper" -describe Postal::ReceivedHeader do +describe ReceivedHeader do before do allow(DNSResolver.local).to receive(:ip_to_hostname).and_return("hostname.com") end From 64b2704b02991c5eed808360503c78222d610b89 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Thu, 22 Feb 2024 22:39:20 +0000 Subject: [PATCH 28/56] refactor: move lib/postal/user_creator to app/util/user_creator --- {lib/postal => app/util}/user_creator.rb | 8 +++++--- script/make_user.rb | 3 +-- 2 files changed, 6 insertions(+), 5 deletions(-) rename {lib/postal => app/util}/user_creator.rb (93%) diff --git a/lib/postal/user_creator.rb b/app/util/user_creator.rb similarity index 93% rename from lib/postal/user_creator.rb rename to app/util/user_creator.rb index 27bb973..49b5d70 100644 --- a/lib/postal/user_creator.rb +++ b/app/util/user_creator.rb @@ -2,10 +2,11 @@ require "highline" -module Postal - module UserCreator +module UserCreator - def self.start(&block) + class << self + + def start(&block) cli = HighLine.new puts "\e[32mPostal User Creator\e[0m" puts "Enter the information required to create a new Postal user." @@ -31,4 +32,5 @@ module Postal end end + end diff --git a/script/make_user.rb b/script/make_user.rb index f5986c7..c98e0d2 100755 --- a/script/make_user.rb +++ b/script/make_user.rb @@ -7,9 +7,8 @@ trap("INT") do end require_relative "../config/environment" -require "postal/user_creator" -Postal::UserCreator.start do |u| +UserCreator.start do |u| u.admin = true u.email_verified_at = Time.now end From b8ad732152ca2cbca6d4aacdb1fc365fe944fad4 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Thu, 22 Feb 2024 22:42:26 +0000 Subject: [PATCH 29/56] refactor: move tracking middleware --- config/application.rb | 4 +- lib/postal/tracking_middleware.rb | 123 ------------------------------ lib/tracking_middleware.rb | 121 +++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 125 deletions(-) delete mode 100644 lib/postal/tracking_middleware.rb create mode 100644 lib/tracking_middleware.rb diff --git a/config/application.rb b/config/application.rb index 12e8c27..13b4677 100644 --- a/config/application.rb +++ b/config/application.rb @@ -35,8 +35,8 @@ module Postal config.action_view.field_error_proc = proc { |t, _| t } # Load the tracking server middleware - require "postal/tracking_middleware" - config.middleware.insert_before ActionDispatch::HostAuthorization, Postal::TrackingMiddleware + require "tracking_middleware" + config.middleware.insert_before ActionDispatch::HostAuthorization, TrackingMiddleware config.hosts << Postal.config.web.host diff --git a/lib/postal/tracking_middleware.rb b/lib/postal/tracking_middleware.rb deleted file mode 100644 index 1593af9..0000000 --- a/lib/postal/tracking_middleware.rb +++ /dev/null @@ -1,123 +0,0 @@ -# frozen_string_literal: true - -module Postal - class TrackingMiddleware - - TRACKING_PIXEL = File.read(Rails.root.join("app", "assets", "images", "tracking_pixel.png")) - - def initialize(app = nil) - @app = app - end - - def call(env) - unless env["HTTP_X_POSTAL_TRACK_HOST"].to_i == 1 - return @app.call(env) - end - - request = Rack::Request.new(env) - - case request.path - when /\A\/img\/([a-z0-9-]+)\/([a-z0-9-]+)/i - server_token = ::Regexp.last_match(1) - message_token = ::Regexp.last_match(2) - dispatch_image_request(request, server_token, message_token) - when /\A\/([a-z0-9-]+)\/([a-z0-9-]+)/i - server_token = ::Regexp.last_match(1) - link_token = ::Regexp.last_match(2) - dispatch_redirect_request(request, server_token, link_token) - else - [200, {}, ["Hello."]] - end - end - - private - - def dispatch_image_request(request, server_token, message_token) - message_db = get_message_db_from_server_token(server_token) - if message_db.nil? - return [404, {}, ["Invalid Server Token"]] - end - - begin - message = message_db.message(token: message_token) - message.create_load(request) - rescue Postal::MessageDB::Message::NotFound - # This message has been removed, we'll just continue to serve the image - rescue StandardError => e - # Somethign else went wrong. We don't want to stop the image loading though because - # this is our problem. Log this exception though. - Sentry.capture_exception(e) if defined?(Sentry) - end - - source_image = request.params["src"] - case source_image - when nil - headers = {} - headers["Content-Type"] = "image/png" - headers["Content-Length"] = TRACKING_PIXEL.bytesize.to_s - [200, headers, [TRACKING_PIXEL]] - when /\Ahttps?:\/\// - response = Postal::HTTP.get(source_image, timeout: 3) - return [404, {}, ["Not found"]] unless response[:code] == 200 - - headers = {} - headers["Content-Type"] = response[:headers]["content-type"]&.first - headers["Last-Modified"] = response[:headers]["last-modified"]&.first - headers["Cache-Control"] = response[:headers]["cache-control"]&.first - headers["Etag"] = response[:headers]["etag"]&.first - headers["Content-Length"] = response[:body].bytesize.to_s - [200, headers, [response[:body]]] - - else - [400, {}, ["Invalid/missing source image"]] - end - end - - def dispatch_redirect_request(request, server_token, link_token) - message_db = get_message_db_from_server_token(server_token) - if message_db.nil? - return [404, {}, ["Invalid Server Token"]] - end - - link = message_db.select(:links, where: { token: link_token }, limit: 1).first - if link.nil? - return [404, {}, ["Link not found"]] - end - - time = Time.now.to_f - if link["message_id"] - message_db.update(:messages, { clicked: time }, where: { id: link["message_id"] }) - message_db.insert(:clicks, { - message_id: link["message_id"], - link_id: link["id"], - ip_address: request.ip, - user_agent: request.user_agent, - timestamp: time - }) - - begin - message_webhook_hash = message_db.message(link["message_id"]).webhook_hash - WebhookRequest.trigger(message_db.server, "MessageLinkClicked", { - message: message_webhook_hash, - url: link["url"], - token: link["token"], - ip_address: request.ip, - user_agent: request.user_agent - }) - rescue Postal::MessageDB::Message::NotFound - # If we can't find the message that this link is associated with, we'll just ignore it - # and not trigger any webhooks. - end - end - - [307, { "Location" => link["url"] }, ["Redirected to: #{link['url']}"]] - end - - def get_message_db_from_server_token(token) - return unless server = ::Server.find_by_token(token) - - server.message_db - end - - end -end diff --git a/lib/tracking_middleware.rb b/lib/tracking_middleware.rb new file mode 100644 index 0000000..bee3a63 --- /dev/null +++ b/lib/tracking_middleware.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +class TrackingMiddleware + + TRACKING_PIXEL = File.read(Rails.root.join("app", "assets", "images", "tracking_pixel.png")) + + def initialize(app = nil) + @app = app + end + + def call(env) + unless env["HTTP_X_POSTAL_TRACK_HOST"].to_i == 1 + return @app.call(env) + end + + request = Rack::Request.new(env) + + case request.path + when /\A\/img\/([a-z0-9-]+)\/([a-z0-9-]+)/i + server_token = ::Regexp.last_match(1) + message_token = ::Regexp.last_match(2) + dispatch_image_request(request, server_token, message_token) + when /\A\/([a-z0-9-]+)\/([a-z0-9-]+)/i + server_token = ::Regexp.last_match(1) + link_token = ::Regexp.last_match(2) + dispatch_redirect_request(request, server_token, link_token) + else + [200, {}, ["Hello."]] + end + end + + private + + def dispatch_image_request(request, server_token, message_token) + message_db = get_message_db_from_server_token(server_token) + if message_db.nil? + return [404, {}, ["Invalid Server Token"]] + end + + begin + message = message_db.message(token: message_token) + message.create_load(request) + rescue Postal::MessageDB::Message::NotFound + # This message has been removed, we'll just continue to serve the image + rescue StandardError => e + # Somethign else went wrong. We don't want to stop the image loading though because + # this is our problem. Log this exception though. + Sentry.capture_exception(e) if defined?(Sentry) + end + + source_image = request.params["src"] + case source_image + when nil + headers = {} + headers["Content-Type"] = "image/png" + headers["Content-Length"] = TRACKING_PIXEL.bytesize.to_s + [200, headers, [TRACKING_PIXEL]] + when /\Ahttps?:\/\// + response = Postal::HTTP.get(source_image, timeout: 3) + return [404, {}, ["Not found"]] unless response[:code] == 200 + + headers = {} + headers["Content-Type"] = response[:headers]["content-type"]&.first + headers["Last-Modified"] = response[:headers]["last-modified"]&.first + headers["Cache-Control"] = response[:headers]["cache-control"]&.first + headers["Etag"] = response[:headers]["etag"]&.first + headers["Content-Length"] = response[:body].bytesize.to_s + [200, headers, [response[:body]]] + + else + [400, {}, ["Invalid/missing source image"]] + end + end + + def dispatch_redirect_request(request, server_token, link_token) + message_db = get_message_db_from_server_token(server_token) + if message_db.nil? + return [404, {}, ["Invalid Server Token"]] + end + + link = message_db.select(:links, where: { token: link_token }, limit: 1).first + if link.nil? + return [404, {}, ["Link not found"]] + end + + time = Time.now.to_f + if link["message_id"] + message_db.update(:messages, { clicked: time }, where: { id: link["message_id"] }) + message_db.insert(:clicks, { + message_id: link["message_id"], + link_id: link["id"], + ip_address: request.ip, + user_agent: request.user_agent, + timestamp: time + }) + + begin + message_webhook_hash = message_db.message(link["message_id"]).webhook_hash + WebhookRequest.trigger(message_db.server, "MessageLinkClicked", { + message: message_webhook_hash, + url: link["url"], + token: link["token"], + ip_address: request.ip, + user_agent: request.user_agent + }) + rescue Postal::MessageDB::Message::NotFound + # If we can't find the message that this link is associated with, we'll just ignore it + # and not trigger any webhooks. + end + end + + [307, { "Location" => link["url"] }, ["Redirected to: #{link['url']}"]] + end + + def get_message_db_from_server_token(token) + return unless server = ::Server.find_by_token(token) + + server.message_db + end + +end From 38465de120492a61372df44fc629286618224b7c Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Thu, 22 Feb 2024 22:42:41 +0000 Subject: [PATCH 30/56] refactor: remove unnecessary modules --- lib/postal/message_db.rb | 6 ------ lib/postal/message_inspectors.rb | 6 ------ 2 files changed, 12 deletions(-) delete mode 100644 lib/postal/message_db.rb delete mode 100644 lib/postal/message_inspectors.rb diff --git a/lib/postal/message_db.rb b/lib/postal/message_db.rb deleted file mode 100644 index 0d01c89..0000000 --- a/lib/postal/message_db.rb +++ /dev/null @@ -1,6 +0,0 @@ -# frozen_string_literal: true - -module Postal - module MessageDB - end -end diff --git a/lib/postal/message_inspectors.rb b/lib/postal/message_inspectors.rb deleted file mode 100644 index 8ddb584..0000000 --- a/lib/postal/message_inspectors.rb +++ /dev/null @@ -1,6 +0,0 @@ -# frozen_string_literal: true - -module Postal - module MessageInspectors - end -end From cfc1c9b73e5b0914aec6d56a62e60561697ec660 Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Thu, 22 Feb 2024 22:44:53 +0000 Subject: [PATCH 31/56] docs: update SECURITY policy --- SECURITY.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index f75f979..7936c08 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,12 +2,12 @@ ## Supported Versions -We only support updates to the 2.x versions of Postal. +We only support updates to the 3.x versions of Postal. | Version | Supported | | ------- | ------------------ | -| 2.x.x | :white_check_mark: | -| < 2.0 | :x: | +| 3.x.x | :white_check_mark: | +| < 3.0 | :x: | ## Reporting a Vulnerability From ecd09a2445f82f4897265caf4a3bf6b6cd2146cb Mon Sep 17 00:00:00 2001 From: Adam Cooke Date: Fri, 23 Feb 2024 14:24:17 +0000 Subject: [PATCH 32/56] chore: upgrade rails to 7.0 and other dependencies --- Gemfile | 12 +- Gemfile.lock | 181 +++++++++--------- app/controllers/application_controller.rb | 3 +- app/controllers/messages_controller.rb | 2 +- app/models/concerns/has_authentication.rb | 17 +- app/models/credential.rb | 2 +- app/models/domain.rb | 26 +-- app/models/webhook.rb | 16 +- app/views/domains/index.html.haml | 2 +- app/views/messages/_deliveries.html.haml | 4 +- app/views/messages/_list.html.haml | 2 +- app/views/messages/_message_header.html.haml | 2 +- app/views/messages/activity.html.haml | 8 +- app/views/messages/suppressions.html.haml | 4 +- app/views/servers/show.html.haml | 3 +- config/application.rb | 2 +- config/boot.rb | 3 - .../initializers/content_security_policy.rb | 26 +++ .../initializers/filter_parameter_logging.rb | 8 +- config/initializers/new_framework_defaults.rb | 23 --- .../new_framework_defaults_7_0.rb | 142 ++++++++++++++ config/initializers/permissions_policy.rb | 12 ++ ..._two_factor_required_to_sessions.authie.rb | 10 + ...add_countries_to_authie_sessions.authie.rb | 12 ++ db/schema.rb | 161 ++++++++-------- script/smtp_server.rb | 3 + script/worker.rb | 3 + 27 files changed, 439 insertions(+), 250 deletions(-) create mode 100644 config/initializers/content_security_policy.rb delete mode 100644 config/initializers/new_framework_defaults.rb create mode 100644 config/initializers/new_framework_defaults_7_0.rb create mode 100644 config/initializers/permissions_policy.rb create mode 100644 db/migrate/20240223141500_add_two_factor_required_to_sessions.authie.rb create mode 100644 db/migrate/20240223141501_add_countries_to_authie_sessions.authie.rb diff --git a/Gemfile b/Gemfile index 8c9fe26..e8837af 100644 --- a/Gemfile +++ b/Gemfile @@ -3,15 +3,12 @@ source "https://rubygems.org" gem "authie" gem "autoprefixer-rails" -gem "basic_ssl" gem "bcrypt" -gem "changey" gem "chronic" gem "dotenv-rails" gem "dynamic_form" gem "encrypto_signo" gem "execjs", "~> 2.7", "< 2.8" -gem "foreman" gem "gelf" gem "haml" gem "hashie" @@ -26,11 +23,10 @@ gem "nifty-utils" gem "nilify_blanks" gem "nio4r" gem "puma" -gem "rails", "= 6.1.7.6" -gem "resolv", "~> 0.2.1" +gem "rails", "= 7.0.8.1" +gem "resolv" gem "secure_headers" gem "sentry-rails" -gem "sentry-ruby" gem "turbolinks", "~> 5" group :development, :assets do @@ -40,10 +36,6 @@ group :development, :assets do gem "uglifier", ">= 1.3.0" end -group :development, :test do - gem "byebug" -end - group :development do gem "annotate" gem "database_cleaner", require: false diff --git a/Gemfile.lock b/Gemfile.lock index 476e924..d0e4868 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,82 +1,85 @@ GEM remote: https://rubygems.org/ specs: - actioncable (6.1.7.6) - actionpack (= 6.1.7.6) - activesupport (= 6.1.7.6) + actioncable (7.0.8.1) + actionpack (= 7.0.8.1) + activesupport (= 7.0.8.1) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (6.1.7.6) - actionpack (= 6.1.7.6) - activejob (= 6.1.7.6) - activerecord (= 6.1.7.6) - activestorage (= 6.1.7.6) - activesupport (= 6.1.7.6) + actionmailbox (7.0.8.1) + actionpack (= 7.0.8.1) + activejob (= 7.0.8.1) + activerecord (= 7.0.8.1) + activestorage (= 7.0.8.1) + activesupport (= 7.0.8.1) mail (>= 2.7.1) - actionmailer (6.1.7.6) - actionpack (= 6.1.7.6) - actionview (= 6.1.7.6) - activejob (= 6.1.7.6) - activesupport (= 6.1.7.6) + net-imap + net-pop + net-smtp + actionmailer (7.0.8.1) + actionpack (= 7.0.8.1) + actionview (= 7.0.8.1) + activejob (= 7.0.8.1) + activesupport (= 7.0.8.1) mail (~> 2.5, >= 2.5.4) + net-imap + net-pop + net-smtp rails-dom-testing (~> 2.0) - actionpack (6.1.7.6) - actionview (= 6.1.7.6) - activesupport (= 6.1.7.6) - rack (~> 2.0, >= 2.0.9) + actionpack (7.0.8.1) + actionview (= 7.0.8.1) + activesupport (= 7.0.8.1) + rack (~> 2.0, >= 2.2.4) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (6.1.7.6) - actionpack (= 6.1.7.6) - activerecord (= 6.1.7.6) - activestorage (= 6.1.7.6) - activesupport (= 6.1.7.6) + actiontext (7.0.8.1) + actionpack (= 7.0.8.1) + activerecord (= 7.0.8.1) + activestorage (= 7.0.8.1) + activesupport (= 7.0.8.1) + globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (6.1.7.6) - activesupport (= 6.1.7.6) + actionview (7.0.8.1) + activesupport (= 7.0.8.1) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.1, >= 1.2.0) - activejob (6.1.7.6) - activesupport (= 6.1.7.6) + activejob (7.0.8.1) + activesupport (= 7.0.8.1) globalid (>= 0.3.6) - activemodel (6.1.7.6) - activesupport (= 6.1.7.6) - activerecord (6.1.7.6) - activemodel (= 6.1.7.6) - activesupport (= 6.1.7.6) - activestorage (6.1.7.6) - actionpack (= 6.1.7.6) - activejob (= 6.1.7.6) - activerecord (= 6.1.7.6) - activesupport (= 6.1.7.6) + activemodel (7.0.8.1) + activesupport (= 7.0.8.1) + activerecord (7.0.8.1) + activemodel (= 7.0.8.1) + activesupport (= 7.0.8.1) + activestorage (7.0.8.1) + actionpack (= 7.0.8.1) + activejob (= 7.0.8.1) + activerecord (= 7.0.8.1) + activesupport (= 7.0.8.1) marcel (~> 1.0) mini_mime (>= 1.1.0) - activesupport (6.1.7.6) + activesupport (7.0.8.1) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) - zeitwerk (~> 2.3) addressable (2.8.6) public_suffix (>= 2.0.2, < 6.0) annotate (3.2.0) activerecord (>= 3.2, < 8.0) rake (>= 10.4, < 14.0) ast (2.4.2) - authie (3.4.0) - secure_random_string + authie (4.1.3) + activerecord (>= 6.1, < 8.0) autoprefixer-rails (10.4.13.0) execjs (~> 2) - basic_ssl (1.0.3) - bcrypt (3.1.18) + base64 (0.2.0) + bcrypt (3.1.20) bigdecimal (3.1.6) builder (3.2.4) - byebug (11.1.3) - changey (1.1.0) - activerecord (>= 4.2, < 7) chronic (0.10.2) coffee-rails (5.0.0) coffee-script (>= 2.2.0) @@ -96,13 +99,13 @@ GEM activerecord (>= 5.a) database_cleaner-core (~> 2.0.0) database_cleaner-core (2.0.1) - date (3.3.3) + date (3.3.4) deep_merge (1.2.2) diff-lcs (1.5.0) - dotenv (2.8.1) - dotenv-rails (2.8.1) - dotenv (= 2.8.1) - railties (>= 3.2) + dotenv (3.0.2) + dotenv-rails (3.0.2) + dotenv (= 3.0.2) + railties (>= 6.1) dynamic_form (1.3.1) actionview (> 5.2.0) activemodel (> 5.2.0) @@ -115,12 +118,11 @@ GEM factory_bot (~> 6.4) railties (>= 5.0.0) ffi (1.15.5) - foreman (0.87.2) gelf (3.1.0) json globalid (1.2.1) activesupport (>= 6.1) - haml (6.1.1) + haml (6.3.0) temple (>= 0.8.2) thor tilt @@ -134,7 +136,8 @@ GEM railties (>= 4.2.0) thor (>= 0.14, < 2.0) json (2.6.3) - jwt (2.7.0) + jwt (2.8.0) + base64 kaminari (1.2.2) activesupport (>= 4.1.0) kaminari-actionview (= 1.2.2) @@ -161,22 +164,22 @@ GEM net-smtp marcel (1.0.2) method_source (1.0.0) - mini_mime (1.1.2) + mini_mime (1.1.5) mini_portile2 (2.8.5) minitest (5.22.2) moonrope (2.0.2) deep_merge (~> 1.0) json rack (>= 1.4) - mysql2 (0.5.5) - net-imap (0.3.4) + mysql2 (0.5.6) + net-imap (0.4.10) date net-protocol net-pop (0.1.2) net-protocol - net-protocol (0.2.1) + net-protocol (0.2.2) timeout - net-smtp (0.3.3) + net-smtp (0.4.0.1) net-protocol nifty-utils (1.1.7) nilify_blanks (1.4.0) @@ -197,24 +200,23 @@ GEM puma (6.4.2) nio4r (~> 2.0) racc (1.7.3) - rack (2.2.8) + rack (2.2.8.1) rack-test (2.1.0) rack (>= 1.3) - rails (6.1.7.6) - actioncable (= 6.1.7.6) - actionmailbox (= 6.1.7.6) - actionmailer (= 6.1.7.6) - actionpack (= 6.1.7.6) - actiontext (= 6.1.7.6) - actionview (= 6.1.7.6) - activejob (= 6.1.7.6) - activemodel (= 6.1.7.6) - activerecord (= 6.1.7.6) - activestorage (= 6.1.7.6) - activesupport (= 6.1.7.6) + rails (7.0.8.1) + actioncable (= 7.0.8.1) + actionmailbox (= 7.0.8.1) + actionmailer (= 7.0.8.1) + actionpack (= 7.0.8.1) + actiontext (= 7.0.8.1) + actionview (= 7.0.8.1) + activejob (= 7.0.8.1) + activemodel (= 7.0.8.1) + activerecord (= 7.0.8.1) + activestorage (= 7.0.8.1) + activesupport (= 7.0.8.1) bundler (>= 1.15.0) - railties (= 6.1.7.6) - sprockets-rails (>= 2.0.0) + railties (= 7.0.8.1) rails-dom-testing (2.2.0) activesupport (>= 5.0.0) minitest @@ -222,16 +224,17 @@ GEM rails-html-sanitizer (1.6.0) loofah (~> 2.21) nokogiri (~> 1.14) - railties (6.1.7.6) - actionpack (= 6.1.7.6) - activesupport (= 6.1.7.6) + railties (7.0.8.1) + actionpack (= 7.0.8.1) + activesupport (= 7.0.8.1) method_source rake (>= 12.2) thor (~> 1.0) + zeitwerk (~> 2.5) rainbow (3.1.1) rake (13.1.0) regexp_parser (2.7.0) - resolv (0.2.2) + resolv (0.3.0) rexml (3.2.5) rouge (4.2.0) rspec (3.12.0) @@ -283,11 +286,10 @@ GEM sprockets-rails tilt secure_headers (6.5.0) - secure_random_string (1.0.0) - sentry-rails (5.8.0) + sentry-rails (5.16.1) railties (>= 5.0) - sentry-ruby (~> 5.8.0) - sentry-ruby (5.8.0) + sentry-ruby (~> 5.16.1) + sentry-ruby (5.16.1) concurrent-ruby (~> 1.0, >= 1.0.2) shoulda-matchers (6.1.0) activesupport (>= 5.2.0) @@ -298,11 +300,11 @@ GEM actionpack (>= 5.2) activesupport (>= 5.2) sprockets (>= 3.0.0) - temple (0.10.0) + temple (0.10.3) thor (1.3.0) - tilt (2.1.0) + tilt (2.3.0) timecop (0.9.8) - timeout (0.3.2) + timeout (0.4.1) turbolinks (5.2.1) turbolinks-source (~> 5.2) turbolinks-source (5.2.0) @@ -330,10 +332,7 @@ DEPENDENCIES annotate authie autoprefixer-rails - basic_ssl bcrypt - byebug - changey chronic coffee-rails (~> 5.0) database_cleaner @@ -342,7 +341,6 @@ DEPENDENCIES encrypto_signo execjs (~> 2.7, < 2.8) factory_bot_rails - foreman gelf haml hashie @@ -358,8 +356,8 @@ DEPENDENCIES nilify_blanks nio4r puma - rails (= 6.1.7.6) - resolv (~> 0.2.1) + rails (= 7.0.8.1) + resolv rspec rspec-rails rubocop @@ -367,7 +365,6 @@ DEPENDENCIES sass-rails secure_headers sentry-rails - sentry-ruby shoulda-matchers timecop turbolinks (~> 5) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 3053575..d2c36cd 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -109,7 +109,8 @@ class ApplicationController < ActionController::Base auth_session.invalidate! reset_session end - Authie::Session.start(self, user: user) + + create_auth_session(user) @current_user = user end diff --git a/app/controllers/messages_controller.rb b/app/controllers/messages_controller.rb index 2e6a25d..ea0e940 100644 --- a/app/controllers/messages_controller.rb +++ b/app/controllers/messages_controller.rb @@ -19,7 +19,7 @@ class MessagesController < ApplicationController @message.from = "test@#{domain.name}" end end - @message.subject = "Test Message at #{Time.zone.now.to_s(:long)}" + @message.subject = "Test Message at #{Time.zone.now.to_fs(:long)}" @message.plain_body = "This is a message to test the delivery of messages through Postal." end diff --git a/app/models/concerns/has_authentication.rb b/app/models/concerns/has_authentication.rb index 80a52ce..4491a77 100644 --- a/app/models/concerns/has_authentication.rb +++ b/app/models/concerns/has_authentication.rb @@ -8,13 +8,7 @@ module HasAuthentication has_secure_password validates :password, length: { minimum: 8, allow_blank: true } - - when_attribute :password_digest, changes_to: :anything do - before_save do - self.password_reset_token = nil - self.password_reset_token_valid_until = nil - end - end + before_save :clear_password_reset_token_on_password_change end class_methods do @@ -42,6 +36,15 @@ module HasAuthentication AppMailer.password_reset(self, return_to).deliver end + private + + def clear_password_reset_token_on_password_change + return unless password_digest_changed? + + self.password_reset_token = nil + self.password_reset_token_valid_until = nil + end + end # -*- SkipSchemaAnnotations diff --git a/app/models/credential.rb b/app/models/credential.rb index 18e3266..8062eb5 100644 --- a/app/models/credential.rb +++ b/app/models/credential.rb @@ -39,7 +39,7 @@ class Credential < ApplicationRecord return if type == "SMTP-IP" return if persisted? - self.key = SecureRandomString.new(24) + self.key = SecureRandom.alphanumeric(24) end def to_param diff --git a/app/models/domain.rb b/app/models/domain.rb index 6c3b2ff..d449847 100644 --- a/app/models/domain.rb +++ b/app/models/domain.rb @@ -61,17 +61,7 @@ class Domain < ApplicationRecord scope :verified, -> { where.not(verified_at: nil) } - when_attribute :verification_method, changes_to: :anything do - before_save do - if verification_method == "DNS" - self.verification_token = Nifty::Utils::RandomString.generate(length: 32) - elsif verification_method == "Email" - self.verification_token = rand(999_999).to_s.ljust(6, "0") - else - self.verification_token = nil - end - end - end + before_save :update_verification_token_on_method_change def verified? verified_at.present? @@ -168,4 +158,18 @@ class Domain < ApplicationRecord false end + private + + def update_verification_token_on_method_change + return unless verification_method_changed? + + if verification_method == "DNS" + self.verification_token = Nifty::Utils::RandomString.generate(length: 32) + elsif verification_method == "Email" + self.verification_token = rand(999_999).to_s.ljust(6, "0") + else + self.verification_token = nil + end + end + end diff --git a/app/models/webhook.rb b/app/models/webhook.rb index a334632..cfa9891 100644 --- a/app/models/webhook.rb +++ b/app/models/webhook.rb @@ -35,12 +35,7 @@ class Webhook < ApplicationRecord scope :enabled, -> { where(enabled: true) } after_save :save_events - - when_attribute :all_events, changes_to: true do - after_save do - webhook_events.destroy_all - end - end + after_save :destroy_events_when_all_events_enabled def events @events ||= webhook_events.map(&:event) @@ -50,13 +45,22 @@ class Webhook < ApplicationRecord @events = value.map(&:to_s).select(&:present?) end + private + def save_events return unless @events @events.each do |event| webhook_events.where(event: event).first_or_create! end + webhook_events.where.not(event: @events).destroy_all end + def destroy_events_when_all_events_enabled + return unless all_events + + webhook_events.destroy_all + end + end diff --git a/app/views/domains/index.html.haml b/app/views/domains/index.html.haml index eafd3a3..2238e01 100644 --- a/app/views/domains/index.html.haml +++ b/app/views/domains/index.html.haml @@ -63,7 +63,7 @@ %ul.domainList__properties - if domain.verified? - %li.domainList__verificationTime Verified on #{domain.verified_at.to_s(:long)} + %li.domainList__verificationTime Verified on #{domain.verified_at.to_fs(:long)} - else %li= link_to "Verify this domain", [:verify, organization, @server, domain], :class => "domainList__verificationLink" %li.domainList__links diff --git a/app/views/messages/_deliveries.html.haml b/app/views/messages/_deliveries.html.haml index 707948b..dc9f4f9 100644 --- a/app/views/messages/_deliveries.html.haml +++ b/app/views/messages/_deliveries.html.haml @@ -11,7 +11,7 @@ %p This message has been held. By releasing the message, we will allow it to continue on its way to its destination. - if @message.hold_expiry - It will be held until #{@message.hold_expiry.to_s(:long)}. + It will be held until #{@message.hold_expiry.to_fs(:long)}. %p.buttonSet = link_to "Release message", retry_organization_server_message_path(organization, @server, message.id), :class => "button button--small", :remote => true, :method => :post = link_to "Cancel hold", cancel_hold_organization_server_message_path(organization, @server, message.id), :class => "button button--small button--danger", :remote => true, :method => :post @@ -33,7 +33,7 @@ %li.deliveryList__item .deliveryList__top .deliveryList__time - = delivery.timestamp.to_s(:long) + = delivery.timestamp.to_fs(:long) .deliveryList__status - if delivery.sent_with_ssl = image_tag 'icons/lock.svg', :class => 'deliveryList__secure' diff --git a/app/views/messages/_list.html.haml b/app/views/messages/_list.html.haml index 79e5fdb..4d1a4ad 100644 --- a/app/views/messages/_list.html.haml +++ b/app/views/messages/_list.html.haml @@ -17,7 +17,7 @@ %dd= message.mail_from || "none" .messageList__meta - %p.messageList__timestamp= message.timestamp.in_time_zone.to_s(:long) + %p.messageList__timestamp= message.timestamp.in_time_zone.to_fs(:long) %p.messageList__status - if message.read? %span.label.label--purple Opened diff --git a/app/views/messages/_message_header.html.haml b/app/views/messages/_message_header.html.haml index 41c537c..0ec98b4 100644 --- a/app/views/messages/_message_header.html.haml +++ b/app/views/messages/_message_header.html.haml @@ -23,7 +23,7 @@ = link_to @message.rcpt_to || "[blank]", send("#{@message.scope}_organization_server_messages_path", organization, @server, :query => "to: #{@message.rcpt_to}"), :class => 'u-link' %dl %dt Received - %dd= @message.timestamp.in_time_zone.to_s(:long) + %dd= @message.timestamp.in_time_zone.to_fs(:long) .navBar.navBar--tertiary %ul diff --git a/app/views/messages/activity.html.haml b/app/views/messages/activity.html.haml index fba7341..6e28cca 100644 --- a/app/views/messages/activity.html.haml +++ b/app/views/messages/activity.html.haml @@ -11,7 +11,7 @@ - for entry in @entries.reverse - if entry.is_a?(Postal::MessageDB::Delivery) %li.messageActivity__event - %p.messageActivity__timestamp= entry.timestamp.to_s(:long) + %p.messageActivity__timestamp= entry.timestamp.to_fs(:long) .messageActivity__details.messageActivity--detailsDelivery %p.messageActivity__subject =# entry.status.underscore.humanize @@ -21,20 +21,20 @@ - elsif entry.is_a?(Postal::MessageDB::Click) %li.messageActivity__event - %p.messageActivity__timestamp= entry.timestamp.to_s(:long) + %p.messageActivity__timestamp= entry.timestamp.to_fs(:long) .messageActivity__details.messageActivity--detailsClick %p.messageActivity__subject Click for #{entry.url} %p.messageActivity__extra Clicked from #{entry.ip_address} (#{entry.user_agent}) - elsif entry.is_a?(Postal::MessageDB::Load) %li.messageActivity__event - %p.messageActivity__timestamp= entry.timestamp.to_s(:long) + %p.messageActivity__timestamp= entry.timestamp.to_fs(:long) .messageActivity__details.messageActivity--detailsLoad %p.messageActivity__subject Message Viewed %p.messageActivity__extra Opened from #{entry.ip_address} (#{entry.user_agent}) %li.messageActivity__event - %p.messageActivity__timestamp= @message.timestamp.to_s(:long) + %p.messageActivity__timestamp= @message.timestamp.to_fs(:long) .messageActivity__details %p.messageActivity__subject Message received by Postal diff --git a/app/views/messages/suppressions.html.haml b/app/views/messages/suppressions.html.haml index 7fbfee8..ffc97a9 100644 --- a/app/views/messages/suppressions.html.haml +++ b/app/views/messages/suppressions.html.haml @@ -22,9 +22,9 @@ %p.suppressionList__address= link_to suppression['address'], outgoing_organization_server_messages_path(organization, @server, :query => "to: #{suppression['address']}") %p.suppressionList__reason= suppression['reason'].capitalize .suppressionList__right - %p.suppressionList__timestamp Added #{Time.zone.at(suppression['timestamp']).to_s(:long)} + %p.suppressionList__timestamp Added #{Time.zone.at(suppression['timestamp']).to_fs(:long)} %p.suppressionList__timestamp - Expires #{Time.zone.at(suppression['keep_until']).to_s(:long)} + Expires #{Time.zone.at(suppression['keep_until']).to_fs(:long)} - if suppression['keep_until'] < Time.now.to_f %span.u-red expired = render 'shared/message_db_pagination', :data => @suppressions, :name => "suppression" diff --git a/app/views/servers/show.html.haml b/app/views/servers/show.html.haml index 3f6558c..3ae1b42 100644 --- a/app/views/servers/show.html.haml +++ b/app/views/servers/show.html.haml @@ -35,7 +35,7 @@ %li #{@first_date.strftime("%A at %l%P")} → %li Today at #{Time.now.strftime("%l%P")} - else - %li #{@first_date.to_date.to_s(:long)} → + %li #{@first_date.to_date.to_fs(:long)} → %li Today .titleWithLinks.u-margin @@ -44,4 +44,3 @@ %li= link_to "View message queue", [:queue, organization, @server], :class => 'titleWithLinks__link' %li= link_to "View full e-mail history", [:outgoing, organization, @server, :messages], :class => 'titleWithLinks__link' = render 'messages/list', :messages => @messages - diff --git a/config/application.rb b/config/application.rb index 13b4677..3d54d96 100644 --- a/config/application.rb +++ b/config/application.rb @@ -17,7 +17,7 @@ Bundler.require(*Rails.groups) module Postal class Application < Rails::Application - config.load_defaults 6.0 + config.load_defaults 7.0 # Disable most generators config.generators do |g| diff --git a/config/boot.rb b/config/boot.rb index 73db71c..192372b 100644 --- a/config/boot.rb +++ b/config/boot.rb @@ -2,9 +2,6 @@ ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__) -$stdout.sync = true -$stderr.sync = true - require "bundler/setup" # Set up gems listed in the Gemfile. require_relative "../lib/postal/config" diff --git a/config/initializers/content_security_policy.rb b/config/initializers/content_security_policy.rb new file mode 100644 index 0000000..691cfa1 --- /dev/null +++ b/config/initializers/content_security_policy.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true +# Be sure to restart your server when you modify this file. + +# Define an application-wide content security policy. +# See the Securing Rails Applications Guide for more information: +# https://guides.rubyonrails.org/security.html#content-security-policy-header + +# Rails.application.configure do +# config.content_security_policy do |policy| +# policy.default_src :self, :https +# policy.font_src :self, :https, :data +# policy.img_src :self, :https, :data +# policy.object_src :none +# policy.script_src :self, :https +# policy.style_src :self, :https +# # Specify URI for violation reports +# # policy.report_uri "/csp-violation-report-endpoint" +# end +# +# # Generate session nonces for permitted importmap and inline scripts +# config.content_security_policy_nonce_generator = ->(request) { request.session.id.to_s } +# config.content_security_policy_nonce_directives = %w(script-src) +# +# # Report violations without enforcing the policy. +# # config.content_security_policy_report_only = true +# end diff --git a/config/initializers/filter_parameter_logging.rb b/config/initializers/filter_parameter_logging.rb index 7a4f47b..ca55f95 100644 --- a/config/initializers/filter_parameter_logging.rb +++ b/config/initializers/filter_parameter_logging.rb @@ -2,5 +2,9 @@ # Be sure to restart your server when you modify this file. -# Configure sensitive parameters which will be filtered from the log file. -Rails.application.config.filter_parameters += [:password] +# Configure parameters to be filtered from the log file. Use this to limit dissemination of +# sensitive information. See the ActiveSupport::ParameterFilter documentation for supported +# notations and behaviors. +Rails.application.config.filter_parameters += [ + :passw, :secret, :token, :_key, :crypt, :salt, :certificate, :otp, :ssn +] diff --git a/config/initializers/new_framework_defaults.rb b/config/initializers/new_framework_defaults.rb deleted file mode 100644 index 420535c..0000000 --- a/config/initializers/new_framework_defaults.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -# Be sure to restart your server when you modify this file. -# -# This file contains migration options to ease your Rails 5.0 upgrade. -# -# Read the Rails 5.0 release notes for more info on each option. - -# Enable per-form CSRF tokens. Previous versions had false. -Rails.application.config.action_controller.per_form_csrf_tokens = true - -# Enable origin-checking CSRF mitigation. Previous versions had false. -Rails.application.config.action_controller.forgery_protection_origin_check = true - -# Make Ruby 2.4 preserve the timezone of the receiver when calling `to_time`. -# Previous versions had false. -ActiveSupport.to_time_preserves_timezone = true - -# Require `belongs_to` associations by default. Previous versions had false. -Rails.application.config.active_record.belongs_to_required_by_default = true - -# Configure SSL options to enable HSTS with subdomains. Previous versions had false. -Rails.application.config.ssl_options = false diff --git a/config/initializers/new_framework_defaults_7_0.rb b/config/initializers/new_framework_defaults_7_0.rb new file mode 100644 index 0000000..a13554e --- /dev/null +++ b/config/initializers/new_framework_defaults_7_0.rb @@ -0,0 +1,142 @@ +# frozen_string_literal: true +# Be sure to restart your server when you modify this file. +# +# This file eases your Rails 7.0 framework defaults upgrade. +# +# Uncomment each configuration one by one to switch to the new default. +# Once your application is ready to run with all new defaults, you can remove +# this file and set the `config.load_defaults` to `7.0`. +# +# Read the Guide for Upgrading Ruby on Rails for more info on each option. +# https://guides.rubyonrails.org/upgrading_ruby_on_rails.html + +# `button_to` view helper will render `