ghaymah_dev/postal
1
0
Fork 0
mirror of https://github.com/postalserver/postal.git synced 2026-10-20 11:17:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2803 from postalserver/v3

Browse Source 
Postal v3
This commit is contained in:
Adam Cooke
2024-03-04 10:28:51 +00:00
committed by GitHub
parent 66d26c8d4f e4638a0c40
commit 2359829e22
241 changed files with 11769 additions and 4717 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/postal/app_logger.rb
-64
View File
@@ -1,64 +0,0 @@
# frozen_string_literal: true
require "logger"
module Postal
class AppLogger < Logger
def initialize(log_name, *args)
@log_name = log_name
super(*args)
self.formatter = LogFormatter.new
end
def add(severity, message = nil, progname = nil)
super
if severity >= @level && n = self.class.graylog_notifier
begin
if message.nil?
message = block_given? ? yield : progname
end
message = message.to_s.force_encoding("UTF-8").scrub
message_without_ansi = begin
message.gsub(/\e\[([\d;]+)?m/, "")
rescue StandardError
message
end
n.notify!(short_message: message_without_ansi, log_name: @log_name, facility: "postal", application_name: "postal", process_name: ENV.fetch("PROC_NAME", nil), pid: Process.pid)
rescue StandardError
# Can't log this to GELF. Soz.
end
end
true
end
def self.graylog?
!!Postal.config.logging.graylog&.host
end
def self.graylog_notifier
@graylog_notifier ||= graylog? ? GELF::Notifier.new(Postal.config.logging.graylog.host, Postal.config.logging.graylog.port) : nil
end
end
class LogFormatter
TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%3N"
COLORS = [32, 34, 35, 31, 32, 33].freeze
def call(severity, datetime, progname, msg)
time = datetime.strftime(TIME_FORMAT)
if number = ENV["PROC_NAME"]
id = number.split(".").last.to_i
proc_text = "\e[#{COLORS[id % COLORS.size]}m[#{ENV['PROC_NAME']}:#{Process.pid}]\e[0m"
else
proc_text = "[#{Process.pid}]"
end
"#{proc_text} [#{time}] #{severity} -- : #{msg}\n"
end
end
end
lib/postal/bounce_message.rb
-57
View File
@@ -1,57 +0,0 @@
# frozen_string_literal: true
module Postal
class BounceMessage
def initialize(server, message)
@server = server
@message = message
end
def raw_message
mail = Mail.new
mail.to = @message.mail_from
mail.from = "Mail Delivery Service <#{@message.route.description}>"
mail.subject = "Mail Delivery Failed (#{@message.subject})"
mail.text_part = body
mail.attachments["Original Message.eml"] = { mime_type: "message/rfc822", encoding: "quoted-printable", content: @message.raw_message }
mail.message_id = "<#{SecureRandom.uuid}@#{Postal.config.dns.return_path}>"
mail.to_s
end
def queue
message = @server.message_db.new_message
message.scope = "outgoing"
message.rcpt_to = @message.mail_from
message.mail_from = @message.route.description
message.domain_id = @message.domain&.id
message.raw_message = raw_message
message.bounce = true
message.bounce_for_id = @message.id
message.save
message.id
end
def postmaster_address
@server.postmaster_address || "postmaster@#{@message.domain&.name || Postal.config.web.host}"
end
private
def body
<<~BODY
This is the mail delivery service responsible for delivering mail to #{@message.route.description}.
The message you've sent cannot be delivered. Your original message is attached to this message.
For further assistance please contact #{postmaster_address}. Please include the details below to help us identify the issue.
Message Token: #{@message.token}@#{@server.token}
Orginal Message ID: #{@message.message_id}
Mail from: #{@message.mail_from}
Rcpt To: #{@message.rcpt_to}
BODY
end
end
end
lib/postal/config.rb
+102 -159
View File
@@ -6,185 +6,128 @@ require "pathname"
require "cgi"
require "openssl"
require "fileutils"
require "konfig"
require "konfig/sources/environment"
require "konfig/sources/yaml"
require "dotenv"
require "klogger"
require_relative "error"
require_relative "version"
require_relative "config_schema"
require_relative "legacy_config_source"
module Postal
# rubocop:disable Lint/EmptyClass
class Config
end
# rubocop:enable Lint/EmptyClass
class << self
def self.host
@host ||= config.web.host || "localhost:5000"
end
def self.protocol
@protocol ||= config.web.protocol || "http"
end
def self.host_with_protocol
@host_with_protocol ||= "#{protocol}://#{host}"
end
def self.app_root
@app_root ||= Pathname.new(File.expand_path("../..", __dir__))
end
def self.config
@config ||= begin
require "hashie/mash"
config = Hashie::Mash.new(defaults)
config = config.deep_merge(yaml_config)
config.deep_merge(local_yaml_config)
# Return the path to the config file
#
# @return [String]
def config_file_path
ENV.fetch("POSTAL_CONFIG_FILE_PATH", "config/postal/postal.yml")
end
end
def self.config_root
if ENV["POSTAL_CONFIG_ROOT"]
@config_root ||= Pathname.new(ENV["POSTAL_CONFIG_ROOT"])
else
@config_root ||= Pathname.new(File.expand_path("../../config/postal", __dir__))
end
end
def initialize_config
sources = []
def self.log_root
if config.logging.root
@log_root ||= Pathname.new(config.logging.root)
else
@log_root ||= app_root.join("log")
end
end
# Load environment variables to begin with. Any config provided
# by an environment variable will override any provided in the
# config file.
Dotenv.load(".env")
sources << Konfig::Sources::Environment.new(ENV)
def self.config_file_path
if env == "default"
@config_file_path ||= File.join(config_root, "postal.yml")
else
@config_file_path ||= File.join(config_root, "postal.#{env}.yml")
end
end
# If a config file exists, we need to load that. Config files can
# either be legacy (v1) or new (v2). Any file without a 'version'
# key is a legacy file whereas new-style config files will include
# the 'version: 2' key/value.
if File.file?(config_file_path)
puts "Loading config from #{config_file_path}"
def self.env
@env ||= ENV.fetch("POSTAL_ENV", "default")
end
def self.yaml_config
@yaml_config ||= File.exist?(config_file_path) ? YAML.load_file(config_file_path) : {}
end
def self.local_config_file_path
@local_config_file_path ||= File.join(config_root, "postal.local.yml")
end
def self.local_yaml_config
@local_yaml_config ||= File.exist?(local_config_file_path) ? YAML.load_file(local_config_file_path) : {}
end
def self.defaults_file_path
@defaults_file_path ||= app_root.join("config", "postal.defaults.yml")
end
def self.defaults
@defaults ||= begin
file = File.read(defaults_file_path)
yaml = ERB.new(file).result
YAML.safe_load(yaml)
end
end
def self.logger_for(name)
@loggers ||= {}
@loggers[name.to_sym] ||= begin
require "postal/app_logger"
if config.logging.stdout || ENV["LOG_TO_STDOUT"]
Postal::AppLogger.new(name, $stdout)
config_file = File.read(config_file_path)
yaml = YAML.safe_load(config_file)
config_version = yaml["version"] || 1
case config_version
when 1
puts "WARNING: Using legacy config file format. Upgrade your postal.yml to use"
puts "version 2 of the Postal configuration or configure using environment"
puts "variables. See https://postalserver.io/config-v2 for details."
sources << LegacyConfigSource.new(yaml)
when 2
sources << Konfig::Sources::YAML.new(config_file)
else
raise "Invalid version specified in Postal config file. Must be 1 or 2."
end
else
FileUtils.mkdir_p(log_root)
Postal::AppLogger.new(name, log_root.join("#{name}.log"), config.logging.max_log_files, config.logging.max_log_file_size.megabytes)
puts "No configuration file found at #{config_file_path}"
puts "Only using environment variables for configuration"
end
# Build configuration with the provided sources.
Konfig::Config.build(ConfigSchema, sources: sources)
end
def host_with_protocol
@host_with_protocol ||= "#{Config.postal.web_protocol}://#{Config.postal.web_hostname}"
end
def logger
@logger ||= begin
k = Klogger.new(nil, destination: Config.logging.enabled? ? $stdout : "/dev/null", highlight: Config.logging.highlighting_enabled?)
k.add_destination(graylog_logging_destination) if Config.gelf.host.present?
k
end
end
end
def self.process_name
@process_name ||= begin
string = "host:#{Socket.gethostname} pid:#{Process.pid}"
string += " procname:#{ENV['PROC_NAME']}" if ENV["PROC_NAME"]
def process_name
@process_name ||= begin
"host:#{Socket.gethostname} pid:#{Process.pid}"
rescue StandardError
"pid:#{Process.pid}"
end
end
def locker_name
string = process_name.dup
string += " job:#{Thread.current[:job_id]}" if Thread.current[:job_id]
string += " thread:#{Thread.current.native_thread_id}"
string
rescue StandardError
"pid:#{Process.pid}"
end
end
def self.locker_name
string = process_name.dup
string += " job:#{Thread.current[:job_id]}" if Thread.current[:job_id]
string
end
def self.smtp_from_name
config.smtp&.from_name || "Postal"
end
def self.smtp_from_address
config.smtp&.from_address || "postal@example.com"
end
def self.smtp_private_key_path
config.smtp_server.tls_private_key_path || config_root.join("smtp.key")
end
def self.smtp_private_key
@smtp_private_key ||= OpenSSL::PKey.read(File.read(smtp_private_key_path))
end
def self.smtp_certificate_path
config.smtp_server.tls_certificate_path || config_root.join("smtp.cert")
end
def self.smtp_certificate_data
@smtp_certificate_data ||= File.read(smtp_certificate_path)
end
def self.smtp_certificates
@smtp_certificates ||= begin
certs = smtp_certificate_data.scan(/-----BEGIN CERTIFICATE-----.+?-----END CERTIFICATE-----/m)
certs.map do |c|
OpenSSL::X509::Certificate.new(c)
end.freeze
end
end
def self.signing_key_path
ENV.fetch("POSTAL_SIGNING_KEY_PATH") { config_root.join("signing.key") }
end
def self.signing_key
@signing_key ||= OpenSSL::PKey::RSA.new(File.read(signing_key_path))
end
def self.rp_dkim_dns_record
public_key = signing_key.public_key.to_s.gsub(/-+[A-Z ]+-+\n/, "").gsub(/\n/, "")
"v=DKIM1; t=s; h=sha256; p=#{public_key};"
end
class ConfigError < Postal::Error
end
def self.check_config!
return if ENV["POSTAL_SKIP_CONFIG_CHECK"].to_i == 1
unless File.exist?(config_file_path)
raise ConfigError, "No config found at #{config_file_path}"
end
return if File.exist?(signing_key_path)
def locker_name_with_suffix(suffix)
"#{locker_name} #{suffix}"
end
def signing_key
@signing_key ||= OpenSSL::PKey::RSA.new(File.read(Config.postal.signing_key_path))
end
def rp_dkim_dns_record
public_key = signing_key.public_key.to_s.gsub(/-+[A-Z ]+-+\n/, "").gsub(/\n/, "")
"v=DKIM1; t=s; h=sha256; p=#{public_key};"
end
def ip_pools?
Config.postal.use_ip_pools?
end
def graylog_logging_destination
@graylog_logging_destination ||= begin
notifier = GELF::Notifier.new(Config.gelf.host, Config.gelf.port, "WAN")
proc do |_logger, payload, group_ids|
short_message = payload.delete(:message) || "[message missing]"
notifier.notify!(short_message: short_message, **{
facility: Config.gelf.facility,
_environment: Config.rails.environment,
_version: Postal::VERSION.to_s,
_group_ids: group_ids.join(" ")
}.merge(payload.transform_keys { |k| "_#{k}".to_sym }.transform_values(&:to_s)))
end
end
end
raise ConfigError, "No signing key found at #{signing_key_path}"
end
def self.ip_pools?
config.general.use_ip_pools?
end
Config = initialize_config
end
lib/postal/config_schema.rb
+479
View File
@@ -0,0 +1,479 @@
# frozen_string_literal: true
require "uri"
module Postal
# REMEMBER: If you change the schema, remember to regenerate the configuration docs
# using the rake command below:
#
# rake postal:generate_config_docs
ConfigSchema = Konfig::Schema.draw do
group :postal do
string :web_hostname do
description "The hostname that the Postal web interface runs on"
default "postal.example.com"
end
string :web_protocol do
description "The HTTP protocol to use for the Postal web interface"
default "https"
end
string :smtp_hostname do
description "The hostname that the Postal SMTP server runs on"
default "postal.example.com"
end
boolean :use_ip_pools do
description "Should IP pools be enabled for this installation?"
default false
end
integer :default_maximum_delivery_attempts do
description "The maximum number of delivery attempts"
default 18
end
integer :default_maximum_hold_expiry_days do
description "The number of days to hold a message before they will be expired"
default 7
end
integer :default_suppression_list_automatic_removal_days do
description "The number of days an address will remain in a suppression list before being removed"
default 30
end
integer :default_spam_threshold do
description "The default threshold at which a message should be treated as spam"
default 5
end
integer :default_spam_failure_threshold do
description "The default threshold at which a message should be treated as spam failure"
default 20
end
boolean :use_local_ns_for_domain_verification do
description "Domain verification and checking usually checks with a domain's nameserver. Enable this to check with the server's local nameservers."
default false
end
boolean :use_resent_sender_header do
description "Append a Resend-Sender header to all outgoing e-mails"
default true
end
string :signing_key_path do
description "Path to the private key used for signing"
default "config/postal/signing.key"
end
string :smtp_relays do
array
description "An array of SMTP relays in the format of smtp://host:port"
transform do |value|
uri = URI.parse(value)
query = uri.query ? CGI.parse(uri.query) : {}
{
host: uri.host,
port: uri.port || 25,
ssl_mode: query["ssl_mode"]&.first || "Auto"
}
end
end
end
group :web_server do
integer :default_port do
description "The default port the web server should listen on unless overriden by the PORT environment variable"
default 5000
end
string :default_bind_address do
description "The default bind address the web server should listen on unless overriden by the BIND_ADDRESS environment variable"
default "127.0.0.1"
end
integer :max_threads do
description "The maximum number of threads which can be used by the web server"
default 5
end
end
group :worker do
integer :default_health_server_port do
description "The default port for the worker health server to listen on"
default 9090
end
string :default_health_server_bind_address do
description "The default bind address for the worker health server to listen on"
default "127.0.0.1"
end
end
group :main_db do
string :host do
description "Hostname for the main MariaDB server"
default "localhost"
end
integer :port do
description "The MariaDB port to connect to"
default 3306
end
string :username do
description "The MariaDB username"
default "postal"
end
string :password do
description "The MariaDB password"
end
string :database do
description "The MariaDB database name"
default "postal"
end
integer :pool_size do
description "The maximum size of the MariaDB connection pool"
default 5
end
string :encoding do
description "The encoding to use when connecting to the MariaDB database"
default "utf8mb4"
end
end
group :message_db do
string :host do
description "Hostname for the MariaDB server which stores the mail server databases"
default "localhost"
end
integer :port do
description "The MariaDB port to connect to"
default 3306
end
string :username do
description "The MariaDB username"
default "postal"
end
string :password do
description "The MariaDB password"
end
string :encoding do
description "The encoding to use when connecting to the MariaDB database"
default "utf8mb4"
end
string :database_name_prefix do
description "The MariaDB prefix to add to database names"
default "postal"
end
end
group :logging do
boolean :rails_log_enabled do
description "Enable the default Rails logger"
default false
end
string :sentry_dsn do
description "A DSN which should be used to report exceptions to Sentry"
end
boolean :enabled do
description "Enable the Postal logger to log to STDOUT"
default true
end
boolean :highlighting_enabled do
description "Enable highlighting of log lines"
default false
end
end
group :gelf do
string :host do
description "GELF-capable host to send logs to"
end
integer :port do
description "GELF port to send logs to"
default 12_201
end
string :facility do
description "The facility name to add to all log entries sent to GELF"
default "postal"
end
end
group :smtp_server do
integer :default_port do
description "The default port the SMTP server should listen on unless overriden by the PORT environment variable"
default 25
end
string :default_bind_address do
description "The default bind address the SMTP server should listen on unless overriden by the BIND_ADDRESS environment variable"
default "127.0.0.1"
end
integer :default_health_server_port do
description "The default port for the SMTP server health server to listen on"
default 9091
end
string :default_health_server_bind_address do
description "The default bind address for the SMTP server health server to listen on"
default "127.0.0.1"
end
boolean :tls_enabled do
description "Enable TLS for the SMTP server (requires certificate)"
default false
end
string :tls_certificate_path do
description "The path to the SMTP server's TLS certificate"
default "config/postal/smtp.cert"
end
string :tls_private_key_path do
description "The path to the SMTP server's TLS private key"
default "config/postal/smtp.key"
end
string :tls_ciphers do
description "Override ciphers to use for SSL"
end
string :ssl_version do
description "The SSL versions which are supported"
default "SSLv23"
end
boolean :proxy_protocol do
description "Enable proxy protocol for use behind some load balancers"
default false
end
boolean :log_connections do
description "Enable connection logging"
default false
end
integer :max_message_size do
description "The maximum message size to accept from the SMTP server (in MB)"
default 14
end
string :log_ip_address_exclusion_matcher do
description "A regular expression to use to exclude connections from logging"
end
end
group :dns do
string :mx_records do
description "The names of the default MX records"
array
default ["mx1.postal.example.com", "mx2.postal.example.com"]
end
string :spf_include do
description "The location of the SPF record"
default "spf.postal.example.com"
end
string :return_path_domain do
description "The return path hostname"
default "rp.postal.example.com"
end
string :route_domain do
description "The domain to use for hosting route-specific addresses"
default "routes.postal.example.com"
end
string :track_domain do
description "The CNAME which tracking domains should be pointed to"
default "track.postal.example.com"
end
string :helo_hostname do
description "The hostname to use in HELO/EHLO when connecting to external SMTP servers"
end
string :dkim_identifier do
description "The identifier to use for DKIM keys in DNS records"
default "postal"
end
string :domain_verify_prefix do
description "The prefix to add before TXT record verification string"
default "postal-verification"
end
string :custom_return_path_prefix do
description "The domain to use on external domains which points to the Postal return path domain"
default "psrp"
end
integer :timeout do
description "The timeout to wait for DNS resolution"
default 5
end
string :resolv_conf_path do
description "The path to the resolv.conf file containing addresses for local nameservers"
default "/etc/resolv.conf"
end
end
group :smtp do
string :host do
description "The hostname to send application-level e-mails to"
default "127.0.0.1"
end
integer :port do
description "The port number to send application-level e-mails to"
default 25
end
string :username do
description "The username to use when authentication to the SMTP server"
end
string :password do
description "The password to use when authentication to the SMTP server"
end
string :from_name do
description "The name to use as the from name outgoing emails from Postal"
default "Postal"
end
string :from_address do
description "The e-mail to use as the from address outgoing emails from Postal"
default "postal@example.com"
end
end
group :rails do
string :environment do
description "The Rails environment to run the application in"
default "production"
end
string :secret_key do
description "The secret key used to sign and encrypt cookies and session data in the application"
end
end
group :rspamd do
boolean :enabled do
description "Enable rspamd for message inspection"
default false
end
string :host do
description "The hostname of the rspamd server"
default "127.0.0.1"
end
integer :port do
description "The port of the rspamd server"
default 11_334
end
boolean :ssl do
description "Enable SSL for the rspamd connection"
default false
end
string :password do
description "The password for the rspamd server"
end
string :flags do
description "Any flags for the rspamd server"
end
end
group :spamd do
boolean :enabled do
description "Enable SpamAssassin for message inspection"
default false
end
string :host do
description "The hostname for the SpamAssassin server"
default "127.0.0.1"
end
integer :port do
description "The port of the SpamAssassin server"
default 783
end
end
group :clamav do
boolean :enabled do
description "Enable ClamAV for message inspection"
default false
end
string :host do
description "The host of the ClamAV server"
default "127.0.0.1"
end
integer :port do
description "The port of the ClamAV server"
default 2000
end
end
group :smtp_client do
integer :open_timeout do
description "The open timeout for outgoing SMTP connections"
default 30
end
integer :read_timeout do
description "The read timeout for outgoing SMTP connections"
default 30
end
end
group :migration_waiter do
boolean :enabled do
description "Wait for all migrations to run before starting a process"
default false
end
integer :attempts do
description "The number of attempts to try waiting for migrations to complete before start"
default 120
end
integer :sleep_time do
description "The number of seconds to wait between each migration check"
default 2
end
end
end
end
lib/postal/dkim_header.rb
-132
View File
@@ -1,132 +0,0 @@
# frozen_string_literal: true
module Postal
class DKIMHeader
def initialize(domain, message)
if domain && domain.dkim_status == "OK"
@domain_name = domain.name
@dkim_key = domain.dkim_key
@dkim_identifier = domain.dkim_identifier
else
@domain_name = Postal.config.dns.return_path
@dkim_key = Postal.signing_key
@dkim_identifier = Postal.config.dns.dkim_identifier
end
@domain = domain
@message = message
@raw_headers, @raw_body = @message.gsub(/\r?\n/, "\r\n").split(/\r\n\r\n/, 2)
end
def dkim_header
"DKIM-Signature: v=1; " + dkim_properties.join("\r\n\t") + signature.scan(/.{1,72}/).join("\r\n\t")
end
private
def headers
@headers ||= @raw_headers.to_s.gsub(/\r?\n\s/, " ").split(/\r?\n/)
end
def header_names
normalized_headers.map { |h| h.split(":")[0].strip }
end
def normalized_headers
[].tap do |new_headers|
dkim_headers = headers.select do |h|
h.match(/
^(
from|sender|reply-to|subject|date|message-id|to|cc|mime-version|content-type|content-transfer-encoding|
resent-to|resent-cc|resent-from|resent-sender|resent-message-id|in-reply-to|references|list-id|list-help|
list-owner|list-unsubscribe|list-subscribe|list-post
):/ix)
end
dkim_headers.each do |h|
new_headers << normalize_header(h)
end
end
end
def normalize_header(content)
content = content.dup
# From the DKIM RFC6376
# https://datatracker.ietf.org/doc/html/rfc6376#section-3.4.2
# Split the key and value.
key, value = content.split(":", 2)
# Convert all header field names (not the header field values) to
# lowercase. For example, convert "SUBJect: AbC" to "subject: AbC".
key.downcase!
# Unfold all header field continuation lines as described in [RFC5322]
value.gsub!(/\r?\n[ \t]+/, " ")
# Convert all sequences of one or more WSP characters to a single SP character.
value.gsub!(/[ \t]+/, " ")
# Delete all WSP characters at the end of each unfolded header field value.
value.gsub!(/[ \t]*\z/, "")
# Delete any WSP characters remaining after the colon separating the header field name from the header field value.
value.gsub!(/\A[ \t]*/, "")
# Join together
key + ":" + value
end
def normalized_body
@normalized_body ||= begin
content = @raw_body.dup
# From the DKIM RFC6376
# https://datatracker.ietf.org/doc/html/rfc6376#section-3.4.4
# a. Reduce whitespace
#
# * Reduce all sequences of WSP within a line to a single SP character.
content.gsub!(/[ \t]+/, " ")
# * Ignore all whitespace at the end of lines. Implementations MUST NOT
# remove the CRLF at the end of the line.
content.gsub!(/ \r\n/, "\r\n")
# b. Ignore all empty lines at the end of the message body.
content.gsub!(/[ \r\n]*\z/, "")
content += "\r\n"
content
end
end
def body_hash
@body_hash ||= Base64.encode64(Digest::SHA256.digest(normalized_body)).strip
end
def dkim_properties
@dkim_properties ||= [].tap do |header|
header << "a=rsa-sha256; c=relaxed/relaxed;"
header << "d=#{@domain_name};"
header << "s=#{@dkim_identifier}; t=#{Time.now.utc.to_i};"
header << "bh=#{body_hash};"
header << "h=#{header_names.join(':')};"
header << "b="
end
end
def dkim_header_for_signing
"dkim-signature:v=1; #{dkim_properties.join(' ')}"
end
def signable_header_string
(normalized_headers + [dkim_header_for_signing]).join("\r\n")
end
def signature
Base64.encode64(@dkim_key.sign(OpenSSL::Digest.new("SHA256"), signable_header_string)).gsub("\n", "")
end
end
end
lib/postal/http_sender.rb
-136
View File
@@ -1,136 +0,0 @@
# frozen_string_literal: true
module Postal
class HTTPSender < Sender
def initialize(endpoint, options = {})
super()
@endpoint = endpoint
@options = options
@log_id = Nifty::Utils::RandomString.generate(length: 8).upcase
end
def send_message(message)
start_time = Time.now
result = SendResult.new
result.log_id = @log_id
request_options = {}
request_options[:sign] = true
request_options[:timeout] = @endpoint.timeout || 5
case @endpoint.encoding
when "BodyAsJSON"
request_options[:json] = parameters(message, flat: false).to_json
when "FormData"
request_options[:params] = parameters(message, flat: true)
end
log "Sending request to #{@endpoint.url}"
response = Postal::HTTP.post(@endpoint.url, request_options)
result.secure = !!response[:secure] # rubocop:disable Style/DoubleNegation
result.details = "Received a #{response[:code]} from #{@endpoint.url}"
log " -> Received: #{response[:code]}"
if response[:body]
log " -> Body: #{response[:body][0, 255]}"
result.output = response[:body].to_s[0, 500].strip
end
if response[:code] >= 200 && response[:code] < 300
#  This is considered a success
result.type = "Sent"
elsif response[:code] >= 500 && response[:code] < 600
# This is temporary. They might fix their server so it should soft fail.
result.type = "SoftFail"
result.retry = true
elsif response[:code].negative?
# Connection/SSL etc... errors
result.type = "SoftFail"
result.retry = true
result.connect_error = true
elsif response[:code] == 429
# Rate limit exceeded, treat as a hard fail and don't send bounces
result.type = "HardFail"
result.suppress_bounce = true
else
# This is permanent. Any other error isn't cool with us.
result.type = "HardFail"
end
result.time = (Time.now - start_time).to_f.round(2)
result
end
private
def log(text)
Postal.logger_for(:http_sender).info("[#{@log_id}] #{text}")
end
def parameters(message, options = {})
case @endpoint.format
when "Hash"
hash = {
id: message.id,
rcpt_to: message.rcpt_to,
mail_from: message.mail_from,
token: message.token,
subject: message.subject,
message_id: message.message_id,
timestamp: message.timestamp.to_f,
size: message.size,
spam_status: message.spam_status,
bounce: message.bounce,
received_with_ssl: message.received_with_ssl,
to: message.headers["to"]&.last,
cc: message.headers["cc"]&.last,
from: message.headers["from"]&.last,
date: message.headers["date"]&.last,
in_reply_to: message.headers["in-reply-to"]&.last,
references: message.headers["references"]&.last,
html_body: message.html_body,
attachment_quantity: message.attachments.size,
auto_submitted: message.headers["auto-submitted"]&.last,
reply_to: message.headers["reply-to"]
}
if @endpoint.strip_replies
hash[:plain_body], hash[:replies_from_plain_body] = Postal::ReplySeparator.separate(message.plain_body)
else
hash[:plain_body] = message.plain_body
end
if @endpoint.include_attachments?
if options[:flat]
message.attachments.each_with_index do |a, i|
hash["attachments[#{i}][filename]"] = a.filename
hash["attachments[#{i}][content_type]"] = a.content_type
hash["attachments[#{i}][size]"] = a.body.to_s.bytesize.to_s
hash["attachments[#{i}][data]"] = Base64.encode64(a.body.to_s)
end
else
hash[:attachments] = message.attachments.map do |a|
{
filename: a.filename,
content_type: a.mime_type,
size: a.body.to_s.bytesize,
data: Base64.encode64(a.body.to_s)
}
end
end
end
hash
when "RawMessage"
{
id: message.id,
rcpt_to: message.rcpt_to,
mail_from: message.mail_from,
message: Base64.encode64(message.raw_message),
base64: true,
size: message.size.to_i
}
else
{}
end
end
end
end
lib/postal/job.rb
-44
View File
@@ -1,44 +0,0 @@
# frozen_string_literal: true
require "nifty/utils/random_string"
module Postal
class Job
def initialize(id, params = {})
@id = id
@params = params
on_initialize
end
attr_reader :id
def params
@params || {}
end
def on_initialize
# Called whenever the class is initialized. Can be overriden.
end
def on_error(exception)
# Called if there's an exception while processing the perform block.
# Receives the exception.
end
def perform
end
def log(text)
Worker.logger.info "[#{@id}] #{text}"
end
def self.queue(queue, params = {})
job_id = Nifty::Utils::RandomString.generate(length: 10).upcase
job_payload = { "params" => params, "class_name" => name, "id" => job_id, "queue" => queue }
Postal::Worker.job_queue(queue).publish(job_payload.to_json, persistent: false)
job_id
end
end
end
lib/postal/legacy_config_source.rb
+122
View File
@@ -0,0 +1,122 @@
# frozen_string_literal: true
require "konfig/sources/abstract"
require "konfig/error"
module Postal
class LegacyConfigSource < Konfig::Sources::Abstract
# This maps all the new configuration values to where they
# exist in the old YAML file. The source will load any YAML
# file that has been provided to this source in order. A
# warning will be generated to the console for configuration
# loaded from this format.
MAPPING = {
"postal.web_hostname" => -> (c) { c.dig("web", "host") },
"postal.web_protocol" => -> (c) { c.dig("web", "protocol") },
"postal.smtp_hostname" => -> (c) { c.dig("dns", "smtp_server_hostname") },
"postal.use_ip_pools" => -> (c) { c.dig("general", "use_ip_pools") },
"logging.sentry_dsn" => -> (c) { c.dig("general", "exception_url") },
"postal.default_maximum_delivery_attempts" => -> (c) { c.dig("general", "maximum_delivery_attempts") },
"postal.default_maximum_hold_expiry_days" => -> (c) { c.dig("general", "maximum_hold_expiry_days") },
"postal.default_suppression_list_automatic_removal_days" => -> (c) { c.dig("general", "suppression_list_removal_delay") },
"postal.use_local_ns_for_domain_verification" => -> (c) { c.dig("general", "use_local_ns_for_domains") },
"postal.default_spam_threshold" => -> (c) { c.dig("general", "default_spam_threshold") },
"postal.default_spam_failure_threshold" => -> (c) { c.dig("general", "default_spam_failure_threshold") },
"postal.use_resent_sender_header" => -> (c) { c.dig("general", "use_resent_sender_header") },
# SMTP relays must be converted to the new URI style format and they'll
# then be transformed back to a hash by the schema transform.
"postal.smtp_relays" => -> (c) { c["smtp_relays"]&.map { |r| "smtp://#{r['hostname']}:#{r['port']}?ssl_mode=#{r['ssl_mode']}" } },
"web_server.default_bind_address" => -> (c) { c.dig("web_server", "bind_address") },
"web_server.default_port" => -> (c) { c.dig("web_server", "port") },
"web_server.max_threads" => -> (c) { c.dig("web_server", "max_threads") },
"main_db.host" => -> (c) { c.dig("main_db", "host") },
"main_db.port" => -> (c) { c.dig("main_db", "port") },
"main_db.username" => -> (c) { c.dig("main_db", "username") },
"main_db.password" => -> (c) { c.dig("main_db", "password") },
"main_db.database" => -> (c) { c.dig("main_db", "database") },
"main_db.pool_size" => -> (c) { c.dig("main_db", "pool_size") },
"main_db.encoding" => -> (c) { c.dig("main_db", "encoding") },
"message_db.host" => -> (c) { c.dig("message_db", "host") },
"message_db.port" => -> (c) { c.dig("message_db", "port") },
"message_db.username" => -> (c) { c.dig("message_db", "username") },
"message_db.password" => -> (c) { c.dig("message_db", "password") },
"message_db.database_name_prefix" => -> (c) { c.dig("message_db", "prefix") },
"logging.rails_log_enabled" => -> (c) { c.dig("logging", "rails_log") },
"gelf.host" => -> (c) { c.dig("logging", "graylog", "host") },
"gelf.port" => -> (c) { c.dig("logging", "graylog", "port") },
"gelf.facility" => -> (c) { c.dig("logging", "graylog", "facility") },
"smtp_server.default_port" => -> (c) { c.dig("smtp_server", "port") },
"smtp_server.default_bind_address" => -> (c) { c.dig("smtp_server", "bind_address") },
"smtp_server.tls_enabled" => -> (c) { c.dig("smtp_server", "tls_enabled") },
"smtp_server.tls_certificate_path" => -> (c) { c.dig("smtp_server", "tls_certificate_path") },
"smtp_server.tls_private_key_path" => -> (c) { c.dig("smtp_server", "tls_private_key_path") },
"smtp_server.tls_ciphers" => -> (c) { c.dig("smtp_server", "tls_ciphers") },
"smtp_server.ssl_version" => -> (c) { c.dig("smtp_server", "ssl_version") },
"smtp_server.proxy_protocol" => -> (c) { c.dig("smtp_server", "proxy_protocol") },
"smtp_server.log_connections" => -> (c) { c.dig("smtp_server", "log_connect") },
"smtp_server.max_message_size" => -> (c) { c.dig("smtp_server", "max_message_size") },
"dns.mx_records" => -> (c) { c.dig("dns", "mx_records") },
"dns.spf_include" => -> (c) { c.dig("dns", "spf_include") },
"dns.return_path_domain" => -> (c) { c.dig("dns", "return_path") },
"dns.route_domain" => -> (c) { c.dig("dns", "route_domain") },
"dns.track_domain" => -> (c) { c.dig("dns", "track_domain") },
"dns.helo_hostname" => -> (c) { c.dig("dns", "helo_hostname") },
"dns.dkim_identifier" => -> (c) { c.dig("dns", "dkim_identifier") },
"dns.domain_verify_prefix" => -> (c) { c.dig("dns", "domain_verify_prefix") },
"dns.custom_return_path_prefix" => -> (c) { c.dig("dns", "custom_return_path_prefix") },
"smtp.host" => -> (c) { c.dig("smtp", "host") },
"smtp.port" => -> (c) { c.dig("smtp", "port") },
"smtp.username" => -> (c) { c.dig("smtp", "username") },
"smtp.password" => -> (c) { c.dig("smtp", "password") },
"smtp.from_name" => -> (c) { c.dig("smtp", "from_name") },
"smtp.from_address" => -> (c) { c.dig("smtp", "from_address") },
"rails.environment" => -> (c) { c.dig("rails", "environment") },
"rails.secret_key" => -> (c) { c.dig("rails", "secret_key") },
"rspamd.enabled" => -> (c) { c.dig("rspamd", "enabled") },
"rspamd.host" => -> (c) { c.dig("rspamd", "host") },
"rspamd.port" => -> (c) { c.dig("rspamd", "port") },
"rspamd.ssl" => -> (c) { c.dig("rspamd", "ssl") },
"rspamd.password" => -> (c) { c.dig("rspamd", "password") },
"rspamd.flags" => -> (c) { c.dig("rspamd", "flags") },
"spamd.enabled" => -> (c) { c.dig("spamd", "enabled") },
"spamd.host" => -> (c) { c.dig("spamd", "host") },
"spamd.port" => -> (c) { c.dig("spamd", "port") },
"clamav.enabled" => -> (c) { c.dig("clamav", "enabled") },
"clamav.host" => -> (c) { c.dig("clamav", "host") },
"clamav.port" => -> (c) { c.dig("clamav", "port") },
"smtp_client.open_timeout" => -> (c) { c.dig("smtp_client", "open_timeout") },
"smtp_client.read_timeout" => -> (c) { c.dig("smtp_client", "read_timeout") }
}.freeze
def initialize(config)
super()
@config = config
end
def get(path, attribute: nil)
path_string = path.join(".")
raise Konfig::ValueNotPresentError unless MAPPING.key?(path_string)
legacy_value = MAPPING[path_string].call(@config)
raise Konfig::ValueNotPresentError if legacy_value.nil?
legacy_value
end
end
end
lib/postal/message_db.rb
-6
View File
@@ -1,6 +0,0 @@
# frozen_string_literal: true
module Postal
module MessageDB
end
end
lib/postal/message_db/connection_pool.rb
+5 -5
View File
@@ -63,11 +63,11 @@ module Postal
def establish_connection
Mysql2::Client.new(
host: Postal.config.message_db.host,
username: Postal.config.message_db.username,
password: Postal.config.message_db.password,
port: Postal.config.message_db.port,
encoding: Postal.config.message_db.encoding || "utf8mb4"
host: Postal::Config.message_db.host,
username: Postal::Config.message_db.username,
password: Postal::Config.message_db.password,
port: Postal::Config.message_db.port,
encoding: Postal::Config.message_db.encoding
)
end
lib/postal/message_db/database.rb
+16 -19
View File
@@ -12,9 +12,10 @@ module Postal
end
def initialize(organization_id, server_id)
def initialize(organization_id, server_id, database_name: nil)
@organization_id = organization_id
@server_id = server_id
@database_name = database_name
end
attr_reader :organization_id
@@ -58,8 +59,8 @@ module Postal
end
#
#  Create a new message with the given attributes. This won't be saved to the database
#  until it has been 'save'd.
# Create a new message with the given attributes. This won't be saved to the database
# until it has been 'save'd.
#
def new_message(attributes = {})
Message.new(self, attributes)
@@ -73,35 +74,35 @@ module Postal
end
#
#  Return the live stats instance
# Return the live stats instance
#
def live_stats
@live_stats ||= LiveStats.new(self)
end
#
#  Return the statistics instance
# Return the statistics instance
#
def statistics
@statistics ||= Statistics.new(self)
end
#
#  Return the provisioner instance
# Return the provisioner instance
#
def provisioner
@provisioner ||= Provisioner.new(self)
end
#
#  Return the provisioner instance
# Return the provisioner instance
#
def suppression_list
@suppression_list ||= SuppressionList.new(self)
end
#
#  Return the provisioner instance
# Return the provisioner instance
#
def webhooks
@webhooks ||= Webhooks.new(self)
@@ -182,7 +183,7 @@ module Postal
end
#
#  A paginated version of select
# A paginated version of select
#
def select_with_pagination(table, page, options = {})
page = page.to_i
@@ -252,7 +253,7 @@ module Postal
#
# Deletes a in the database. Accepts a table name, and some options which
#  are shown below:
# are shown below:
#
# :where => The condition to apply to the query
#
@@ -268,10 +269,10 @@ module Postal
end
#
#  Return the correct database name
# Return the correct database name
#
def database_name
@database_name ||= "#{Postal.config.message_db.prefix}-server-#{@server_id}"
@database_name ||= "#{Postal::Config.message_db.database_name_prefix}-server-#{@server_id}"
end
#
@@ -325,12 +326,12 @@ module Postal
result = connection.query(query, cast_booleans: true)
time = Time.now.to_f - start_time
logger.debug " \e[4;34mMessageDB Query (#{time.round(2)}s) \e[0m \e[33m#{query}\e[0m"
if time > 0.5 && query =~ /\A(SELECT|UPDATE|DELETE) /
if time > 0.05 && query =~ /\A(SELECT|UPDATE|DELETE) /
id = Nifty::Utils::RandomString.generate(length: 6).upcase
explain_result = ResultForExplainPrinter.new(connection.query("EXPLAIN #{query}"))
slow_query_logger.info "[#{id}] EXPLAIN #{query}"
logger.info " [#{id}] EXPLAIN #{query}"
ActiveRecord::ConnectionAdapters::MySQL::ExplainPrettyPrinter.new.pp(explain_result, time).split("\n").each do |line|
slow_query_logger.info "[#{id}] " + line
logger.info " [#{id}] " + line
end
end
result
@@ -340,10 +341,6 @@ module Postal
defined?(Rails) ? Rails.logger : Logger.new($stdout)
end
def slow_query_logger
Postal.logger_for(:slow_message_db_queries)
end
def with_mysql(&block)
self.class.connection_pool.use(&block)
end
lib/postal/message_db/message.rb
+42 -23
View File
@@ -39,6 +39,10 @@ module Postal
@attributes = attributes
end
def reload
self.class.find_one(@database, @attributes["id"])
end
#
# Return the server for this message
#
@@ -99,7 +103,7 @@ module Postal
end
#
#  Return the time that the last delivery was attempted
# Return the time that the last delivery was attempted
#
def last_delivery_attempt
@last_delivery_attempt ||= @attributes["last_delivery_attempt"] ? Time.zone.at(@attributes["last_delivery_attempt"]) : nil
@@ -124,7 +128,7 @@ module Postal
#
def create_delivery(status, options = {})
delivery = Delivery.create(self, options.merge(status: status))
hold_expiry = status == "Held" ? Postal.config.general.maximum_hold_expiry_days.days.from_now.to_f : nil
hold_expiry = status == "Held" ? Postal::Config.postal.default_maximum_hold_expiry_days.days.from_now.to_f : nil
update(status: status, last_delivery_attempt: delivery.timestamp.to_f, held: status == "Held", hold_expiry: hold_expiry)
delivery
end
@@ -168,14 +172,14 @@ module Postal
end
#
#  Return all activity entries
# Return all activity entries
#
def activity_entries
@activity_entries ||= (deliveries + clicks + loads).sort_by(&:timestamp)
end
#
#  Provide access to set and get acceptable attributes
# Provide access to set and get acceptable attributes
#
def method_missing(name, value = nil, &block)
if @attributes.key?(name.to_s)
@@ -198,11 +202,11 @@ module Postal
end
#
#  Save this message
# Save this message
#
def save
def save(queue_on_create: true)
save_raw_message
persisted? ? _update : _create
persisted? ? _update : _create(queue: queue_on_create)
self
end
@@ -219,7 +223,7 @@ module Postal
end
#
#  Delete the message from the database
# Delete the message from the database
#
def delete
return unless persisted?
@@ -228,7 +232,7 @@ module Postal
end
#
#  Return the headers
# Return the headers
#
def raw_headers
if raw_table
@@ -239,7 +243,7 @@ module Postal
end
#
#  Return the full raw message body for this message.
# Return the full raw message body for this message.
#
def raw_body
if raw_table
@@ -299,7 +303,7 @@ module Postal
end
#
#  Return the HTML body for this message
# Return the HTML body for this message
#
def html_body
mail&.html_body
@@ -346,8 +350,14 @@ module Postal
#
# Create a new item in the message queue for this message
#
def add_to_message_queue(options = {})
QueuedMessage.create!(message: self, server_id: @database.server_id, batch_key: batch_key, domain: recipient_domain, route_id: route_id, manual: options[:manual]).id
def add_to_message_queue(**options)
QueuedMessage.create!({
message: self,
server_id: @database.server_id,
batch_key: batch_key,
domain: recipient_domain,
route_id: route_id
}.merge(options))
end
#
@@ -403,7 +413,7 @@ module Postal
def add_outgoing_headers
headers = []
if domain
dkim = Postal::DKIMHeader.new(domain, raw_message)
dkim = DKIMHeader.new(domain, raw_message)
headers << dkim.dkim_header
end
headers << "X-Postal-MsgID: #{token}"
@@ -445,7 +455,11 @@ module Postal
#
def bounce!(bounce_message)
create_delivery("Bounced", details: "We've received a bounce message for this e-mail. See <msg:#{bounce_message.id}> for details.")
SendWebhookJob.queue(:main, server_id: database.server_id, event: "MessageBounced", payload: { _original_message: id, _bounce: bounce_message.id })
WebhookRequest.trigger(server, "MessageBounced", {
original_message: webhook_hash,
bounce: bounce_message.webhook_hash
})
end
#
@@ -461,11 +475,16 @@ module Postal
def create_load(request)
update("loaded" => Time.now.to_f) if loaded.nil?
database.insert(:loads, { message_id: id, ip_address: request.ip, user_agent: request.user_agent, timestamp: Time.now.to_f })
SendWebhookJob.queue(:main, server_id: database.server_id, event: "MessageLoaded", payload: { _message: id, ip_address: request.ip, user_agent: request.user_agent })
WebhookRequest.trigger(server, "MessageLoaded", {
message: webhook_hash,
ip_address: request.ip,
user_agent: request.user_agent
})
end
#
#  Create a new link
# Create a new link
#
def create_link(url)
hash = Digest::SHA1.hexdigest(url.to_s)
@@ -475,7 +494,7 @@ module Postal
end
#
#  Return a message object that this message is a reply to
# Return a message object that this message is a reply to
#
def original_messages
return nil unless bounce
@@ -492,7 +511,7 @@ module Postal
# Was thsi message sent to a return path?
#
def rcpt_to_return_path?
!!(rcpt_to =~ /@#{Regexp.escape(Postal.config.dns.custom_return_path_prefix)}\./)
!!(rcpt_to =~ /@#{Regexp.escape(Postal::Config.dns.custom_return_path_prefix)}\./)
end
#
@@ -512,7 +531,7 @@ module Postal
end
#
#  Return all spam checks for this message
# Return all spam checks for this message
#
def spam_checks
@spam_checks ||= database.select(:spam_checks, where: { message_id: id })
@@ -533,7 +552,7 @@ module Postal
def parse_content
parse_result = Postal::MessageParser.new(self)
if parse_result.actioned?
#  Somethign was changed, update the raw message
# Somethign was changed, update the raw message
@database.update(raw_table, { data: parse_result.new_body }, where: { id: raw_body_id })
@database.update(raw_table, { data: parse_result.new_headers }, where: { id: raw_headers_id })
@raw = parse_result.new_body
@@ -563,7 +582,7 @@ module Postal
@database.update("messages", @attributes.except(:id), where: { id: @attributes["id"] })
end
def _create
def _create(queue: true)
self.timestamp = Time.now.to_f if timestamp.blank?
self.status = "Pending" if status.blank?
self.token = Nifty::Utils::RandomString.generate(length: 12) if token.blank?
@@ -572,7 +591,7 @@ module Postal
@database.statistics.increment_all(timestamp, scope)
Statistic.global.increment!(:total_messages)
Statistic.global.increment!("total_#{scope}".to_sym)
add_to_message_queue
add_to_message_queue if queue
end
def mail
lib/postal/message_db/provisioner.rb
+1 -1
View File
@@ -127,7 +127,7 @@ module Postal
end
#
#  Remove messages from the messages table that are too old to retain
# Remove messages from the messages table that are too old to retain
#
def remove_messages(max_age = 60)
time = (Time.now.utc.to_date - max_age.days).to_time.end_of_day
lib/postal/message_db/suppression_list.rb
+1 -1
View File
@@ -9,7 +9,7 @@ module Postal
end
def add(type, address, options = {})
keep_until = (options[:days] || Postal.config.general.suppression_list_removal_delay).days.from_now.to_f
keep_until = (options[:days] || Postal::Config.postal.default_suppression_list_automatic_removal_days).days.from_now.to_f
if existing = @database.select("suppressions", where: { type: type, address: address }, limit: 1).first
reason = options[:reason] || existing["reason"]
@database.update("suppressions", { reason: reason, keep_until: keep_until }, where: { id: existing["id"] })
lib/postal/message_db/webhooks.rb
+1 -1
View File
@@ -12,7 +12,7 @@ module Postal
@database.insert(:webhook_requests, attributes)
end
def list(page)
def list(page = 1)
result = @database.select_with_pagination(:webhook_requests, page, order: :timestamp, direction: "desc")
result[:records] = result[:records].map { |i| Request.new(i) }
result
lib/postal/message_inspector.rb
+7 -7
View File
@@ -15,7 +15,7 @@ module Postal
private
def logger
Postal.logger_for(:message_inspection)
Postal.logger
end
class << self
@@ -24,14 +24,14 @@ module Postal
# installation.
def inspectors
[].tap do |inspectors|
if Postal.config.rspamd&.enabled
inspectors << MessageInspectors::Rspamd.new(Postal.config.rspamd)
elsif Postal.config.spamd&.enabled
inspectors << MessageInspectors::SpamAssassin.new(Postal.config.spamd)
if Postal::Config.rspamd.enabled?
inspectors << MessageInspectors::Rspamd.new(Postal::Config.rspamd)
elsif Postal::Config.spamd.enabled?
inspectors << MessageInspectors::SpamAssassin.new(Postal::Config.spamd)
end
if Postal.config.clamav&.enabled
inspectors << MessageInspectors::Clamav.new(Postal.config.clamav)
if Postal::Config.clamav.enabled?
inspectors << MessageInspectors::Clamav.new(Postal::Config.clamav)
end
end
end
lib/postal/message_inspectors.rb
-6
View File
@@ -1,6 +0,0 @@
# frozen_string_literal: true
module Postal
module MessageInspectors
end
end
lib/postal/message_requeuer.rb
-34
View File
@@ -1,34 +0,0 @@
# frozen_string_literal: true
module Postal
class MessageRequeuer
def run
Signal.trap("INT") { @running ? @exit = true : Process.exit(0) }
Signal.trap("TERM") { @running ? @exit = true : Process.exit(0) }
log "Running message requeuer..."
loop do
@running = true
QueuedMessage.requeue_all
@running = false
check_exit
sleep 5
end
end
private
def log(text)
Postal.logger_for(:message_requeuer).info text
end
def check_exit
return unless @exit
log "Exiting"
Process.exit(0)
end
end
end
lib/postal/mx_lookup.rb
-36
View File
@@ -1,36 +0,0 @@
# frozen_string_literal: true
module Postal
class MXLookup
class << self
def lookup(domain)
records = resolve(domain)
records = sort(records)
records.map { |m| m[1] }
end
private
def sort(records)
records.sort do |a, b|
if a[0] == b[0]
[-1, 1].sample
else
a[0] <=> b[0]
end
end
end
def resolve(domain)
Resolv::DNS.open do |dns|
dns.timeouts = [10, 5]
dns.getresources(domain, Resolv::DNS::Resource::IN::MX).map { |m| [m.preference.to_i, m.exchange.to_s] }
end
end
end
end
end
lib/postal/query_string.rb
-38
View File
@@ -1,38 +0,0 @@
# frozen_string_literal: true
module Postal
class QueryString
def initialize(string)
@string = string.strip + " "
end
def [](value)
hash[value.to_s]
end
delegate :empty?, to: :hash
def hash
@hash ||= @string.scan(/([a-z]+):\s*(?:(\d{2,4}-\d{2}-\d{2}\s\d{2}:\d{2})|"(.*?)"|(.*?))(\s|\z)/).each_with_object({}) do |(key, date, string_with_spaces, value), hash|
if date
actual_value = date
elsif string_with_spaces
actual_value = string_with_spaces
elsif value == "[blank]"
actual_value = nil
else
actual_value = value
end
if hash.keys.include?(key.to_s)
hash[key.to_s] = [hash[key.to_s]].flatten
hash[key.to_s] << actual_value
else
hash[key.to_s] = actual_value
end
end
end
end
end
lib/postal/rabbit_mq.rb
-38
View File
@@ -1,38 +0,0 @@
# frozen_string_literal: true
require "postal/config"
require "bunny"
module Postal
module RabbitMQ
def self.create_connection
bunny_host = ["localhost"]
if Postal.config.rabbitmq&.host.is_a?(Array)
bunny_host = Postal.config.rabbitmq&.host
elsif Postal.config.rabbitmq&.host.is_a?(String)
bunny_host = [Postal.config.rabbitmq&.host]
end
conn = Bunny.new(
hosts: bunny_host,
port: Postal.config.rabbitmq&.port || 5672,
tls: Postal.config.rabbitmq&.tls || false,
verify_peer: Postal.config.rabbitmq&.verify_peer || true,
tls_ca_certificates: Postal.config.rabbitmq&.tls_ca_certificates || ["/etc/ssl/certs/ca-certificates.crt"],
username: Postal.config.rabbitmq&.username || "guest",
password: Postal.config.rabbitmq&.password || "guest",
vhost: Postal.config.rabbitmq&.vhost || nil
)
conn.start
conn
end
def self.create_channel
conn = create_connection
conn.create_channel(nil, Postal.config.workers.threads)
end
end
end
lib/postal/received_header.rb
-45
View File
@@ -1,45 +0,0 @@
# frozen_string_literal: true
module Postal
class ReceivedHeader
OUR_HOSTNAMES = {
smtp: Postal.config.dns.smtp_server_hostname,
http: Postal.config.web.host
}.freeze
class << self
def generate(server, helo, ip_address, method)
our_hostname = OUR_HOSTNAMES[method]
if our_hostname.nil?
raise Error, "`method` is invalid (must be one of #{OUR_HOSTNAMES.join(', ')})"
end
header = "by #{our_hostname} with #{method.to_s.upcase}; #{Time.now.utc.rfc2822}"
if server.nil? || server.privacy_mode == false
hostname = resolve_hostname(ip_address)
header = "from #{helo} (#{hostname} [#{ip_address}]) #{header}"
end
header
end
private
def resolve_hostname(ip_address)
Resolv::DNS.open do |dns|
dns.timeouts = [10, 5]
begin
dns.getname(ip_address)
rescue StandardError
ip_address
end
end
end
end
end
end
lib/postal/reply_separator.rb
-36
View File
@@ -1,36 +0,0 @@
# frozen_string_literal: true
module Postal
class ReplySeparator
RULES = [
/^-{2,10} $.*/m,
/^>*\s*----- ?Original Message ?-----.*/m,
/^>*\s*From:[^\r\n]*[\r\n]+Sent:.*/m,
/^>*\s*From:[^\r\n]*[\r\n]+Date:.*/m,
/^>*\s*-----Urspr.ngliche Nachricht----- .*/m,
/^>*\s*Le[^\r\n]{10,200}a .crit ?:\s*$.*/,
/^>*\s*__________________.*/m,
/^>*\s*On.{10,200}wrote:\s*$.*/m,
/^>*\s*Sent from my.*/m,
/^>*\s*=== Please reply above this line ===.*/m,
/(^>.*\n?){10,}/
].freeze
def self.separate(text)
return "" unless text.is_a?(String)
text = text.gsub("\r", "")
stripped = String.new
RULES.each do |rule|
text.gsub!(rule) do
stripped = ::Regexp.last_match(0).to_s + "\n" + stripped
""
end
end
stripped = stripped.strip
[text.strip, stripped.presence]
end
end
end
lib/postal/rspec_helpers.rb
-24
View File
@@ -1,24 +0,0 @@
# frozen_string_literal: true
module Postal
module RspecHelpers
def with_global_server(&block)
server = Server.find(GLOBAL_SERVER.id)
block.call(server)
ensure
server.message_db.provisioner.clean
end
def create_plain_text_message(server, text, to = "test@example.com", override_attributes = {})
domain = create(:domain, owner: server)
attributes = { from: "test@#{domain.name}", subject: "Test Plain Text Message" }.merge(override_attributes)
attributes[:to] = to
attributes[:plain_body] = text
message = OutgoingMessagePrototype.new(server, "127.0.0.1", "testsuite", attributes)
result = message.create_message(to)
server.message_db.message(result[:id])
end
end
end
lib/postal/send_result.rb
-17
View File
@@ -1,17 +0,0 @@
# frozen_string_literal: true
module Postal
class SendResult
attr_accessor :type
attr_accessor :details
attr_accessor :retry
attr_accessor :output
attr_accessor :secure
attr_accessor :connect_error
attr_accessor :log_id
attr_accessor :time
attr_accessor :suppress_bounce
end
end
lib/postal/sender.rb
-16
View File
@@ -1,16 +0,0 @@
# frozen_string_literal: true
module Postal
class Sender
def start
end
def send_message(message)
end
def finish
end
end
end
lib/postal/smtp_sender.rb
-298
View File
@@ -1,298 +0,0 @@
# frozen_string_literal: true
require "resolv"
module Postal
class SMTPSender < Sender
def initialize(domain, source_ip_address, options = {})
super()
@domain = domain
@source_ip_address = source_ip_address
@options = options
@smtp_client = nil
@connection_errors = []
@hostnames = []
@log_id = Nifty::Utils::RandomString.generate(length: 8).upcase
end
def start
servers.each do |server|
if server.is_a?(SMTPEndpoint)
hostname = server.hostname
port = server.port || 25
ssl_mode = server.ssl_mode
elsif server.is_a?(Hash)
hostname = server[:hostname]
port = server[:port] || 25
ssl_mode = server[:ssl_mode] || "Auto"
else
hostname = server
port = 25
ssl_mode = "Auto"
end
@hostnames << hostname
[:aaaa, :a].each do |ip_type|
if @source_ip_address && @source_ip_address.ipv6.blank? && ip_type == :aaaa
# Don't try to use IPv6 if the IP address we're sending from doesn't support it.
next
end
begin
@remote_ip = lookup_ip_address(ip_type, hostname)
if @remote_ip.nil?
if ip_type == :a
# As we can't resolve the last IP, we'll put this
@connection_errors << "Could not resolve #{hostname}"
end
next
end
smtp_client = Net::SMTP.new(@remote_ip, port)
smtp_client.open_timeout = Postal.config.smtp_client.open_timeout
smtp_client.read_timeout = Postal.config.smtp_client.read_timeout
smtp_client.tls_hostname = hostname
if @source_ip_address
# Set the source IP as appropriate
smtp_client.source_address = ip_type == :aaaa ? @source_ip_address.ipv6 : @source_ip_address.ipv4
end
case ssl_mode
when "Auto"
smtp_client.enable_starttls_auto(self.class.ssl_context_without_verify)
when "STARTTLS"
smtp_client.enable_starttls(self.class.ssl_context_with_verify)
when "TLS"
smtp_client.enable_tls(self.class.ssl_context_with_verify)
else
smtp_client.disable_starttls
smtp_client.disable_tls
end
smtp_client.start(@source_ip_address ? @source_ip_address.hostname : self.class.default_helo_hostname)
log "Connected to #{@remote_ip}:#{port} (#{hostname})"
rescue StandardError => e
if e.is_a?(OpenSSL::SSL::SSLError) && ssl_mode == "Auto"
log "SSL error (#{e.message}), retrying without SSL"
ssl_mode = nil
retry
end
log "Cannot connect to #{@remote_ip}:#{port} (#{hostname}) (#{e.class}: #{e.message})"
@connection_errors << e.message unless @connection_errors.include?(e.message)
begin
smtp_client.finish
rescue StandardError
nil
end
smtp_client = nil
end
if smtp_client
@smtp_client = smtp_client
return true
end
end
end
@connection_errors
end
def reconnect
log "Reconnecting"
begin
@smtp_client&.finish
rescue StandardError
nil
end
start
end
def safe_rset
# Something went wrong sending the last email. Reset the connection if possible, else disconnect.
@smtp_client.rset
rescue StandardError
# Don't reconnect, this would be rather rude if we don't have any more emails to send.
begin
@smtp_client.finish
rescue StandardError
nil
end
end
def send_message(message, force_rcpt_to = nil)
start_time = Time.now
result = SendResult.new
result.log_id = @log_id
if @smtp_client && !@smtp_client.started?
# For some reason we had an SMTP connection but it's no longer connected.
# Make a new one.
start
end
if @smtp_client
result.secure = @smtp_client.secure_socket?
end
begin
if message.bounce
mail_from = ""
elsif message.domain.return_path_status == "OK"
mail_from = "#{message.server.token}@#{message.domain.return_path_domain}"
else
mail_from = "#{message.server.token}@#{Postal.config.dns.return_path}"
end
if Postal.config.general.use_resent_sender_header
raw_message = "Resent-Sender: #{mail_from}\r\n" + message.raw_message
else
raw_message = message.raw_message
end
tries = 0
begin
if @smtp_client.nil?
log "-> No SMTP server available for #{@domain}"
log "-> Hostnames: #{@hostnames.inspect}"
log "-> Errors: #{@connection_errors.inspect}"
result.type = "SoftFail"
result.retry = true
result.details = "No SMTP servers were available for #{@domain}. Tried #{@hostnames.to_sentence}"
result.output = @connection_errors.join(", ")
result.connect_error = true
return result
else
@smtp_client.rset_errors
rcpt_to = force_rcpt_to || @options[:force_rcpt_to] || message.rcpt_to
log "Sending message #{message.server.id}::#{message.id} to #{rcpt_to}"
smtp_result = @smtp_client.send_message(raw_message, mail_from, [rcpt_to])
end
rescue Errno::ECONNRESET, Errno::EPIPE, OpenSSL::SSL::SSLError
raise unless (tries += 1) < 2
reconnect
retry
end
result.type = "Sent"
result.details = "Message for #{rcpt_to} accepted by #{destination_host_description}"
if @smtp_client.source_address
result.details += " (from #{@smtp_client.source_address})"
end
result.output = smtp_result.string
log "Message sent ##{message.id} to #{destination_host_description} for #{rcpt_to}"
rescue Net::SMTPServerBusy, Net::SMTPAuthenticationError, Net::SMTPSyntaxError, Net::SMTPUnknownError, Net::ReadTimeout => e
log "#{e.class}: #{e.message}"
result.type = "SoftFail"
result.retry = true
result.details = "Temporary SMTP delivery error when sending to #{destination_host_description}"
result.output = e.message
if e.to_s =~ /(\d+) seconds/
result.retry = ::Regexp.last_match(1).to_i + 10
elsif e.to_s =~ /(\d+) minutes/
result.retry = (::Regexp.last_match(1).to_i * 60) + 10
end
safe_rset
rescue Net::SMTPFatalError => e
log "#{e.class}: #{e.message}"
result.type = "HardFail"
result.details = "Permanent SMTP delivery error when sending to #{destination_host_description}"
result.output = e.message
safe_rset
rescue StandardError => e
log "#{e.class}: #{e.message}"
if defined?(Sentry)
Sentry.capture_exception(e, extra: { log_id: @log_id, server_id: message.server.id, message_id: message.id })
end
result.type = "SoftFail"
result.retry = true
result.details = "An error occurred while sending the message to #{destination_host_description}"
result.output = e.message
safe_rset
end
result.time = (Time.now - start_time).to_f.round(2)
result
end
def finish
log "Finishing up"
@smtp_client&.finish
end
private
def servers
@options[:servers] || self.class.relay_hosts || @servers ||= begin
mx_servers = MXLookup.lookup(@domain)
if mx_servers.empty?
mx_servers = [@domain] # This will be resolved to an A or AAAA record later
end
mx_servers
end
end
def log(text)
Postal.logger_for(:smtp_sender).info "[#{@log_id}] #{text}"
end
def destination_host_description
"#{@hostnames.last} (#{@remote_ip})"
end
def lookup_ip_address(type, hostname)
records = []
Resolv::DNS.open do |dns|
dns.timeouts = [10, 5]
case type
when :a
records = dns.getresources(hostname, Resolv::DNS::Resource::IN::A)
when :aaaa
records = dns.getresources(hostname, Resolv::DNS::Resource::IN::AAAA)
end
end
records.first&.address&.to_s&.downcase
end
class << self
def ssl_context_with_verify
@ssl_context_with_verify ||= begin
c = OpenSSL::SSL::SSLContext.new
c.verify_mode = OpenSSL::SSL::VERIFY_PEER
c.cert_store = OpenSSL::X509::Store.new
c.cert_store.set_default_paths
c
end
end
def ssl_context_without_verify
@ssl_context_without_verify ||= begin
c = OpenSSL::SSL::SSLContext.new
c.verify_mode = OpenSSL::SSL::VERIFY_NONE
c
end
end
def default_helo_hostname
Postal.config.dns.helo_hostname || Postal.config.dns.smtp_server_hostname || "localhost"
end
def relay_hosts
hosts = Postal.config.smtp_relays.map do |relay|
next unless relay.hostname.present?
{
hostname: relay.hostname,
port: relay.port,
ssl_mode: relay.ssl_mode
}
end.compact
hosts.empty? ? nil : hosts
end
end
end
end
lib/postal/smtp_server.rb
-6
View File
@@ -1,6 +0,0 @@
# frozen_string_literal: true
module Postal
module SMTPServer
end
end
lib/postal/smtp_server/client.rb
-508
View File
@@ -1,508 +0,0 @@
# frozen_string_literal: true
require "resolv"
require "nifty/utils/random_string"
module Postal
module SMTPServer
class Client
CRAM_MD5_DIGEST = OpenSSL::Digest.new("md5")
LOG_REDACTION_STRING = "[redacted]"
attr_reader :logging_enabled
attr_reader :credential
attr_reader :ip_address
attr_reader :recipients
attr_reader :headers
attr_reader :state
attr_reader :helo_name
def initialize(ip_address)
@logging_enabled = true
@ip_address = ip_address
if @ip_address
check_ip_address
@state = :welcome
else
@state = :preauth
end
transaction_reset
end
def check_ip_address
return unless @ip_address && Postal.config.smtp_server.log_exclude_ips && @ip_address =~ Regexp.new(Postal.config.smtp_server.log_exclude_ips)
@logging_enabled = false
end
def transaction_reset
@recipients = []
@mail_from = nil
@data = nil
@headers = nil
end
def id
@id ||= Nifty::Utils::RandomString.generate(length: 6).upcase
end
def handle(data)
if @state == :preauth
return proxy(data)
end
log "\e[32m<= #{sanitize_input_for_log(data.strip)}\e[0m"
if @proc
@proc.call(data)
else
handle_command(data)
end
end
def finished?
@finished || false
end
def start_tls?
@start_tls || false
end
attr_writer :start_tls
def handle_command(data)
case data
when /^QUIT/i then quit
when /^STARTTLS/i then starttls
when /^EHLO/i then ehlo(data)
when /^HELO/i then helo(data)
when /^RSET/i then rset
when /^NOOP/i then noop
when /^AUTH PLAIN/i then auth_plain(data)
when /^AUTH LOGIN/i then auth_login(data)
when /^AUTH CRAM-MD5/i then auth_cram_md5(data)
when /^MAIL FROM/i then mail_from(data)
when /^RCPT TO/i then rcpt_to(data)
when /^DATA/i then data(data)
else
"502 Invalid/unsupported command"
end
end
def log(text)
return false unless @logging_enabled
Postal.logger_for(:smtp_server).debug "[#{id}] #{text}"
end
private
def proxy(data)
if m = data.match(/\APROXY (.+) (.+) (.+) (.+) (.+)\z/)
@ip_address = m[2]
check_ip_address
@state = :welcome
log "\e[35m Client identified as #{@ip_address}\e[0m"
"220 #{Postal.config.dns.smtp_server_hostname} ESMTP Postal/#{id}"
else
@finished = true
"502 Proxy Error"
end
end
def quit
@finished = true
"221 Closing Connection"
end
def starttls
if Postal.config.smtp_server.tls_enabled?
@start_tls = true
@tls = true
"220 Ready to start TLS"
else
"502 TLS not available"
end
end
def ehlo(data)
@helo_name = data.strip.split(" ", 2)[1]
transaction_reset
@state = :welcomed
[
"250-My capabilities are",
Postal.config.smtp_server.tls_enabled? && !@tls ? "250-STARTTLS" : nil,
"250 AUTH CRAM-MD5 PLAIN LOGIN"
].compact
end
def helo(data)
@helo_name = data.strip.split(" ", 2)[1]
transaction_reset
@state = :welcomed
"250 #{Postal.config.dns.smtp_server_hostname}"
end
def rset
transaction_reset
@state = :welcomed
"250 OK"
end
def noop
"250 OK"
end
def auth_plain(data)
handler = proc do |idata|
@proc = nil
idata = Base64.decode64(idata)
parts = idata.split("\0")
username = parts[-2]
password = parts[-1]
unless username && password
next "535 Authenticated failed - protocol error"
end
authenticate(password)
end
data = data.gsub(/AUTH PLAIN ?/i, "")
if data.strip == ""
@proc = handler
@password_expected_next = true
"334"
else
handler.call(data)
end
end
def auth_login(data)
password_handler = proc do |idata|
@proc = nil
password = Base64.decode64(idata)
authenticate(password)
end
username_handler = proc do
@proc = password_handler
@password_expected_next = true
"334 UGFzc3dvcmQ6" # "Password:"
end
data = data.gsub(/AUTH LOGIN ?/i, "")
if data.strip == ""
@proc = username_handler
"334 VXNlcm5hbWU6" # "Username:"
else
username_handler.call(nil)
end
end
def authenticate(password)
if @credential = Credential.where(type: "SMTP", key: password).first
@credential.use
"235 Granted for #{@credential.server.organization.permalink}/#{@credential.server.permalink}"
else
log "\e[33m WARN: AUTH failure for #{@ip_address}\e[0m"
"535 Invalid credential"
end
end
def auth_cram_md5(data)
challenge = Digest::SHA1.hexdigest(Time.now.to_i.to_s + rand(100_000).to_s)
challenge = "<#{challenge[0, 20]}@#{Postal.config.dns.smtp_server_hostname}>"
handler = proc do |idata|
@proc = nil
username, password = Base64.decode64(idata).split(" ", 2).map { |a| a.chomp }
org_permlink, server_permalink = username.split(/[\/_]/, 2)
server = ::Server.includes(:organization).where(organizations: { permalink: org_permlink }, permalink: server_permalink).first
if server.nil?
log "\e[33m WARN: AUTH failure for #{@ip_address}\e[0m"
next "535 Denied"
end
grant = nil
server.credentials.where(type: "SMTP").each do |credential|
correct_response = OpenSSL::HMAC.hexdigest(CRAM_MD5_DIGEST, credential.key, challenge)
next unless password == correct_response
@credential = credential
@credential.use
grant = "235 Granted for #{credential.server.organization.permalink}/#{credential.server.permalink}"
break
end
if grant.nil?
log "\e[33m WARN: AUTH failure for #{@ip_address}\e[0m"
next "535 Denied"
end
grant
end
@proc = handler
"334 " + Base64.encode64(challenge).gsub(/[\r\n]/, "")
end
def mail_from(data)
unless in_state(:welcomed, :mail_from_received)
return "503 EHLO/HELO first please"
end
@state = :mail_from_received
transaction_reset
if data =~ /AUTH=/
# Discard AUTH= parameter and anything that follows.
# We don't need this parameter as we don't trust any client to set it
mail_from_line = data.sub(/ *AUTH=.*/, "")
else
mail_from_line = data
end
@mail_from = mail_from_line.gsub(/MAIL FROM\s*:\s*/i, "").gsub(/.*</, "").gsub(/>.*/, "").strip
"250 OK"
end
def rcpt_to(data)
unless in_state(:mail_from_received, :rcpt_to_received)
return "503 EHLO/HELO and MAIL FROM first please"
end
rcpt_to = data.gsub(/RCPT TO\s*:\s*/i, "").gsub(/.*</, "").gsub(/>.*/, "").strip
if rcpt_to.blank?
return "501 RCPT TO should not be empty"
end
uname, domain = rcpt_to.split("@", 2)
if domain.blank?
return "501 Invalid RCPT TO"
end
uname, tag = uname.split("+", 2)
if domain == Postal.config.dns.return_path || domain =~ /\A#{Regexp.escape(Postal.config.dns.custom_return_path_prefix)}\./
# This is a return path
@state = :rcpt_to_received
if server = ::Server.where(token: uname).first
if server.suspended?
"535 Mail server has been suspended"
else
log "Added bounce on server #{server.id}"
@recipients << [:bounce, rcpt_to, server]
"250 OK"
end
else
"550 Invalid server token"
end
elsif domain == Postal.config.dns.route_domain
# This is an email direct to a route. This isn't actually supported yet.
@state = :rcpt_to_received
if route = Route.where(token: uname).first
if route.server.suspended?
"535 Mail server has been suspended"
elsif route.mode == "Reject"
"550 Route does not accept incoming messages"
else
log "Added route #{route.id} to recipients (tag: #{tag.inspect})"
actual_rcpt_to = "#{route.name}#{tag ? "+#{tag}" : ''}@#{route.domain.name}"
@recipients << [:route, actual_rcpt_to, route.server, { route: route }]
"250 OK"
end
else
"550 Invalid route token"
end
elsif @credential
# This is outgoing mail for an authenticated user
@state = :rcpt_to_received
if @credential.server.suspended?
"535 Mail server has been suspended"
else
log "Added external address '#{rcpt_to}'"
@recipients << [:credential, rcpt_to, @credential.server]
"250 OK"
end
elsif uname && domain && route = Route.find_by_name_and_domain(uname, domain)
# This is incoming mail for a route
@state = :rcpt_to_received
if route.server.suspended?
"535 Mail server has been suspended"
elsif route.mode == "Reject"
"550 Route does not accept incoming messages"
else
log "Added route #{route.id} to recipients (tag: #{tag.inspect})"
@recipients << [:route, rcpt_to, route.server, { route: route }]
"250 OK"
end
else
# User is trying to relay but is not authenticated. Try to authenticate by IP address
@credential = Credential.where(type: "SMTP-IP").all.sort_by { |c| c.ipaddr&.prefix || 0 }.reverse.find do |credential|
credential.ipaddr.include?(@ip_address) || (credential.ipaddr.ipv4? && credential.ipaddr.ipv4_mapped.include?(@ip_address))
end
if @credential
# Retry with credential
@credential.use
rcpt_to(data)
else
"530 Authentication required"
end
end
end
def data(_data)
unless in_state(:rcpt_to_received)
return "503 HELO/EHLO, MAIL FROM and RCPT TO before sending data"
end
@data = String.new.force_encoding("BINARY")
@headers = {}
@receiving_headers = true
received_header = Postal::ReceivedHeader.generate(@credential&.server, @helo_name, @ip_address, :smtp)
.force_encoding("BINARY")
@data << "Received: #{received_header}\r\n"
@headers["received"] = [received_header]
handler = proc do |idata|
if idata == "."
@logging_enabled = true
@proc = nil
finished
else
idata = idata.to_s.sub(/\A\.\./, ".")
if @credential&.server&.log_smtp_data?
# We want to log if enabled
else
log "Not logging further message data."
@logging_enabled = false
end
if @receiving_headers
if idata.blank?
@receiving_headers = false
elsif idata.to_s =~ /^\s/
# This is a continuation of a header
if @header_key && @headers[@header_key.downcase] && @headers[@header_key.downcase].last
@headers[@header_key.downcase].last << idata.to_s
end
else
@header_key, value = idata.split(/:\s*/, 2)
@headers[@header_key.downcase] ||= []
@headers[@header_key.downcase] << value
end
end
@data << idata
@data << "\r\n"
nil
end
end
@proc = handler
"354 Go ahead"
end
def finished
if @data.bytesize > Postal.config.smtp_server.max_message_size.megabytes.to_i
transaction_reset
@state = :welcomed
return format("552 Message too large (maximum size %dMB)", Postal.config.smtp_server.max_message_size)
end
if @headers["received"].grep(/by #{Postal.config.dns.smtp_server_hostname}/).count > 4
transaction_reset
@state = :welcomed
return "550 Loop detected"
end
authenticated_domain = nil
if @credential
authenticated_domain = @credential.server.find_authenticated_domain_from_headers(@headers)
if authenticated_domain.nil?
transaction_reset
@state = :welcomed
return "530 From/Sender name is not valid"
end
end
@recipients.each do |recipient|
type, rcpt_to, server, options = recipient
case type
when :credential
# Outgoing messages are just inserted
message = server.message_db.new_message
message.rcpt_to = rcpt_to
message.mail_from = @mail_from
message.raw_message = @data
message.received_with_ssl = @tls
message.scope = "outgoing"
message.domain_id = authenticated_domain&.id
message.credential_id = @credential.id
message.save
when :bounce
if rp_route = server.routes.where(name: "__returnpath__").first
# If there's a return path route, we can use this to create the message
rp_route.create_messages do |msg|
msg.rcpt_to = rcpt_to
msg.mail_from = @mail_from
msg.raw_message = @data
msg.received_with_ssl = @tls
msg.bounce = 1
end
else
# There's no return path route, we just need to insert the mesage
# without going through the route.
message = server.message_db.new_message
message.rcpt_to = rcpt_to
message.mail_from = @mail_from
message.raw_message = @data
message.received_with_ssl = @tls
message.scope = "incoming"
message.bounce = 1
message.save
end
when :route
options[:route].create_messages do |message|
message.rcpt_to = rcpt_to
message.mail_from = @mail_from
message.raw_message = @data
message.received_with_ssl = @tls
end
end
end
transaction_reset
@state = :welcomed
"250 OK"
end
def in_state(*states)
states.include?(@state)
end
def sanitize_input_for_log(data)
if @password_expected_next
@password_expected_next = false
if data =~ /\A[a-z0-9]{3,}=*\z/i
return LOG_REDACTION_STRING
end
end
data = data.dup
data.gsub!(/(.*AUTH \w+) (.*)\z/i) { "#{::Regexp.last_match(1)} #{LOG_REDACTION_STRING}" }
data
end
end
end
end
lib/postal/smtp_server/server.rb
-272
View File
@@ -1,272 +0,0 @@
# frozen_string_literal: true
require "ipaddr"
require "nio"
module Postal
module SMTPServer
class Server
def initialize(options = {})
@options = options
@options[:debug] ||= false
prepare_environment
end
def run
listen
run_event_loop
end
private
def prepare_environment
$\ = "\r\n"
BasicSocket.do_not_reverse_lookup = true
trap("TERM") do
$stdout.puts "Received TERM signal, shutting down."
unlisten
end
trap("INT") do
$stdout.puts "Received INT signal, shutting down."
unlisten
end
end
def ssl_context
@ssl_context ||= begin
ssl_context = OpenSSL::SSL::SSLContext.new
ssl_context.cert = Postal.smtp_certificates[0]
ssl_context.extra_chain_cert = Postal.smtp_certificates[1..]
ssl_context.key = Postal.smtp_private_key
ssl_context.ssl_version = Postal.config.smtp_server.ssl_version if Postal.config.smtp_server.ssl_version
ssl_context.ciphers = Postal.config.smtp_server.tls_ciphers if Postal.config.smtp_server.tls_ciphers
ssl_context
end
end
def listen
@server = TCPServer.open(Postal.config.smtp_server.bind_address, Postal.config.smtp_server.port)
@server.autoclose = false
@server.close_on_exec = false
if defined?(Socket::SOL_SOCKET) && defined?(Socket::SO_KEEPALIVE)
@server.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true)
end
if defined?(Socket::SOL_TCP) && defined?(Socket::TCP_KEEPIDLE) && defined?(Socket::TCP_KEEPINTVL) && defined?(Socket::TCP_KEEPCNT)
@server.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPIDLE, 50)
@server.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPINTVL, 10)
@server.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPCNT, 5)
end
logger.info "Listening on #{Postal.config.smtp_server.bind_address}:#{Postal.config.smtp_server.port}"
end
def unlisten
# Instruct the nio loop to unlisten and wake it
@unlisten = true
@io_selector.wakeup
end
def run_event_loop
# Set up an instance of nio4r to monitor for connections and data
@io_selector = NIO::Selector.new
# Register the SMTP listener
@io_selector.register(@server, :r)
# Create a hash to contain a buffer for each client.
buffers = Hash.new { |h, k| h[k] = String.new.force_encoding("BINARY") }
loop do
# Wait for an event to occur
@io_selector.select do |monitor|
# Get the IO from the nio monitor
io = monitor.io
# Is this event an incoming connection?
if io.is_a?(TCPServer)
begin
# Accept the connection
new_io = io.accept
if Postal.config.smtp_server.proxy_protocol
# If we are using the haproxy proxy protocol, we will be sent the
# client's IP later. Delay the welcome process.
client = Client.new(nil)
if Postal.config.smtp_server.log_connect
logger.debug "[#{client.id}] \e[35m Connection opened from #{new_io.remote_address.ip_address}\e[0m"
end
else
# We're not using the proxy protocol so we already know the client's IP
client = Client.new(new_io.remote_address.ip_address)
if Postal.config.smtp_server.log_connect
logger.debug "[#{client.id}] \e[35m Connection opened from #{new_io.remote_address.ip_address}\e[0m"
end
# We know who the client is, welcome them.
client.log "\e[35m Client identified as #{new_io.remote_address.ip_address}\e[0m"
new_io.print("220 #{Postal.config.dns.smtp_server_hostname} ESMTP Postal/#{client.id}")
end
# Register the client and its socket with nio4r
monitor = @io_selector.register(new_io, :r)
monitor.value = client
rescue StandardError => e
# If something goes wrong, log as appropriate and disconnect the client
if defined?(Sentry)
Sentry.capture_exception(e, extra: { log_id: begin
client.id
rescue StandardError
nil
end })
end
logger.error "An error occurred while accepting a new client."
logger.error "#{e.class}: #{e.message}"
e.backtrace.each do |line|
logger.error line
end
begin
new_io.close
rescue StandardError
nil
end
end
else
# This event is not an incoming connection so it must be data from a client
begin
# Get the client from the nio monitor
client = monitor.value
# For now we assume the connection isn't closed
eof = false
# Is the client negotiating a TLS handshake?
if client.start_tls?
begin
# Can we accept the TLS connection at this time?
io.accept_nonblock
# We were able to accept the connection, the client is no longer handshaking
client.start_tls = false
rescue IO::WaitReadable, IO::WaitWritable => e
# Could not accept without blocking
# We will try again later
next
rescue OpenSSL::SSL::SSLError => e
client.log "SSL Negotiation Failed: #{e.message}"
eof = true
end
else
# The client is not negotiating a TLS handshake at this time
begin
# Read 10kiB of data at a time from the socket.
buffers[io] << io.readpartial(10_240)
# There is an extra step for SSL sockets
if io.is_a?(OpenSSL::SSL::SSLSocket)
buffers[io] << io.readpartial(10_240) while io.pending.positive?
end
rescue EOFError, Errno::ECONNRESET, Errno::ETIMEDOUT
# Client went away
eof = true
end
# Normalize all \r\n and \n to \r\n, but ignore only \r.
# A \r\n may be split in 2 buffers (\n in one buffer and \r in the other)
buffers[io] = buffers[io].gsub(/\r/, "").encode(buffers[io].encoding, crlf_newline: true)
# We line buffer, so look to see if we have received a newline
# and keep doing so until all buffered lines have been processed.
while buffers[io].index("\r\n")
# Extract the line
line, buffers[io] = buffers[io].split("\r\n", 2)
# Send the received line to the client object for processing
result = client.handle(line)
# If the client object returned some data, write it back to the client
next if result.nil?
result = [result] unless result.is_a?(Array)
result.compact.each do |iline|
client.log "\e[34m=> #{iline.strip}\e[0m"
begin
io.write(iline.to_s + "\r\n")
io.flush
rescue Errno::ECONNRESET
# Client disconnected before we could write response
eof = true
end
end
end
# Did the client request STARTTLS?
if !eof && client.start_tls?
# Deregister the unencrypted IO
@io_selector.deregister(io)
buffers.delete(io)
io = OpenSSL::SSL::SSLSocket.new(io, ssl_context)
# Close the underlying IO when the TLS socket is closed
io.sync_close = true
# Register the new TLS socket with nio
monitor = @io_selector.register(io, :r)
monitor.value = client
end
end
# Has the client requested we close the connection?
if client.finished? || eof
client.log "\e[35m Connection closed\e[0m"
# Deregister the socket and close it
@io_selector.deregister(io)
buffers.delete(io)
io.close
# If we have no more clients or listeners left, exit the process
if @io_selector.empty?
Process.exit(0)
end
end
rescue StandardError => e
# Something went wrong, log as appropriate
client_id = client ? client.id : "------"
if defined?(Sentry)
Sentry.capture_exception(e, extra: { log_id: begin
client.id
rescue StandardError
nil
end })
end
logger.error "[#{client_id}] An error occurred while processing data from a client."
logger.error "[#{client_id}] #{e.class}: #{e.message}"
e.backtrace.each do |iline|
logger.error "[#{client_id}] #{iline}"
end
# Close all IO and forget this client
begin
@io_selector.deregister(io)
rescue StandardError
nil
end
buffers.delete(io)
begin
io.close
rescue StandardError
nil
end
if @io_selector.empty?
Process.exit(0)
end
end
end
end
# If unlisten has been called, stop listening
next unless @unlisten
@io_selector.deregister(@server)
@server.close
# If there's nothing left to do, shut down the process
if @io_selector.empty?
Process.exit(0)
end
# Clear the request
@unlisten = false
end
end
def logger
Postal.logger_for(:smtp_server)
end
end
end
end
lib/postal/tracking_middleware.rb
-119
View File
@@ -1,119 +0,0 @@
# frozen_string_literal: true
module Postal
class TrackingMiddleware
TRACKING_PIXEL = File.read(Rails.root.join("app", "assets", "images", "tracking_pixel.png"))
def initialize(app = nil)
@app = app
end
def call(env)
unless env["HTTP_X_POSTAL_TRACK_HOST"].to_i == 1
return @app.call(env)
end
request = Rack::Request.new(env)
case request.path
when /\A\/img\/([a-z0-9-]+)\/([a-z0-9-]+)/i
server_token = ::Regexp.last_match(1)
message_token = ::Regexp.last_match(2)
dispatch_image_request(request, server_token, message_token)
when /\A\/([a-z0-9-]+)\/([a-z0-9-]+)/i
server_token = ::Regexp.last_match(1)
link_token = ::Regexp.last_match(2)
dispatch_redirect_request(request, server_token, link_token)
else
[200, {}, ["Hello."]]
end
end
private
def dispatch_image_request(request, server_token, message_token)
message_db = get_message_db_from_server_token(server_token)
if message_db.nil?
return [404, {}, ["Invalid Server Token"]]
end
begin
message = message_db.message(token: message_token)
message.create_load(request)
rescue Postal::MessageDB::Message::NotFound
# This message has been removed, we'll just continue to serve the image
rescue StandardError => e
# Somethign else went wrong. We don't want to stop the image loading though because
# this is our problem. Log this exception though.
Sentry.capture_exception(e) if defined?(Sentry)
end
source_image = request.params["src"]
case source_image
when nil
headers = {}
headers["Content-Type"] = "image/png"
headers["Content-Length"] = TRACKING_PIXEL.bytesize.to_s
[200, headers, [TRACKING_PIXEL]]
when /\Ahttps?:\/\//
response = Postal::HTTP.get(source_image, timeout: 3)
return [404, {}, ["Not found"]] unless response[:code] == 200
headers = {}
headers["Content-Type"] = response[:headers]["content-type"]&.first
headers["Last-Modified"] = response[:headers]["last-modified"]&.first
headers["Cache-Control"] = response[:headers]["cache-control"]&.first
headers["Etag"] = response[:headers]["etag"]&.first
headers["Content-Length"] = response[:body].bytesize.to_s
[200, headers, [response[:body]]]
else
[400, {}, ["Invalid/missing source image"]]
end
end
def dispatch_redirect_request(request, server_token, link_token)
message_db = get_message_db_from_server_token(server_token)
if message_db.nil?
return [404, {}, ["Invalid Server Token"]]
end
link = message_db.select(:links, where: { token: link_token }, limit: 1).first
if link.nil?
return [404, {}, ["Link not found"]]
end
time = Time.now.to_f
if link["message_id"]
message_db.update(:messages, { clicked: time }, where: { id: link["message_id"] })
message_db.insert(:clicks, {
message_id: link["message_id"],
link_id: link["id"],
ip_address: request.ip,
user_agent: request.user_agent,
timestamp: time
})
SendWebhookJob.queue(:main,
server_id: message_db.server_id,
event: "MessageLinkClicked",
payload: {
_message: link["message_id"],
url: link["url"],
token: link["token"],
ip_address: request.ip,
user_agent: request.user_agent
})
end
[307, { "Location" => link["url"] }, ["Redirected to: #{link['url']}"]]
end
def get_message_db_from_server_token(token)
return unless server = ::Server.find_by_token(token)
server.message_db
end
end
end
lib/postal/user_creator.rb
-34
View File
@@ -1,34 +0,0 @@
# frozen_string_literal: true
require "highline"
module Postal
module UserCreator
def self.start(&block)
cli = HighLine.new
puts "\e[32mPostal User Creator\e[0m"
puts "Enter the information required to create a new Postal user."
puts "This tool is usually only used to create your initial admin user."
puts
user = User.new
user.email_address = cli.ask("E-Mail Address".ljust(20, " ") + ": ")
user.first_name = cli.ask("First Name".ljust(20, " ") + ": ")
user.last_name = cli.ask("Last Name".ljust(20, " ") + ": ")
user.password = cli.ask("Initial Password".ljust(20, " ") + ": ") { |value| value.echo = "*" }
block.call(user) if block_given?
puts
if user.save
puts "User has been created with e-mail address \e[32m#{user.email_address}\e[0m"
else
puts "\e[31mFailed to create user\e[0m"
user.errors.full_messages.each do |error|
puts " * #{error}"
end
end
puts
end
end
end
lib/postal/worker.rb
-220
View File
@@ -1,220 +0,0 @@
# frozen_string_literal: true
module Postal
class Worker
def initialize(queues)
@initial_queues = queues
@active_queues = {}
@process_name = $0
@running_jobs = []
end
def work
logger.info "Worker running with #{Postal.config.workers.threads} threads"
Signal.trap("INT") do
@exit = true
set_process_name
end
Signal.trap("TERM") do
@exit = true
set_process_name
end
self.class.job_channel.prefetch(Postal.config.workers.threads)
@initial_queues.each { |queue| join_queue(queue) }
exit_checks = 0
loop do
if @exit && @running_jobs.empty?
logger.info "Exiting immediately because no jobs running"
exit 0
elsif @exit
if exit_checks >= 60
logger.info "Job did not finish in a timely manner. Exiting"
exit 0
end
if exit_checks.zero?
logger.info "Exit requested but job is running. Waiting for job to finish."
end
sleep 60
exit_checks += 1
else
manage_ip_queues
sleep 1
end
end
end
private
def receive_job(delivery_info, properties, body)
message = begin
JSON.parse(body)
rescue StandardError
nil
end
if message && message["class_name"]
@running_jobs << message["id"]
set_process_name
start_time = Time.now
Thread.current[:job_id] = message["id"]
logger.info "[#{message['id']}] Started processing \e[34m#{message['class_name']}\e[0m job"
begin
klass = message["class_name"].constantize.new(message["id"], message["params"])
klass.perform
GC.start
rescue StandardError => e
klass.on_error(e) if defined?(klass)
logger.warn "[#{message['id']}] \e[31m#{e.class}: #{e.message}\e[0m"
e.backtrace.each do |line|
logger.warn "[#{message['id']}] " + line
end
if defined?(Sentry)
Sentry.capture_exception(e, extra: { job_id: message["id"] })
end
ensure
logger.info "[#{message['id']}] Finished processing \e[34m#{message['class_name']}\e[0m job in #{Time.now - start_time}s"
end
end
ensure
Thread.current[:job_id] = nil
self.class.job_channel.ack(delivery_info.delivery_tag)
@running_jobs.delete(message["id"]) if message["id"]
set_process_name
if @exit && @running_jobs.empty?
logger.info "Exiting because all jobs have finished."
exit 0
end
end
def join_queue(queue)
if @active_queues[queue]
logger.info "Attempted to join queue #{queue} but already joined."
else
consumer = self.class.job_queue(queue).subscribe(manual_ack: true) do |delivery_info, properties, body|
receive_job(delivery_info, properties, body)
end
@active_queues[queue] = consumer
logger.info "Joined \e[32m#{queue}\e[0m queue"
end
end
def leave_queue(queue)
if consumer = @active_queues[queue]
consumer.cancel
@active_queues.delete(queue)
logger.info "Left \e[32m#{queue}\e[0m queue"
else
logger.info "Not joined #{queue} so cannot leave"
end
end
def manage_ip_queues
@ip_queues ||= []
@ip_to_id_mapping ||= {}
@unassigned_ips ||= []
@pairs ||= {}
@counter ||= 0
if @counter >= 15
@ip_to_id_mapping = {}
@unassigned_ips = []
@counter = 0
else
@counter += 1
end
# Get all IP addresses on the system
current_ip_addresses = Socket.ip_address_list.map(&:ip_address)
# Map them to an actual ID in the database if we can and cache that
needed_ip_ids = []
current_ip_addresses.each do |ip|
need = nil
if id = @ip_to_id_mapping[ip]
# We know this IPs ID, we'll just use that.
need = id
elsif @unassigned_ips.include?(ip)
# We know this IP isn't valid. We don't need to do anything
elsif !self.class.local_ip?(ip) && ip_address = IPAddress.where("ipv4 = ? OR ipv6 = ?", ip, ip).first
# We need to look this up
@pairs[ip_address.ipv4] = ip_address.ipv6
@ip_to_id_mapping[ip] = ip_address.id
need = ip_address.id
else
@unassigned_ips << ip
end
next unless need
pair = @pairs[ip] || @pairs.key(ip)
if pair.nil? || current_ip_addresses.include?(pair)
needed_ip_ids << @ip_to_id_mapping[ip]
else
logger.info "Host has '#{ip}' but its pair (#{pair}) isn't here. Cannot add now."
end
end
# Make an array of needed queue names
# Work out what we need to actually do here
missing_queues = needed_ip_ids - @ip_queues
unwanted_queues = @ip_queues - needed_ip_ids
# Leave the queues we don't want any more
unwanted_queues.each do |id|
leave_queue("outgoing-#{id}")
@ip_queues.delete(id)
ip_addresses_to_clear = []
@ip_to_id_mapping.each do |iip, iid|
if id == iid
ip_addresses_to_clear << iip
end
end
ip_addresses_to_clear.each { |ip| @ip_to_id_mapping.delete(ip) }
end
# Join any missing queues
missing_queues.uniq.each do |id|
join_queue("outgoing-#{id}")
@ip_queues << id
end
end
def set_process_name
prefix = @process_name.to_s
prefix += " [exiting]" if @exit
if @running_jobs.empty?
$0 = "#{prefix} (idle)"
else
$0 = "#{prefix} (running #{@running_jobs.join(', ')})"
end
end
def logger
self.class.logger
end
class << self
def logger
Postal.logger_for(:worker)
end
def job_channel
@job_channel ||= Postal::RabbitMQ.create_channel
end
def job_queue(name)
@job_queues ||= {}
@job_queues[name] ||= job_channel.queue("deliver-jobs-#{name}", durable: true, arguments: { "x-message-ttl" => 60_000 })
end
def local_ip?(ip)
!!(ip =~ /\A(127\.|fe80:|::)/)
end
end
end
end
lib/postal/yaml_config_exporter.rb
+37
View File
@@ -0,0 +1,37 @@
# frozen_string_literal: true
require "konfig/exporters/abstract"
module Postal
class YamlConfigExporter < Konfig::Exporters::Abstract
def export
contents = []
contents << "version: 2"
contents << ""
@schema.groups.each do |group_name, group|
contents << "#{group_name}:"
group.attributes.each do |name, attr|
contents << " # #{attr.description}"
if attr.array?
if attr.default.blank?
contents << " #{name}: []"
else
contents << " #{name}:"
attr.default.each do |d|
contents << " - #{d}"
end
end
else
contents << " #{name}: #{attr.default}"
end
end
contents << ""
end
contents.join("\n")
end
end
end