1
0
مراية لـ https://github.com/postalserver/postal.git تم المزامنة 2026-01-17 13:39:46 +00:00

refactor: refactor the SMTP sender

هذا الالتزام موجود في:
Adam Cooke
2024-02-29 10:32:57 +00:00
الأصل be0df7b463
التزام 633c509a45
11 ملفات معدلة مع 1291 إضافات و256 حذوفات

عرض الملف

@@ -75,17 +75,15 @@ class DNSResolver
# @return [Array<String>]
def effective_ns(name)
records = []
dns do |dns|
parts = name.split(".")
(parts.size - 1).times do |n|
d = parts[n, parts.size - n + 1].join(".")
parts = name.split(".")
(parts.size - 1).times do |n|
d = parts[n, parts.size - n + 1].join(".")
records = get_resources(d, Resolv::DNS::Resource::IN::NS).map do |s|
s.name.to_s
end
break if records.present?
records = get_resources(d, Resolv::DNS::Resource::IN::NS).map do |s|
s.name.to_s
end
break if records.present?
end
records

عرض الملف

@@ -0,0 +1,169 @@
# frozen_string_literal: true
module SMTPClient
class Endpoint
class SMTPSessionNotStartedError < StandardError
end
attr_reader :server
attr_reader :ip_address
attr_accessor :smtp_client
# @param server [Server] the server that this IP address is for
# @param ip_address [String] the IP address
def initialize(server, ip_address)
@server = server
@ip_address = ip_address
end
# Return a description of this server with its IP address
#
# @return [String]
def description
"#{@ip_address}:#{@server.port} (#{@server.hostname})"
end
# Return a string representation of this server
#
# @return [String]
def to_s
description
end
# Return true if this is an IPv6 address
#
# @return [Boolean]
def ipv6?
@ip_address.include?(":")
end
# Return true if this is an IPv4 address
#
# @return [Boolean]
def ipv4?
!ipv6?
end
# Start a new SMTP session and store the client with this server for future use as needed
#
# @param source_ip_address [IPAddress] the IP address to use as the source address for the connection
# @param allow_ssl [Boolean] whether to allow SSL for this connection, if false SSL mode is ignored
#
# @return [Net::SMTP]
def start_smtp_session(source_ip_address: nil, allow_ssl: true)
@smtp_client = Net::SMTP.new(@ip_address, @server.port)
@smtp_client.open_timeout = Postal::Config.smtp_client.open_timeout
@smtp_client.read_timeout = Postal::Config.smtp_client.read_timeout
@smtp_client.tls_hostname = @server.hostname
if source_ip_address
@source_ip_address = source_ip_address
end
if @source_ip_address
@smtp_client.source_address = ipv6? ? @source_ip_address.ipv6 : @source_ip_address.ipv4
end
if allow_ssl
case @server.ssl_mode
when SSLModes::AUTO
@smtp_client.enable_starttls_auto(self.class.ssl_context_without_verify)
when SSLModes::STARTTLS
@smtp_client.enable_starttls(self.class.ssl_context_with_verify)
when SSLModes::TLS
@smtp_client.enable_tls(self.class.ssl_context_with_verify)
else
@smtp_client.disable_starttls
@smtp_client.disable_tls
end
else
@smtp_client.disable_starttls
@smtp_client.disable_tls
end
@smtp_client.start(@source_ip_address ? @source_ip_address.hostname : self.class.default_helo_hostname)
@smtp_client
end
# Send a message to the current SMTP session (or create one if there isn't one for this endpoint).
# If sending messsage encouters some connection errors, retry again after re-establishing the SMTP
# session.
#
# @param raw_message [String] the raw message to send
# @param mail_from [String] the MAIL FROM address
# @param rcpt_to [String] the RCPT TO address
# @param retry_on_connection_error [Boolean] whether to retry the connection if there is a connection error
#
# @return [void]
def send_message(raw_message, mail_from, rcpt_to, retry_on_connection_error: true)
raise SMTPSessionNotStartedError if @smtp_client.nil? || (@smtp_client && !@smtp_client.started?)
@smtp_client.rset_errors
@smtp_client.send_message(raw_message, mail_from, [rcpt_to])
rescue Errno::ECONNRESET, Errno::EPIPE, OpenSSL::SSL::SSLError
if retry_on_connection_error
finish_smtp_session
start_smtp_session
return send_message(raw_message, mail_from, rcpt_to, retry_on_connection_error: false)
end
raise
end
# Reset the current SMTP session for this server if possible otherwise
# finish the session
#
# @return [void]
def reset_smtp_session
@smtp_client&.rset
rescue StandardError
finish_smtp_session
end
# Finish the current SMTP session for this server if possible.
#
# @return [void]
def finish_smtp_session
@smtp_client&.finish
rescue StandardError
nil
ensure
@smtp_client = nil
end
class << self
# Return the default HELO hostname to present to SMTP servers that
# we connect to
#
# @return [String]
def default_helo_hostname
Postal::Config.dns.helo_hostname ||
Postal::Config.postal.smtp_hostname ||
"localhost"
end
def ssl_context_with_verify
@ssl_context_with_verify ||= begin
c = OpenSSL::SSL::SSLContext.new
c.verify_mode = OpenSSL::SSL::VERIFY_PEER
c.cert_store = OpenSSL::X509::Store.new
c.cert_store.set_default_paths
c
end
end
def ssl_context_without_verify
@ssl_context_without_verify ||= begin
c = OpenSSL::SSL::SSLContext.new
c.verify_mode = OpenSSL::SSL::VERIFY_NONE
c
end
end
end
end
end

عرض الملف

@@ -0,0 +1,35 @@
# frozen_string_literal: true
module SMTPClient
class Server
attr_reader :hostname
attr_reader :port
attr_accessor :ssl_mode
def initialize(hostname, port: 25, ssl_mode: SSLModes::AUTO)
@hostname = hostname
@port = port
@ssl_mode = ssl_mode
end
# Return all IP addresses for this server by resolving its hostname.
# IPv6 addresses will be returned first.
#
# @return [Array<SMTPClient::Endpoint>]
def endpoints
ips = []
DNSResolver.local.aaaa(@hostname).each do |ip|
ips << Endpoint.new(self, ip)
end
DNSResolver.local.a(@hostname).each do |ip|
ips << Endpoint.new(self, ip)
end
ips
end
end
end

عرض الملف

@@ -0,0 +1,12 @@
# frozen_string_literal: true
module SMTPClient
module SSLModes
AUTO = "Auto"
STARTTLS = "STARTLS"
TLS = "TLS"
NONE = "None"
end
end

عرض الملف

@@ -2,293 +2,255 @@
class SMTPSender < BaseSender
def initialize(domain, source_ip_address, options = {})
attr_reader :endpoints
# @param domain [String] the domain to send mesages to
# @param source_ip_address [IPAddress] the IP address to send messages from
# @param log_id [String] an ID to use when logging requests
def initialize(domain, source_ip_address = nil, servers: nil, log_id: nil, rcpt_to: nil)
super()
@domain = domain
@source_ip_address = source_ip_address
@options = options
@smtp_client = nil
@rcpt_to = rcpt_to
# An array of servers to forcefully send the message to
@servers = servers
# Stores all connection errors which we have seen during this send sesssion.
@connection_errors = []
@hostnames = []
@log_id = Nifty::Utils::RandomString.generate(length: 8).upcase
# Stores all endpoints that we have attempted to deliver mail to
@endpoints = []
# Generate a log ID which can be used if none has been provided to trace
# this SMTP session.
@log_id = log_id || SecureRandom.alphanumeric(8).upcase
end
def start
servers = @servers || self.class.smtp_relays || resolve_mx_records_for_domain || []
servers.each do |server|
if server.is_a?(SMTPEndpoint)
hostname = server.hostname
port = server.port || 25
ssl_mode = server.ssl_mode
elsif server.is_a?(Hash)
hostname = server[:hostname]
port = server[:port] || 25
ssl_mode = server[:ssl_mode] || "Auto"
else
hostname = server
port = 25
ssl_mode = "Auto"
end
@hostnames << hostname
[:aaaa, :a].each do |ip_type|
if @source_ip_address && @source_ip_address.ipv6.blank? && ip_type == :aaaa
# Don't try to use IPv6 if the IP address we're sending from doesn't support it.
next
end
begin
@remote_ip = lookup_ip_address(ip_type, hostname)
if @remote_ip.nil?
if ip_type == :a
# As we can't resolve the last IP, we'll put this
@connection_errors << "Could not resolve #{hostname}"
end
next
end
smtp_client = Net::SMTP.new(@remote_ip, port)
smtp_client.open_timeout = Postal::Config.smtp_client.open_timeout
smtp_client.read_timeout = Postal::Config.smtp_client.read_timeout
smtp_client.tls_hostname = hostname
if @source_ip_address
# Set the source IP as appropriate
smtp_client.source_address = ip_type == :aaaa ? @source_ip_address.ipv6 : @source_ip_address.ipv4
end
case ssl_mode
when "Auto"
smtp_client.enable_starttls_auto(self.class.ssl_context_without_verify)
when "STARTTLS"
smtp_client.enable_starttls(self.class.ssl_context_with_verify)
when "TLS"
smtp_client.enable_tls(self.class.ssl_context_with_verify)
else
smtp_client.disable_starttls
smtp_client.disable_tls
end
smtp_client.start(@source_ip_address ? @source_ip_address.hostname : self.class.default_helo_hostname)
log "Connected to #{@remote_ip}:#{port} (#{hostname})"
rescue StandardError => e
if e.is_a?(OpenSSL::SSL::SSLError) && ssl_mode == "Auto"
log "SSL error (#{e.message}), retrying without SSL"
ssl_mode = nil
retry
end
log "Cannot connect to #{@remote_ip}:#{port} (#{hostname}) (#{e.class}: #{e.message})"
@connection_errors << e.message unless @connection_errors.include?(e.message)
begin
smtp_client.finish
rescue StandardError
nil
end
smtp_client = nil
end
if smtp_client
@smtp_client = smtp_client
return true
end
server.endpoints.each do |endpoint|
result = connect_to_endpoint(endpoint)
return endpoint if result
end
end
@connection_errors
false
end
def reconnect
log "Reconnecting"
begin
@smtp_client&.finish
rescue StandardError
nil
end
start
end
def safe_rset
# Something went wrong sending the last email. Reset the connection if possible, else disconnect.
@smtp_client.rset
rescue StandardError
# Don't reconnect, this would be rather rude if we don't have any more emails to send.
begin
@smtp_client.finish
rescue StandardError
nil
end
end
def send_message(message, force_rcpt_to = nil)
start_time = Time.now
result = SendResult.new
result.log_id = @log_id
if @smtp_client && !@smtp_client.started?
# For some reason we had an SMTP connection but it's no longer connected.
# Make a new one.
start
end
if @smtp_client
result.secure = @smtp_client.secure_socket?
end
begin
if message.bounce
mail_from = ""
elsif message.domain.return_path_status == "OK"
mail_from = "#{message.server.token}@#{message.domain.return_path_domain}"
else
mail_from = "#{message.server.token}@#{Postal::Config.dns.return_path_domain}"
end
if Postal::Config.postal.use_resent_sender_header
raw_message = "Resent-Sender: #{mail_from}\r\n" + message.raw_message
else
raw_message = message.raw_message
end
tries = 0
begin
if @smtp_client.nil?
log "-> No SMTP server available for #{@domain}"
log "-> Hostnames: #{@hostnames.inspect}"
log "-> Errors: #{@connection_errors.inspect}"
result.type = "SoftFail"
result.retry = true
result.details = "No SMTP servers were available for #{@domain}. Tried #{@hostnames.to_sentence}"
result.output = @connection_errors.join(", ")
result.connect_error = true
return result
def send_message(message)
# If we don't have a current endpoint than we should raise an error.
if @current_endpoint.nil?
return create_result("SoftFail") do |r|
r.retry = true
r.details = "No SMTP servers were available for #{@domain}."
if @endpoints.empty?
r.details += " No hosts to try."
else
@smtp_client.rset_errors
rcpt_to = force_rcpt_to || @options[:force_rcpt_to] || message.rcpt_to
log "Sending message #{message.server.id}::#{message.id} to #{rcpt_to}"
smtp_result = @smtp_client.send_message(raw_message, mail_from, [rcpt_to])
hostnames = @endpoints.map { |e| e.server.hostname }.uniq
r.details += " Tried #{hostnames.to_sentence}."
end
rescue Errno::ECONNRESET, Errno::EPIPE, OpenSSL::SSL::SSLError
raise unless (tries += 1) < 2
reconnect
retry
r.output = @connection_errors.join(", ")
r.connect_error = true
end
result.type = "Sent"
result.details = "Message for #{rcpt_to} accepted by #{destination_host_description}"
if @smtp_client.source_address
result.details += " (from #{@smtp_client.source_address})"
end
result.output = smtp_result.string
log "Message sent ##{message.id} to #{destination_host_description} for #{rcpt_to}"
rescue Net::SMTPServerBusy, Net::SMTPAuthenticationError, Net::SMTPSyntaxError, Net::SMTPUnknownError, Net::ReadTimeout => e
log "#{e.class}: #{e.message}"
result.type = "SoftFail"
result.retry = true
result.details = "Temporary SMTP delivery error when sending to #{destination_host_description}"
result.output = e.message
if e.to_s =~ /(\d+) seconds/
result.retry = ::Regexp.last_match(1).to_i + 10
elsif e.to_s =~ /(\d+) minutes/
result.retry = (::Regexp.last_match(1).to_i * 60) + 10
end
safe_rset
rescue Net::SMTPFatalError => e
log "#{e.class}: #{e.message}"
result.type = "HardFail"
result.details = "Permanent SMTP delivery error when sending to #{destination_host_description}"
result.output = e.message
safe_rset
rescue StandardError => e
log "#{e.class}: #{e.message}"
if defined?(Sentry)
Sentry.capture_exception(e, extra: { log_id: @log_id, server_id: message.server.id, message_id: message.id })
end
result.type = "SoftFail"
result.retry = true
result.details = "An error occurred while sending the message to #{destination_host_description}"
result.output = e.message
safe_rset
end
result.time = (Time.now - start_time).to_f.round(2)
result
mail_from = determine_mail_from_for_message(message)
raw_message = message.raw_message
# Append the Resent-Sender header to the mesage to include the
# MAIL FROM if the installation is configured to use that?
if Postal::Config.postal.use_resent_sender_header?
raw_message = "Resent-Sender: #{mail_from}\r\n" + raw_message
end
rcpt_to = determine_rcpt_to_for_message(message)
logger.info "Sending message #{message.server.id}::#{message.id} to #{rcpt_to}"
send_message_to_smtp_client(raw_message, mail_from, rcpt_to)
end
def finish
log "Finishing up"
@smtp_client&.finish
@endpoints.each(&:finish_smtp_session)
end
private
def servers
@options[:servers] || self.class.relay_hosts || @servers ||= begin
mx_servers = DNSResolver.local.mx(@domain).map(&:last)
if mx_servers.empty?
mx_servers = [@domain] # This will be resolved to an A or AAAA record later
# Take a message and attempt to send it to the SMTP server that we are
# currently connected to. If there is a connection error, we will just
# reset the client and retry again once.
#
# @param raw_message [String] the raw message to send
# @param mail_from [String] the MAIL FROM address to use
# @param rcpt_to [String] the RCPT TO address to use
# @param retry_on_connection_error [Boolean] if true, we will retry the connection if there is an error
#
# @return [SendResult]
def send_message_to_smtp_client(raw_message, mail_from, rcpt_to, retry_on_connection_error: true)
start_time = Time.now
smtp_result = @current_endpoint.send_message(raw_message, mail_from, [rcpt_to])
logger.info "Accepted by #{@current_endpoint} for #{rcpt_to}"
create_result("Sent", start_time) do |r|
r.details = "Message for #{rcpt_to} accepted by #{@current_endpoint}"
r.details += " (from #{@current_endpoint.smtp_client.source_address})" if @current_endpoint.smtp_client.source_address
r.output = smtp_result.string
end
rescue Net::SMTPServerBusy, Net::SMTPAuthenticationError, Net::SMTPSyntaxError, Net::SMTPUnknownError, Net::ReadTimeout => e
logger.error "#{e.class}: #{e.message}"
@current_endpoint.reset_smtp_session
create_result("SoftFail", start_time) do |r|
r.details = "Temporary SMTP delivery error when sending to #{@current_endpoint}"
r.output = e.message
if e.message =~ /(\d+) seconds/
r.retry = ::Regexp.last_match(1).to_i + 10
elsif e.message =~ /(\d+) minutes/
r.retry = (::Regexp.last_match(1).to_i * 60) + 10
else
r.retry = true
end
mx_servers
end
rescue Net::SMTPFatalError => e
logger.error "#{e.class}: #{e.message}"
@current_endpoint.reset_smtp_session
create_result("HardFail", start_time) do |r|
r.details = "Permanent SMTP delivery error when sending to #{@current_endpoint}"
r.output = e.message
end
rescue StandardError => e
logger.error "#{e.class}: #{e.message}"
@current_endpoint.reset_smtp_session
if defined?(Sentry)
# Sentry.capture_exception(e, extra: { log_id: @log_id, server_id: message.server.id, message_id: message.id })
end
create_result("SoftFail", start_time) do |r|
r.type = "SoftFail"
r.retry = true
r.details = "An error occurred while sending the message to #{@current_endpoint}"
r.output = e.message
end
end
def log(text)
Postal.logger.info text, id: @log_id, component: "smtp-sender"
end
# Return the MAIL FROM which should be used for the given message
#
# @param message [MessageDB::Message]
# @return [String]
def determine_mail_from_for_message(message)
return "" if message.bounce
def destination_host_description
"#{@hostnames.last} (#{@remote_ip})"
end
def lookup_ip_address(type, hostname)
records = []
case type
when :a
records = DNSResolver.local.a(hostname)
when :aaaa
records = DNSResolver.local.aaaa(hostname)
# If the domain has a valid custom return path configured, return
# that.
if message.domain.return_path_status == "OK"
return "#{message.server.token}@#{message.domain.return_path_domain}"
end
records.first&.to_s&.downcase
"#{message.server.token}@#{Postal::Config.dns.return_path_domain}"
end
# Return the RCPT TO to use for the given message in this sending session
#
# @param message [MessageDB::Message]
# @return [String]
def determine_rcpt_to_for_message(message)
return @rcpt_to if @rcpt_to
message.rcpt_to
end
# Return an array of server hostnames which should receive this message
#
# @return [Array<String>]
def resolve_mx_records_for_domain
hostnames = DNSResolver.local.mx(@domain).map(&:last)
return [SMTPClient::Server.new(@domain)] if hostnames.empty?
hostnames.map { |hostname| SMTPClient::Server.new(hostname) }
end
# Attempt to begin an SMTP sesssion for the given endpoint. If successful, this endpoint
# becomes the current endpoints for the SMTP sender.
#
# Returns true if the session was established.
# Returns false if the session could not be established.
#
# @param endpoint [SMTPClient::Endpoint]
# @return [Boolean]
def connect_to_endpoint(endpoint, allow_ssl: true)
if @source_ip_address && @source_ip_address.ipv6.blank? && endpoint.ipv6?
# Don't try to use IPv6 if the IP address we're sending from doesn't support it.
return false
end
# Add this endpoint to the list of endpoints that we have attempted to connect to
@endpoints << endpoint unless @endpoints.include?(endpoint)
endpoint.start_smtp_session(allow_ssl: allow_ssl, source_ip_address: @source_ip_address)
logger.info "Connected to #{endpoint}"
@current_endpoint = endpoint
true
rescue StandardError => e
# Disconnect the SMTP client if we get any errors to avoid leaving
# a connection around.
endpoint.finish_smtp_session
# If we get an SSL error, we can retry a connection without
# ssl.
if e.is_a?(OpenSSL::SSL::SSLError) && endpoint.server.ssl_mode == "Auto"
logger.error "SSL error (#{e.message}), retrying without SSL"
return connect_to_endpoint(endpoint, allow_ssl: false)
end
# Otherwise, just log the connection error and return false
logger.error "Cannot connect to #{endpoint} (#{e.class}: #{e.message})"
@connection_errors << e.message unless @connection_errors.include?(e.message)
false
end
# Create a new result object
#
# @param type [String] the type of result
# @param start_time [Time] the time the operation started
# @yieldparam [SendResult] the result object
# @yieldreturn [void]
#
# @return [SendResult]
def create_result(type, start_time = nil)
result = SendResult.new
result.type = type
result.log_id = @log_id
result.secure = @current_endpoint&.smtp_client&.secure_socket? ? true : false
yield result if block_given?
if start_time
result.time = (Time.now - start_time).to_f.round(2)
end
result
end
def logger
@logger ||= Postal.logger.create_tagged_logger(log_id: @log_id)
end
class << self
def ssl_context_with_verify
@ssl_context_with_verify ||= begin
c = OpenSSL::SSL::SSLContext.new
c.verify_mode = OpenSSL::SSL::VERIFY_PEER
c.cert_store = OpenSSL::X509::Store.new
c.cert_store.set_default_paths
c
end
end
# Return an array of SMTP relays as configured. Returns nil
# if no SMTP relays are configured.
#
def smtp_relays
return @smtp_relays if instance_variable_defined?("@smtp_relays")
def ssl_context_without_verify
@ssl_context_without_verify ||= begin
c = OpenSSL::SSL::SSLContext.new
c.verify_mode = OpenSSL::SSL::VERIFY_NONE
c
end
end
def default_helo_hostname
Postal::Config.dns.helo_hostname ||
Postal::Config.postal.smtp_hostname ||
"localhost"
end
def relay_hosts
relays = Postal::Config.postal.smtp_relays
return nil if relays.nil?
hosts = relays.map do |relay|
relays.map do |relay|
next unless relay.host.present?
{
hostname: relay.host,
port: relay.port,
ssl_mode: relay.ssl_mode
}
SMTPClient::Server.new(relay.host, relay.port, ssl_mode: relay.ssl_mode)
end.compact
hosts.empty? ? nil : hosts
@smtp_relays = hosts.empty? ? nil : hosts
end
end