ghaymah_dev/postal
1
0
Fork 0
mirror of https://github.com/postalserver/postal.git synced 2026-10-20 11:17:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

style(rubocop): fix all safe auto correctable offenses

Browse Source
This commit is contained in:
Charlie Smurthwaite
2023-03-16 15:50:53 +00:00
parent 02c93a4850
commit fd289c46fd
204 changed files with 2611 additions and 2486 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/postal/app_logger.rb
+17 -8
View File
@@ -1,6 +1,7 @@
require 'logger'
require "logger"
module Postal
class AppLogger < Logger
def initialize(log_name, *args)
@@ -16,10 +17,14 @@ module Postal
if message.nil?
message = block_given? ? yield : progname
end
message = message.to_s.force_encoding('UTF-8').scrub
message_without_ansi = message.gsub(/\e\[([\d\;]+)?m/, '') rescue message
n.notify!(:short_message => message_without_ansi, :log_name => @log_name, :facility => 'postal', :application_name => 'postal', :process_name => ENV['PROC_NAME'], :pid => Process.pid)
rescue => e
message = message.to_s.force_encoding("UTF-8").scrub
message_without_ansi = begin
message.gsub(/\e\[([\d;]+)?m/, "")
rescue StandardError
message
end
n.notify!(short_message: message_without_ansi, log_name: @log_name, facility: "postal", application_name: "postal", process_name: ENV.fetch("PROC_NAME", nil), pid: Process.pid)
rescue StandardError => e
# Can't log this to GELF. Soz.
end
end
@@ -33,21 +38,25 @@ module Postal
def self.graylog_notifier
@graylog_notifier ||= graylog? ? GELF::Notifier.new(Postal.config.logging.graylog.host, Postal.config.logging.graylog.port) : nil
end
end
class LogFormatter
TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%3N".freeze
COLORS = [32,34,35,31,32,33]
COLORS = [32, 34, 35, 31, 32, 33]
def call(severity, datetime, progname, msg)
time = datetime.strftime(TIME_FORMAT)
if number = ENV['PROC_NAME']
id = number.split('.').last.to_i
if number = ENV["PROC_NAME"]
id = number.split(".").last.to_i
proc_text = "\e[#{COLORS[id % COLORS.size]}m[#{ENV['PROC_NAME']}:#{Process.pid}]\e[0m"
else
proc_text = "[#{Process.pid}]"
end
"#{proc_text} [#{time}] #{severity} -- : #{msg}\n"
end
end
end
lib/postal/bounce_message.rb
+4 -4
View File
@@ -12,18 +12,18 @@ module Postal
mail.from = "Mail Delivery Service <#{@message.route.description}>"
mail.subject = "Mail Delivery Failed (#{@message.subject})"
mail.text_part = body
mail.attachments['Original Message.eml'] = {:mime_type => 'message/rfc822', :encoding => 'quoted-printable', :content => @message.raw_message}
mail.attachments["Original Message.eml"] = { mime_type: "message/rfc822", encoding: "quoted-printable", content: @message.raw_message }
mail.message_id = "<#{SecureRandom.uuid}@#{Postal.config.dns.return_path}>"
mail.to_s
end
def queue
message = @server.message_db.new_message
message.scope = 'outgoing'
message.scope = "outgoing"
message.rcpt_to = @message.mail_from
message.mail_from = @message.route.description
message.domain_id = @message.domain&.id
message.raw_message = self.raw_message
message.raw_message = raw_message
message.bounce = 1
message.bounce_for_id = @message.id
message.save
@@ -37,7 +37,7 @@ module Postal
private
def body
<<-BODY.strip_heredoc
<<~BODY
This is the mail delivery service responsible for delivering mail to #{@message.route.description}.
The message you've sent cannot be delivered. Your original message is attached to this message.
lib/postal/config.rb
+44 -50
View File
@@ -1,10 +1,10 @@
require 'yaml'
require 'pathname'
require 'cgi'
require 'openssl'
require 'fileutils'
require_relative 'error'
require_relative 'version'
require "yaml"
require "pathname"
require "cgi"
require "openssl"
require "fileutils"
require_relative "error"
require_relative "version"
module Postal
@@ -21,50 +21,44 @@ module Postal
end
def self.app_root
@app_root ||= Pathname.new(File.expand_path('../../../', __FILE__))
@app_root ||= Pathname.new(File.expand_path("../..", __dir__))
end
def self.config
@config ||= begin
require 'hashie/mash'
config = Hashie::Mash.new(self.defaults)
config = config.deep_merge(self.yaml_config)
config.deep_merge(self.local_yaml_config)
require "hashie/mash"
config = Hashie::Mash.new(defaults)
config = config.deep_merge(yaml_config)
config.deep_merge(local_yaml_config)
end
end
def self.config_root
@config_root ||= begin
if ENV['POSTAL_CONFIG_ROOT']
Pathname.new(ENV['POSTAL_CONFIG_ROOT'])
else
Pathname.new(File.expand_path("../../../config/postal", __FILE__))
end
if ENV["POSTAL_CONFIG_ROOT"]
@config_root ||= Pathname.new(ENV["POSTAL_CONFIG_ROOT"])
else
@config_root ||= Pathname.new(File.expand_path("../../config/postal", __dir__))
end
end
def self.log_root
@log_root ||= begin
if config.logging.root
Pathname.new(config.logging.root)
else
app_root.join('log')
end
if config.logging.root
@log_root ||= Pathname.new(config.logging.root)
else
@log_root ||= app_root.join("log")
end
end
def self.config_file_path
@config_file_path ||= begin
if env == 'default'
File.join(config_root, 'postal.yml')
else
File.join(config_root, "postal.#{env}.yml")
end
if env == "default"
@config_file_path ||= File.join(config_root, "postal.yml")
else
@config_file_path ||= File.join(config_root, "postal.#{env}.yml")
end
end
def self.env
@env ||= ENV.fetch('POSTAL_ENV', 'default')
@env ||= ENV.fetch("POSTAL_ENV", "default")
end
def self.yaml_config
@@ -72,7 +66,7 @@ module Postal
end
def self.local_config_file_path
@local_config_file_path ||= File.join(config_root, 'postal.local.yml')
@local_config_file_path ||= File.join(config_root, "postal.local.yml")
end
def self.local_yaml_config
@@ -80,11 +74,11 @@ module Postal
end
def self.defaults_file_path
@defaults_file_path ||= app_root.join('config', 'postal.defaults.yml')
@defaults_file_path ||= app_root.join("config", "postal.defaults.yml")
end
def self.defaults
@defaults ||= YAML.load_file(self.defaults_file_path)
@defaults ||= YAML.load_file(defaults_file_path)
end
def self.database_url
@@ -98,8 +92,8 @@ module Postal
def self.logger_for(name)
@loggers ||= {}
@loggers[name.to_sym] ||= begin
require 'postal/app_logger'
if config.logging.stdout || ENV['LOG_TO_STDOUT']
require "postal/app_logger"
if config.logging.stdout || ENV["LOG_TO_STDOUT"]
Postal::AppLogger.new(name, STDOUT)
else
FileUtils.mkdir_p(log_root)
@@ -111,9 +105,9 @@ module Postal
def self.process_name
@process_name ||= begin
string = "host:#{Socket.gethostname} pid:#{Process.pid}"
string += " procname:#{ENV['PROC_NAME']}" if ENV['PROC_NAME']
string += " procname:#{ENV['PROC_NAME']}" if ENV["PROC_NAME"]
string
rescue
rescue StandardError
"pid:#{Process.pid}"
end
end
@@ -133,7 +127,7 @@ module Postal
end
def self.smtp_private_key_path
config.smtp_server.tls_private_key_path || config_root.join('smtp.key')
config.smtp_server.tls_private_key_path || config_root.join("smtp.key")
end
def self.smtp_private_key
@@ -141,7 +135,7 @@ module Postal
end
def self.smtp_certificate_path
config.smtp_server.tls_certificate_path || config_root.join('smtp.cert')
config.smtp_server.tls_certificate_path || config_root.join("smtp.cert")
end
def self.smtp_certificate_data
@@ -150,7 +144,7 @@ module Postal
def self.smtp_certificates
@smtp_certificates ||= begin
certs = self.smtp_certificate_data.scan(/-----BEGIN CERTIFICATE-----.+?-----END CERTIFICATE-----/m)
certs = smtp_certificate_data.scan(/-----BEGIN CERTIFICATE-----.+?-----END CERTIFICATE-----/m)
certs.map do |c|
OpenSSL::X509::Certificate.new(c)
end.freeze
@@ -158,7 +152,7 @@ module Postal
end
def self.signing_key_path
config_root.join('signing.key')
config_root.join("signing.key")
end
def self.signing_key
@@ -166,7 +160,7 @@ module Postal
end
def self.rp_dkim_dns_record
public_key = signing_key.public_key.to_s.gsub(/\-+[A-Z ]+\-+\n/, '').gsub(/\n/, '')
public_key = signing_key.public_key.to_s.gsub(/-+[A-Z ]+-+\n/, "").gsub(/\n/, "")
"v=DKIM1; t=s; h=sha256; p=#{public_key};"
end
@@ -174,19 +168,19 @@ module Postal
end
def self.check_config!
return if ENV['POSTAL_SKIP_CONFIG_CHECK'].to_i == 1
return if ENV["POSTAL_SKIP_CONFIG_CHECK"].to_i == 1
unless File.exist?(self.config_file_path)
raise ConfigError, "No config found at #{self.config_file_path}"
unless File.exist?(config_file_path)
raise ConfigError, "No config found at #{config_file_path}"
end
unless File.exists?(self.signing_key_path)
raise ConfigError, "No signing key found at #{self.signing_key_path}"
end
return if File.exist?(signing_key_path)
raise ConfigError, "No signing key found at #{signing_key_path}"
end
def self.ip_pools?
self.config.general.use_ip_pools?
config.general.use_ip_pools?
end
end
lib/postal/countries.rb
+2
View File
@@ -1,5 +1,7 @@
module Postal
module Countries
NAMES = ["Afghanistan", "Aland Islands", "Albania", "Algeria", "American Samoa", "Andorra", "Angola", "Anguilla", "Antarctica", "Antigua and Barbuda", "Argentina", "Armenia", "Aruba", "Australia", "Austria", "Azerbaijan", "Bahamas", "Bahrain", "Bangladesh", "Barbados", "Belarus", "Belgium", "Belize", "Benin", "Bermuda", "Bhutan", "Bolivia, Plurinational State of", "Bosnia and Herzegovina", "Botswana", "Bouvet Island", "Brazil", "British Indian Ocean Territory", "Brunei Darussalam", "Bulgaria", "Burkina Faso", "Burundi", "Cambodia", "Cameroon", "Canada", "Cape Verde", "Cayman Islands", "Central African Republic", "Chad", "Chile", "China", "Christmas Island", "Cocos (keeling) Islands", "Colombia", "Comoros", "Congo", "Congo, the Democratic Republic of the", "Cook Islands", "Costa Rica", "Cote D'ivoire", "Croatia", "Cuba", "Cyprus", "Czech Republic", "Denmark", "Djibouti", "Dominica", "Dominican Republic", "Ecuador", "Egypt", "El Salvador", "Equatorial Guinea", "Eritrea", "Estonia", "Ethiopia", "Falkland Islands (malvinas)", "Faroe Islands", "Fiji", "Finland", "France", "French Guiana", "French Polynesia", "French Southern Territories", "Gabon", "Gambia", "Georgia", "Germany", "Ghana", "Gibraltar", "Greece", "Greenland", "Grenada", "Guadeloupe", "Guam", "Guatemala", "Guernsey", "Guinea", "Guinea-bissau", "Guyana", "Haiti", "Heard Island and Mcdonald Islands", "Holy See (Vatican City State)", "Honduras", "Hong Kong", "Hungary", "Iceland", "India", "Indonesia", "Iran, Islamic Republic of", "Iraq", "Ireland", "Isle of Man", "Israel", "Italy", "Jamaica", "Japan", "Jersey", "Jordan", "Kazakhstan", "Kenya", "Kiribati", "Korea, Democratic People's Republic of", "Korea, Republic of", "Kuwait", "Kyrgyzstan", "Lao People's Democratic Republic", "Latvia", "Lebanon", "Lesotho", "Liberia", "Libyan Arab Jamahiriya", "Liechtenstein", "Lithuania", "Luxembourg", "Macao", "Macedonia, the Former Yugoslav Republic of", "Madagascar", "Malawi", "Malaysia", "Maldives", "Mali", "Malta", "Marshall Islands", "Martinique", "Mauritania", "Mauritius", "Mayotte", "Mexico", "Micronesia, Federated States of", "Moldova, Republic of", "Monaco", "Mongolia", "Montenegro", "Montserrat", "Morocco", "Mozambique", "Myanmar", "Namibia", "Nauru", "Nepal", "Netherlands", "Netherlands Antilles", "New Caledonia", "New Zealand", "Nicaragua", "Niger", "Nigeria", "Niue", "Norfolk Island", "Northern Mariana Islands", "Norway", "Oman", "Pakistan", "Palau", "Palestinian Territory, Occupied", "Panama", "Papua New Guinea", "Paraguay", "Peru", "Philippines", "Pitcairn", "Poland", "Portugal", "Puerto Rico", "Qatar", "Reunion", "Romania", "Russian Federation", "Rwanda", "Saint Barthelemy", "Saint Helena, Ascension and Tristan Da Cunha", "Saint Kitts and Nevis", "Saint Lucia", "Saint Martin", "Saint Pierre and Miquelon", "Saint Vincent and the Grenadines", "Samoa", "San Marino", "Sao Tome and Principe", "Saudi Arabia", "Senegal", "Serbia", "Seychelles", "Sierra Leone", "Singapore", "Slovakia", "Slovenia", "Solomon Islands", "Somalia", "South Africa", "South Georgia and the South Sandwich Islands", "Spain", "Sri Lanka", "Sudan", "Suriname", "Svalbard and Jan Mayen", "Swaziland", "Sweden", "Switzerland", "Syrian Arab Republic", "Taiwan", "Tajikistan", "Tanzania, United Republic of", "Thailand", "Timor-leste", "Togo", "Tokelau", "Tonga", "Trinidad and Tobago", "Tunisia", "Turkey", "Turkmenistan", "Turks and Caicos Islands", "Tuvalu", "Uganda", "Ukraine", "United Arab Emirates", "United Kingdom", "United States", "United States Minor Outlying Islands", "Uruguay", "Uzbekistan", "Vanuatu", "Venezuela, Bolivarian Republic of", "Viet Nam", "Virgin Islands, British", "Virgin Islands, U.S.", "Wallis and Futuna", "Western Sahara", "Yemen", "Zambia", "Zimbabwe"]
end
end
lib/postal/dkim_header.rb
+15 -15
View File
@@ -2,14 +2,14 @@ module Postal
class DKIMHeader
def initialize(domain, message)
if domain && domain.dkim_status == 'OK'
if domain && domain.dkim_status == "OK"
@domain_name = domain.name
@dkim_key = domain.dkim_key
@dkim_identifier = domain.dkim_identifier
else
@domain_name = Postal.config.dns.return_path
@dkim_key = Postal.signing_key
@dkim_identifier = 'postal'
@dkim_identifier = "postal"
end
@domain = domain
@message = message
@@ -23,15 +23,15 @@ module Postal
private
def headers
@headers ||= @raw_headers.to_s.gsub(/\r?\n\s/, ' ').split(/\r?\n/)
@headers ||= @raw_headers.to_s.gsub(/\r?\n\s/, " ").split(/\r?\n/)
end
def header_names
normalized_headers.map{ |h| h.split(':')[0].strip }
normalized_headers.map { |h| h.split(":")[0].strip }
end
def normalized_headers
Array.new.tap do |new_headers|
[].tap do |new_headers|
headers.select { |h| h.match(/^(from|sender|reply-to|subject|date|message-id|to|cc|mime-version|content-type|content-transfer-encoding|resent-to|resent-cc|resent-from|resent-sender|resent-message-id|in-reply-to|references|list-id|list-help|list-owner|list-unsubscribe|list-subscribe|list-post):/i) }.each do |h|
new_headers << normalize_header(h)
end
@@ -45,26 +45,26 @@ module Postal
# https://datatracker.ietf.org/doc/html/rfc6376#section-3.4.2
# Split the key and value.
key, value = content.split(':', 2)
key, value = content.split(":", 2)
# Convert all header field names (not the header field values) to
# lowercase. For example, convert "SUBJect: AbC" to "subject: AbC".
key.downcase!
# Unfold all header field continuation lines as described in [RFC5322]
value.gsub!(/\r?\n[ \t]+/, ' ')
value.gsub!(/\r?\n[ \t]+/, " ")
# Convert all sequences of one or more WSP characters to a single SP character.
value.gsub!(/[ \t]+/, ' ')
value.gsub!(/[ \t]+/, " ")
# Delete all WSP characters at the end of each unfolded header field value.
value.gsub!(/[ \t]*\z/, '')
value.gsub!(/[ \t]*\z/, "")
# Delete any WSP characters remaining after the colon separating the header field name from the header field value.
value.gsub!(/\A[ \t]*/, '')
value.gsub!(/\A[ \t]*/, "")
# Join together
key + ':' + value
key + ":" + value
end
def normalized_body
@@ -77,14 +77,14 @@ module Postal
# a. Reduce whitespace
#
# * Reduce all sequences of WSP within a line to a single SP character.
content.gsub!(/[ \t]+/, ' ')
content.gsub!(/[ \t]+/, " ")
# * Ignore all whitespace at the end of lines. Implementations MUST NOT
# remove the CRLF at the end of the line.
content.gsub!(/ \r\n/, "\r\n")
# b. Ignore all empty lines at the end of the message body.
content.gsub!(/[ \r\n]*\z/, '')
content.gsub!(/[ \r\n]*\z/, "")
content += "\r\n"
content
@@ -96,7 +96,7 @@ module Postal
end
def dkim_properties
@dkim_properties ||= Array.new.tap do |header|
@dkim_properties ||= [].tap do |header|
header << "a=rsa-sha256; c=relaxed/relaxed;"
header << "d=#{@domain_name};"
header << "s=#{@dkim_identifier}; t=#{Time.now.utc.to_i};"
@@ -115,7 +115,7 @@ module Postal
end
def signature
Base64.encode64(@dkim_key.sign(OpenSSL::Digest::SHA256.new, signable_header_string)).gsub("\n", '')
Base64.encode64(@dkim_key.sign(OpenSSL::Digest.new("SHA256"), signable_header_string)).gsub("\n", "")
end
end
lib/postal/error.rb
+2
View File
@@ -1,7 +1,9 @@
module Postal
module Errors
end
class Error < StandardError
end
end
lib/postal/helpers.rb
+2 -1
View File
@@ -3,7 +3,8 @@ module Postal
def self.strip_name_from_address(address)
return nil if address.nil?
address.gsub(/.*</, '').gsub(/>.*/, '').gsub(/\(.+?\)/, '').strip
address.gsub(/.*</, "").gsub(/>.*/, "").gsub(/\(.+?\)/, "").strip
end
end
lib/postal/http.rb
+24 -25
View File
@@ -1,5 +1,5 @@
require 'net/https'
require 'uri'
require "net/https"
require "uri"
module Postal
module HTTP
@@ -16,7 +16,7 @@ module Postal
options[:headers] ||= {}
uri = URI.parse(url)
request = method.new((uri.path.length == 0 ? "/" : uri.path) + (uri.query ? "?" + uri.query : ""))
options[:headers].each { |k,v| request.add_field k, v }
options[:headers].each { |k, v| request.add_field k, v }
if options[:username] || uri.user
request.basic_auth(options[:username] || uri.user, options[:password] || uri.password)
@@ -29,7 +29,7 @@ module Postal
elsif options[:json].is_a?(String)
# If we have a JSON string, set the content type and body to be the JSON
# data
request.add_field 'Content-Type', 'application/json'
request.add_field "Content-Type", "application/json"
request.body = options[:json]
elsif options[:text_body]
@@ -38,15 +38,15 @@ module Postal
end
if options[:sign]
signature = EncryptoSigno.sign(Postal.signing_key, request.body.to_s).gsub("\n", '')
request.add_field 'X-Postal-Signature', signature
signature = EncryptoSigno.sign(Postal.signing_key, request.body.to_s).gsub("\n", "")
request.add_field "X-Postal-Signature", signature
end
request['User-Agent'] = options[:user_agent] || "Postal/#{Postal.version}"
request["User-Agent"] = options[:user_agent] || "Postal/#{Postal.version}"
connection = Net::HTTP.new(uri.host, uri.port)
if uri.scheme == 'https'
if uri.scheme == "https"
connection.use_ssl = true
connection.verify_mode = OpenSSL::SSL::VERIFY_PEER
ssl = true
@@ -59,36 +59,35 @@ module Postal
Timeout.timeout(timeout) do
result = connection.request(request)
{
:code => result.code.to_i,
:body => result.body,
:headers => result.to_hash,
:secure => @ssl
code: result.code.to_i,
body: result.body,
headers: result.to_hash,
secure: @ssl
}
end
rescue OpenSSL::SSL::SSLError => e
{
:code => -3,
:body => "Invalid SSL certificate",
:headers =>{},
:secure => @ssl
code: -3,
body: "Invalid SSL certificate",
headers: {},
secure: @ssl
}
rescue SocketError, Errno::ECONNRESET, EOFError, Errno::EINVAL, Errno::ENETUNREACH, Errno::EHOSTUNREACH, Errno::ECONNREFUSED => e
{
:code => -2,
:body => e.message,
:headers => {},
:secure => @ssl
code: -2,
body: e.message,
headers: {},
secure: @ssl
}
rescue Timeout::Error => e
{
:code => -1,
:body => "Timed out after #{timeout}s",
:headers => {},
:secure => @ssl
code: -1,
body: "Timed out after #{timeout}s",
headers: {},
secure: @ssl
}
end
end
end
end
lib/postal/http_sender.rb
+45 -45
View File
@@ -4,7 +4,7 @@ module Postal
def initialize(endpoint, options = {})
@endpoint = endpoint
@options = options
@log_id = Nifty::Utils::RandomString.generate(:length => 8).upcase
@log_id = Nifty::Utils::RandomString.generate(length: 8).upcase
end
def send_message(message)
@@ -16,10 +16,10 @@ module Postal
request_options[:sign] = true
request_options[:timeout] = @endpoint.timeout || 5
case @endpoint.encoding
when 'BodyAsJSON'
request_options[:json] = parameters(message, :flat => false).to_json
when 'FormData'
request_options[:params] = parameters(message, :flat => true)
when "BodyAsJSON"
request_options[:json] = parameters(message, flat: false).to_json
when "FormData"
request_options[:params] = parameters(message, flat: true)
end
log "Sending request to #{@endpoint.url}"
@@ -28,28 +28,28 @@ module Postal
result.details = "Received a #{response[:code]} from #{@endpoint.url}"
log " -> Received: #{response[:code]}"
if response[:body]
log " -> Body: #{response[:body][0,255]}"
log " -> Body: #{response[:body][0, 255]}"
result.output = response[:body].to_s[0, 500].strip
end
if response[:code] >= 200 && response[:code] < 300
# This is considered a success
result.type = 'Sent'
#  This is considered a success
result.type = "Sent"
elsif response[:code] >= 500 && response[:code] < 600
# This is temporary. They might fix their server so it should soft fail.
result.type = 'SoftFail'
result.type = "SoftFail"
result.retry = true
elsif response[:code] < 0
# Connection/SSL etc... errors
result.type = 'SoftFail'
result.type = "SoftFail"
result.retry = true
result.connect_error = true
elsif response[:code] == 429
# Rate limit exceeded, treat as a hard fail and don't send bounces
result.type = 'HardFail'
result.type = "HardFail"
result.suppress_bounce = true
else
# This is permanent. Any other error isn't cool with us.
result.type = 'HardFail'
result.type = "HardFail"
end
result.time = (Time.now - start_time).to_f.round(2)
result
@@ -63,29 +63,29 @@ module Postal
def parameters(message, options = {})
case @endpoint.format
when 'Hash'
when "Hash"
hash = {
:id => message.id,
:rcpt_to => message.rcpt_to,
:mail_from => message.mail_from,
:token => message.token,
:subject => message.subject,
:message_id => message.message_id,
:timestamp => message.timestamp.to_f,
:size => message.size,
:spam_status => message.spam_status,
:bounce => message.bounce == 1 ? true : false,
:received_with_ssl => message.received_with_ssl == 1,
:to => message.headers['to']&.last,
:cc => message.headers['cc']&.last,
:from => message.headers['from']&.last,
:date => message.headers['date']&.last,
:in_reply_to => message.headers['in-reply-to']&.last,
:references => message.headers['references']&.last,
:html_body => message.html_body,
:attachment_quantity => message.attachments.size,
:auto_submitted => message.headers['auto-submitted']&.last,
:reply_to => message.headers['reply-to']
id: message.id,
rcpt_to: message.rcpt_to,
mail_from: message.mail_from,
token: message.token,
subject: message.subject,
message_id: message.message_id,
timestamp: message.timestamp.to_f,
size: message.size,
spam_status: message.spam_status,
bounce: message.bounce == 1,
received_with_ssl: message.received_with_ssl == 1,
to: message.headers["to"]&.last,
cc: message.headers["cc"]&.last,
from: message.headers["from"]&.last,
date: message.headers["date"]&.last,
in_reply_to: message.headers["in-reply-to"]&.last,
references: message.headers["references"]&.last,
html_body: message.html_body,
attachment_quantity: message.attachments.size,
auto_submitted: message.headers["auto-submitted"]&.last,
reply_to: message.headers["reply-to"]
}
if @endpoint.strip_replies
@@ -105,24 +105,24 @@ module Postal
else
hash[:attachments] = message.attachments.map do |a|
{
:filename => a.filename,
:content_type => a.mime_type,
:size => a.body.to_s.bytesize,
:data => Base64.encode64(a.body.to_s)
filename: a.filename,
content_type: a.mime_type,
size: a.body.to_s.bytesize,
data: Base64.encode64(a.body.to_s)
}
end
end
end
hash
when 'RawMessage'
when "RawMessage"
{
:id => message.id,
:rcpt_to => message.rcpt_to,
:mail_from => message.mail_from,
:message => Base64.encode64(message.raw_message),
:base64 => true,
:size => message.size.to_i
id: message.id,
rcpt_to: message.rcpt_to,
mail_from: message.mail_from,
message: Base64.encode64(message.raw_message),
base64: true,
size: message.size.to_i
}
else
{}
lib/postal/job.rb
+7 -7
View File
@@ -1,16 +1,15 @@
require 'nifty/utils/random_string'
require "nifty/utils/random_string"
module Postal
class Job
def initialize(id, params = {})
@id = id
@params = params.with_indifferent_access
on_initialize
end
def id
@id
end
attr_reader :id
def params
@params || {}
@@ -33,14 +32,15 @@ module Postal
end
def self.queue(queue, params = {})
job_id = Nifty::Utils::RandomString.generate(:length => 10).upcase
job_payload = {'params' => params, 'class_name' => self.name, 'id' => job_id, 'queue' => queue}
Postal::Worker.job_queue(queue).publish(job_payload.to_json, :persistent => false)
job_id = Nifty::Utils::RandomString.generate(length: 10).upcase
job_payload = { "params" => params, "class_name" => name, "id" => job_id, "queue" => queue }
Postal::Worker.job_queue(queue).publish(job_payload.to_json, persistent: false)
job_id
end
def self.perform(params = {})
new(nil, params).perform
end
end
end
lib/postal/message_db.rb
+2
View File
@@ -1,5 +1,6 @@
module Postal
module MessageDB
extend ActiveSupport::Autoload
eager_autoload do
autoload :Click
@@ -14,5 +15,6 @@ module Postal
autoload :SuppressionList
autoload :Webhooks
end
end
end
lib/postal/message_db/click.rb
+4 -4
View File
@@ -3,10 +3,10 @@ module Postal
class Click
def initialize(attributes, link)
@url = link['url']
@ip_address = attributes['ip_address']
@user_agent = attributes['user_agent']
@timestamp = Time.zone.at(attributes['timestamp'])
@url = link["url"]
@ip_address = attributes["ip_address"]
@user_agent = attributes["user_agent"]
@timestamp = Time.zone.at(attributes["timestamp"])
end
attr_reader :ip_address
lib/postal/message_db/database.rb
+48 -48
View File
@@ -22,10 +22,10 @@ module Postal
#
def schema_version
@schema_version ||= begin
last_migration = select(:migrations, :order => :version, :direction => 'DESC', :limit => 1).first
last_migration ? last_migration['version'] : 0
last_migration = select(:migrations, order: :version, direction: "DESC", limit: 1).first
last_migration ? last_migration["version"] : 0
rescue Mysql2::Error => e
e.message =~ /doesn\'t exist/ ? 0 : raise
e.message =~ /doesn't exist/ ? 0 : raise
end
end
@@ -48,8 +48,8 @@ module Postal
end
#
# Create a new message with the given attributes. This won't be saved to the database
# until it has been 'save'd.
#  Create a new message with the given attributes. This won't be saved to the database
#  until it has been 'save'd.
#
def new_message(attributes = {})
Message.new(self, attributes)
@@ -59,39 +59,39 @@ module Postal
# Return the total size of all stored messages
#
def total_size
query("SELECT SUM(size) AS size FROM `#{database_name}`.`raw_message_sizes`").first['size'] || 0
query("SELECT SUM(size) AS size FROM `#{database_name}`.`raw_message_sizes`").first["size"] || 0
end
#
# Return the live stats instance
#  Return the live stats instance
#
def live_stats
@live_stats ||= LiveStats.new(self)
end
#
# Return the statistics instance
#  Return the statistics instance
#
def statistics
@statistics ||= Statistics.new(self)
end
#
# Return the provisioner instance
#  Return the provisioner instance
#
def provisioner
@provisioner ||= Provisioner.new(self)
end
#
# Return the provisioner instance
#  Return the provisioner instance
#
def suppression_list
@suppression_list ||= SuppressionList.new(self)
end
#
# Return the provisioner instance
#  Return the provisioner instance
#
def webhooks
@webhooks ||= Webhooks.new(self)
@@ -111,15 +111,13 @@ module Postal
table_name = raw_table_name_for_date(date)
begin
headers, body = data.split(/\r?\n\r?\n/, 2)
headers_id = insert(table_name, :data => headers)
body_id = insert(table_name, :data => body)
headers_id = insert(table_name, data: headers)
body_id = insert(table_name, data: body)
rescue Mysql2::Error => e
if e.message =~ /doesn\'t exist/
provisioner.create_raw_table(table_name)
retry
else
raise
end
raise unless e.message =~ /doesn't exist/
provisioner.create_raw_table(table_name)
retry
end
[table_name, headers_id, body_id]
end
@@ -142,17 +140,18 @@ module Postal
if options[:count]
sql_query << " COUNT(id) AS count"
elsif options[:fields]
sql_query << " " + options[:fields].map { |f| "`#{f}`" }.join(', ')
sql_query << (" " + options[:fields].map { |f| "`#{f}`" }.join(", "))
else
sql_query << " *"
end
sql_query << " FROM `#{database_name}`.`#{table}`"
if options[:where] && !options[:where].empty?
sql_query << " " + build_where_string(options[:where], ' AND ')
if options[:where].present?
sql_query << (" " + build_where_string(options[:where], " AND "))
end
if options[:order]
direction = (options[:direction] || 'ASC').upcase
raise Postal::Error, "Invalid direction #{options[:direction]}" unless ['ASC', 'DESC'].include?(direction)
direction = (options[:direction] || "ASC").upcase
raise Postal::Error, "Invalid direction #{options[:direction]}" unless ["ASC", "DESC"].include?(direction)
sql_query << " ORDER BY `#{options[:order]}` #{direction}"
end
@@ -166,14 +165,14 @@ module Postal
result = query(sql_query)
if options[:count]
result.first['count']
result.first["count"]
else
result.to_a
end
end
#
# A paginated version of select
#  A paginated version of select
#
def select_with_pagination(table, page, options = {})
page = page.to_i
@@ -183,8 +182,8 @@ module Postal
offset = (page - 1) * per_page
result = {}
result[:total] = select(table, options.merge(:count => true))
result[:records] = select(table, options.merge(:limit => per_page, :offset => offset))
result[:total] = select(table, options.merge(count: true))
result[:records] = select(table, options.merge(limit: per_page, offset: offset))
result[:per_page] = per_page
result[:total_pages], remainder = result[:total].divmod(per_page)
result[:total_pages] += 1 if remainder > 0
@@ -204,7 +203,7 @@ module Postal
sql_query = "UPDATE `#{database_name}`.`#{table}` SET"
sql_query << " #{hash_to_sql(attributes)}"
if options[:where]
sql_query << " " + build_where_string(options[:where])
sql_query << (" " + build_where_string(options[:where]))
end
with_mysql do |mysql|
query_on_connection(mysql, sql_query)
@@ -218,8 +217,8 @@ module Postal
#
def insert(table, attributes)
sql_query = "INSERT INTO `#{database_name}`.`#{table}`"
sql_query << " (" + attributes.keys.map { |k| "`#{k}`" }.join(', ') + ")"
sql_query << " VALUES (" + attributes.values.map { |v| escape(v) }.join(', ') + ")"
sql_query << (" (" + attributes.keys.map { |k| "`#{k}`" }.join(", ") + ")")
sql_query << (" VALUES (" + attributes.values.map { |v| escape(v) }.join(", ") + ")")
with_mysql do |mysql|
query_on_connection(mysql, sql_query)
mysql.last_id
@@ -234,16 +233,16 @@ module Postal
nil
else
sql_query = "INSERT INTO `#{database_name}`.`#{table}`"
sql_query << " (" + keys.map { |k| "`#{k}`" }.join(', ') + ")"
sql_query << (" (" + keys.map { |k| "`#{k}`" }.join(", ") + ")")
sql_query << " VALUES "
sql_query << values.map { |v| "(" + v.map { |v| escape(v) }.join(', ') + ")" }.join(', ')
sql_query << values.map { |v| "(" + v.map { |v| escape(v) }.join(", ") + ")" }.join(", ")
query(sql_query)
end
end
#
# Deletes a in the database. Accepts a table name, and some options which
# are shown below:
#  are shown below:
#
# :where => The condition to apply to the query
#
@@ -251,7 +250,7 @@ module Postal
#
def delete(table, options = {})
sql_query = "DELETE FROM `#{database_name}`.`#{table}`"
sql_query << " " + build_where_string(options[:where], ' AND ')
sql_query << (" " + build_where_string(options[:where], " AND "))
with_mysql do |mysql|
query_on_connection(mysql, sql_query)
mysql.affected_rows
@@ -259,7 +258,7 @@ module Postal
end
#
# Return the correct database name
#  Return the correct database name
#
def database_name
@database_name ||= "#{Postal.config.message_db.prefix}-server-#{@server_id}"
@@ -269,8 +268,10 @@ module Postal
# Run a query, log it and return the result
#
class ResultForExplainPrinter
attr_reader :columns
attr_reader :rows
def initialize(result)
if result.first
@columns = result.first.keys
@@ -280,6 +281,7 @@ module Postal
@rows = []
end
end
end
def stringify_keys(hash)
@@ -291,17 +293,15 @@ module Postal
def escape(value)
with_mysql do |mysql|
if value == true
'1'
"1"
elsif value == false
'0'
"0"
elsif value.nil?
'NULL'
"NULL"
elsif value.to_s.length == 0
"NULL"
else
if value.to_s.length == 0
'NULL'
else
"'" + mysql.escape(value.to_s) + "'"
end
"'" + mysql.escape(value.to_s) + "'"
end
end
end
@@ -320,7 +320,7 @@ module Postal
time = Time.now.to_f - start_time
logger.debug " \e[4;34mMessageDB Query (#{time.round(2)}s) \e[0m \e[33m#{query}\e[0m"
if time > 0.5 && query =~ /\A(SELECT|UPDATE|DELETE) /
id = Nifty::Utils::RandomString.generate(:length => 6).upcase
id = Nifty::Utils::RandomString.generate(length: 6).upcase
explain_result = ResultForExplainPrinter.new(connection.query("EXPLAIN #{query}"))
slow_query_logger.info "[#{id}] EXPLAIN #{query}"
for line in ActiveRecord::ConnectionAdapters::MySQL::ExplainPrettyPrinter.new.pp(explain_result, time).split("\n")
@@ -342,16 +342,16 @@ module Postal
MessageDB::MySQL.client(&block)
end
def build_where_string(attributes, joiner = ', ')
def build_where_string(attributes, joiner = ", ")
"WHERE #{hash_to_sql(attributes, joiner)}"
end
def hash_to_sql(hash, joiner = ', ')
def hash_to_sql(hash, joiner = ", ")
hash.map do |key, value|
if value.is_a?(Array) && value.all? { |v| v.is_a?(Integer) }
"`#{key}` IN (#{value.join(', ')})"
elsif value.is_a?(Array)
escaped_values = value.map { |v| escape(v) }.join(', ')
escaped_values = value.map { |v| escape(v) }.join(", ")
"`#{key}` IN (#{escaped_values})"
elsif value.is_a?(Hash)
sql = []
lib/postal/message_db/delivery.rb
+28 -30
View File
@@ -4,9 +4,9 @@ module Postal
def self.create(message, attributes = {})
attributes = message.database.stringify_keys(attributes)
attributes = attributes.merge('message_id' => message.id, 'timestamp' => Time.now.to_f)
id = message.database.insert('deliveries', attributes)
delivery = Delivery.new(message, attributes.merge('id' => id))
attributes = attributes.merge("message_id" => message.id, "timestamp" => Time.now.to_f)
id = message.database.insert("deliveries", attributes)
delivery = Delivery.new(message, attributes.merge("id" => id))
delivery.update_statistics
delivery.send_webhooks
delivery
@@ -18,52 +18,50 @@ module Postal
end
def method_missing(name, value = nil, &block)
if @attributes.has_key?(name.to_s)
@attributes[name.to_s]
else
nil
end
return unless @attributes.has_key?(name.to_s)
@attributes[name.to_s]
end
def timestamp
@timestamp ||= @attributes['timestamp'] ? Time.zone.at(@attributes['timestamp']) : nil
@timestamp ||= @attributes["timestamp"] ? Time.zone.at(@attributes["timestamp"]) : nil
end
def update_statistics
if self.status == 'Held'
@message.database.statistics.increment_all(self.timestamp, 'held')
if status == "Held"
@message.database.statistics.increment_all(timestamp, "held")
end
if self.status == 'Bounced' || self.status == 'HardFail'
@message.database.statistics.increment_all(self.timestamp, 'bounces')
end
return unless status == "Bounced" || status == "HardFail"
@message.database.statistics.increment_all(timestamp, "bounces")
end
def send_webhooks
if self.webhook_event
WebhookRequest.trigger(@message.database.server_id, self.webhook_event, self.webhook_hash)
end
return unless webhook_event
WebhookRequest.trigger(@message.database.server_id, webhook_event, webhook_hash)
end
def webhook_hash
{
:message => @message.webhook_hash,
:status => self.status,
:details => self.details,
:output => self.output.to_s.force_encoding('UTF-8').scrub.truncate(512),
:sent_with_ssl => self.sent_with_ssl,
:timestamp => @attributes['timestamp'],
:time => self.time
message: @message.webhook_hash,
status: status,
details: details,
output: output.to_s.force_encoding("UTF-8").scrub.truncate(512),
sent_with_ssl: sent_with_ssl,
timestamp: @attributes["timestamp"],
time: time
}
end
def webhook_event
@webhook_event ||= case self.status
when 'Sent' then 'MessageSent'
when 'SoftFail' then 'MessageDelayed'
when 'HardFail' then 'MessageDeliveryFailed'
when 'Held' then 'MessageHeld'
end
@webhook_event ||= case status
when "Sent" then "MessageSent"
when "SoftFail" then "MessageDelayed"
when "HardFail" then "MessageDeliveryFailed"
when "Held" then "MessageHeld"
end
end
end
lib/postal/message_db/live_stats.rb
+7 -8
View File
@@ -25,15 +25,14 @@ module Postal
if minutes > 60
raise Postal::Error, "Live stats can only return data for the last 60 minutes."
end
options[:types] ||= [:incoming, :outgoing]
if options[:types].empty?
raise Postal::Error, "You must provide at least one type to return"
else
time = minutes.minutes.ago.beginning_of_minute.utc.to_f
types = options[:types].map {|t| "#{@database.escape(t.to_s)}"}.join(', ')
result = @database.query("SELECT SUM(count) as count FROM `#{@database.database_name}`.`live_stats` WHERE `type` IN (#{types}) AND timestamp > #{time}").first
result['count'] || 0
end
raise Postal::Error, "You must provide at least one type to return" if options[:types].empty?
time = minutes.minutes.ago.beginning_of_minute.utc.to_f
types = options[:types].map { |t| "#{@database.escape(t.to_s)}" }.join(", ")
result = @database.query("SELECT SUM(count) as count FROM `#{@database.database_name}`.`live_stats` WHERE `type` IN (#{types}) AND timestamp > #{time}").first
result["count"] || 0
end
end
lib/postal/message_db/load.rb
+3 -3
View File
@@ -3,9 +3,9 @@ module Postal
class Load
def initialize(attributes)
@ip_address = attributes['ip_address']
@user_agent = attributes['user_agent']
@timestamp = Time.zone.at(attributes['timestamp'])
@ip_address = attributes["ip_address"]
@user_agent = attributes["user_agent"]
@timestamp = Time.zone.at(attributes["timestamp"])
end
attr_reader :ip_address
lib/postal/message_db/message.rb
+134 -140
View File
@@ -6,16 +6,14 @@ module Postal
end
def self.find_one(database, query)
query = {:id => query.to_i} if query.is_a?(Integer)
if message = database.select('messages', :where => query, :limit => 1).first
Message.new(database, message)
else
raise NotFound, "No message found matching provided query #{query}"
end
query = { id: query.to_i } if query.is_a?(Integer)
raise NotFound, "No message found matching provided query #{query}" unless message = database.select("messages", where: query, limit: 1).first
Message.new(database, message)
end
def self.find(database, options = {})
if messages = database.select('messages', options)
if messages = database.select("messages", options)
if messages.is_a?(Array)
messages.map { |m| Message.new(database, m) }
else
@@ -27,7 +25,7 @@ module Postal
end
def self.find_with_pagination(database, page, options = {})
messages = database.select_with_pagination('messages', page, options)
messages = database.select_with_pagination("messages", page, options)
messages[:records] = messages[:records].map { |m| Message.new(database, m) }
messages
end
@@ -50,26 +48,24 @@ module Postal
# Return the credential for this message
#
def credential
@credential ||= self.credential_id ? Credential.find_by_id(self.credential_id) : nil
@credential ||= credential_id ? Credential.find_by_id(credential_id) : nil
end
#
# Return the route for this message
#
def route
@route ||= self.route_id ? Route.find_by_id(self.route_id) : nil
@route ||= route_id ? Route.find_by_id(route_id) : nil
end
#
# Return the endpoint for this message
#
def endpoint
@endpoint ||= begin
if self.endpoint_type && self.endpoint_id
self.endpoint_type.constantize.find_by_id(self.endpoint_id)
elsif self.route && self.route.mode == 'Endpoint'
self.route.endpoint
end
if endpoint_type && endpoint_id
@endpoint ||= endpoint_type.constantize.find_by_id(endpoint_id)
elsif route && route.mode == "Endpoint"
@endpoint ||= route.endpoint
end
end
@@ -77,41 +73,41 @@ module Postal
# Return the credential for this message
#
def domain
@domain ||= self.domain_id ? Domain.find_by_id(self.domain_id) : nil
@domain ||= domain_id ? Domain.find_by_id(domain_id) : nil
end
#
# Copy appropriate attributes from the raw message to the message itself
#
def copy_attributes_from_raw_message
if self.raw_message
self.subject = self.headers['subject']&.last.to_s[0,200]
self.message_id = self.headers['message-id']&.last
if self.message_id
self.message_id = self.message_id.gsub(/.*</, '').gsub(/>.*/, '').strip
end
end
return unless raw_message
self.subject = headers["subject"]&.last.to_s[0, 200]
self.message_id = headers["message-id"]&.last
return unless message_id
self.message_id = message_id.gsub(/.*</, "").gsub(/>.*/, "").strip
end
#
# Return the timestamp for this message
#
def timestamp
@timestamp ||= @attributes['timestamp'] ? Time.zone.at(@attributes['timestamp']) : nil
@timestamp ||= @attributes["timestamp"] ? Time.zone.at(@attributes["timestamp"]) : nil
end
#
# Return the time that the last delivery was attempted
#  Return the time that the last delivery was attempted
#
def last_delivery_attempt
@last_delivery_attempt ||= @attributes['last_delivery_attempt'] ? Time.zone.at(@attributes['last_delivery_attempt']) : nil
@last_delivery_attempt ||= @attributes["last_delivery_attempt"] ? Time.zone.at(@attributes["last_delivery_attempt"]) : nil
end
#
# Return the hold expiry for this message
#
def hold_expiry
@hold_expiry ||= @attributes['hold_expiry'] ? Time.zone.at(@attributes['hold_expiry']) : nil
@hold_expiry ||= @attributes["hold_expiry"] ? Time.zone.at(@attributes["hold_expiry"]) : nil
end
#
@@ -125,9 +121,9 @@ module Postal
# Add a delivery attempt for this message
#
def create_delivery(status, options = {})
delivery = Delivery.create(self, options.merge(:status => status))
hold_expiry = status == 'Held' ? Postal.config.general.maximum_hold_expiry_days.days.from_now.to_f : nil
self.update(:status => status, :last_delivery_attempt => delivery.timestamp.to_f, :held => status == 'Held' ? 1 : 0, :hold_expiry => hold_expiry)
delivery = Delivery.create(self, options.merge(status: status))
hold_expiry = status == "Held" ? Postal.config.general.maximum_hold_expiry_days.days.from_now.to_f : nil
update(status: status, last_delivery_attempt: delivery.timestamp.to_f, held: status == "Held" ? 1 : 0, hold_expiry: hold_expiry)
delivery
end
@@ -135,10 +131,8 @@ module Postal
# Return all deliveries for this object
#
def deliveries
@deliveries ||= begin
@database.select('deliveries', :where => {:message_id => self.id}, :order => :timestamp).map do |hash|
Delivery.new(self, hash)
end
@deliveries ||= @database.select("deliveries", where: { message_id: id }, order: :timestamp).map do |hash|
Delivery.new(self, hash)
end
end
@@ -147,13 +141,13 @@ module Postal
#
def clicks
@clicks ||= begin
clicks = @database.select('clicks', :where => {:message_id => self.id}, :order => :timestamp)
clicks = @database.select("clicks", where: { message_id: id }, order: :timestamp)
if clicks.empty?
[]
else
links = @database.select('links', :where => {:id => clicks.map { |c| c['link_id'].to_i }}).group_by { |l| l['id'] }
links = @database.select("links", where: { id: clicks.map { |c| c["link_id"].to_i } }).group_by { |l| l["id"] }
clicks.map do |hash|
Click.new(hash, links[hash['link_id']].first)
Click.new(hash, links[hash["link_id"]].first)
end
end
end
@@ -164,7 +158,7 @@ module Postal
#
def loads
@loads ||= begin
loads = @database.select('loads', :where => {:message_id => self.id}, :order => :timestamp)
loads = @database.select("loads", where: { message_id: id }, order: :timestamp)
loads.map do |hash|
Load.new(hash)
end
@@ -172,22 +166,20 @@ module Postal
end
#
# Return all activity entries
#  Return all activity entries
#
def activity_entries
@activity_entries ||= (deliveries + clicks + loads).sort_by(&:timestamp)
end
#
# Provide access to set and get acceptable attributes
#  Provide access to set and get acceptable attributes
#
def method_missing(name, value = nil, &block)
if @attributes.has_key?(name.to_s)
@attributes[name.to_s]
elsif name.to_s =~ /\=\z/
@attributes[name.to_s.gsub('=', '').to_s] = value
else
nil
elsif name.to_s =~ /=\z/
@attributes[name.to_s.gsub("=", "").to_s] = value
end
end
@@ -195,11 +187,11 @@ module Postal
# Has this message been persisted to the database yet?
#
def persisted?
!@attributes['id'].nil?
!@attributes["id"].nil?
end
#
# Save this message
#  Save this message
#
def save
save_raw_message
@@ -213,38 +205,38 @@ module Postal
def update(attributes_to_change)
@attributes = @attributes.merge(database.stringify_keys(attributes_to_change))
if persisted?
@database.update('messages', attributes_to_change, :where => {:id => self.id})
@database.update("messages", attributes_to_change, where: { id: id })
else
_create
end
end
#
# Delete the message from the database
#  Delete the message from the database
#
def delete
if persisted?
@database.delete('messages', :where => {:id => self.id})
end
return unless persisted?
@database.delete("messages", where: { id: id })
end
#
# Return the headers
#  Return the headers
#
def raw_headers
if self.raw_table
@raw_headers ||= @database.select(self.raw_table, :where => {:id => self.raw_headers_id}).first&.send(:[], 'data') || ""
if raw_table
@raw_headers ||= @database.select(raw_table, where: { id: raw_headers_id }).first&.send(:[], "data") || ""
else
""
end
end
#
# Return the full raw message body for this message.
#  Return the full raw message body for this message.
#
def raw_body
if self.raw_table
@raw ||= @database.select(self.raw_table, :where => {:id => self.raw_body_id}).first&.send(:[], 'data') || ""
if raw_table
@raw ||= @database.select(raw_table, where: { id: raw_body_id }).first&.send(:[], "data") || ""
else
""
end
@@ -261,35 +253,35 @@ module Postal
# Set the raw message ready for saving later
#
def raw_message=(raw)
@pending_raw_message = raw.force_encoding('BINARY')
@pending_raw_message = raw.force_encoding("BINARY")
end
#
# Save the raw message to the database as appropriate
#
def save_raw_message
if @pending_raw_message
self.size = @pending_raw_message.bytesize
date = Date.today
table_name, headers_id, body_id = @database.insert_raw_message(@pending_raw_message, date)
self.raw_table = table_name
self.raw_headers_id = headers_id
self.raw_body_id = body_id
@raw = nil
@raw_headers = nil
@headers = nil
@mail = nil
@pending_raw_message = nil
copy_attributes_from_raw_message
@database.query("UPDATE `#{@database.database_name}`.`raw_message_sizes` SET size = size + #{self.size} WHERE table_name = '#{table_name}'")
end
return unless @pending_raw_message
self.size = @pending_raw_message.bytesize
date = Date.today
table_name, headers_id, body_id = @database.insert_raw_message(@pending_raw_message, date)
self.raw_table = table_name
self.raw_headers_id = headers_id
self.raw_body_id = body_id
@raw = nil
@raw_headers = nil
@headers = nil
@mail = nil
@pending_raw_message = nil
copy_attributes_from_raw_message
@database.query("UPDATE `#{@database.database_name}`.`raw_message_sizes` SET size = size + #{size} WHERE table_name = '#{table_name}'")
end
#
# Is there a raw message?
#
def raw_message?
!!self.raw_table
!!raw_table
end
#
@@ -300,7 +292,7 @@ module Postal
end
#
# Return the HTML body for this message
#  Return the HTML body for this message
#
def html_body
mail&.html_body
@@ -310,7 +302,7 @@ module Postal
# Return the HTML body with any tracking links
#
def html_body_without_tracking_image
html_body.gsub(/\<p class\=['"]ampimg['"].*?\<\/p\>/, '')
html_body.gsub(/<p class=['"]ampimg['"].*?<\/p>/, "")
end
#
@@ -325,7 +317,7 @@ module Postal
#
def headers
@headers ||= begin
mail = Mail.new(self.raw_headers)
mail = Mail.new(raw_headers)
mail.header.fields.each_with_object({}) do |field, hash|
hash[field.name.downcase] ||= []
begin
@@ -341,27 +333,27 @@ module Postal
# Return the recipient domain for this message
#
def recipient_domain
self.rcpt_to ? self.rcpt_to.split('@').last : nil
rcpt_to ? rcpt_to.split("@").last : nil
end
#
# Create a new item in the message queue for this message
#
def add_to_message_queue(options = {})
QueuedMessage.create!(:message => self, :server_id => @database.server_id, :batch_key => self.batch_key, :domain => self.recipient_domain, :route_id => self.route_id, :manual => options[:manual]).id
QueuedMessage.create!(message: self, server_id: @database.server_id, batch_key: batch_key, domain: recipient_domain, route_id: route_id, manual: options[:manual]).id
end
#
# Return a suitable batch key for this message
#
def batch_key
case self.scope
when 'outgoing'
case scope
when "outgoing"
key = "outgoing-"
key += self.recipient_domain.to_s
when 'incoming'
key += recipient_domain.to_s
when "incoming"
key = "incoming-"
key += "rt:#{self.route_id}-ep:#{self.endpoint_id}-#{self.endpoint_type}"
key += "rt:#{route_id}-ep:#{endpoint_id}-#{endpoint_type}"
else
key = nil
end
@@ -372,29 +364,30 @@ module Postal
# Return the queued message
#
def queued_message
@queued_message ||= self.id ? QueuedMessage.where(:message_id => self.id, :server_id => @database.server_id).first : nil
@queued_message ||= id ? QueuedMessage.where(message_id: id, server_id: @database.server_id).first : nil
end
#
# Return the spam status
#
def spam_status
return 'NotChecked' unless inspected == 1
spam == 1 ? 'Spam' : 'NotSpam'
return "NotChecked" unless inspected == 1
spam == 1 ? "Spam" : "NotSpam"
end
#
# Has this message been held?
#
def held?
status == 'Held'
status == "Held"
end
#
# Does this message have our DKIM header yet?
#
def has_outgoing_headers?
!!(raw_headers =~ /^X\-Postal\-MsgID\:/i)
!!(raw_headers =~ /^X-Postal-MsgID:/i)
end
#
@@ -402,11 +395,11 @@ module Postal
#
def add_outgoing_headers
headers = []
if self.domain
dkim = Postal::DKIMHeader.new(self.domain, self.raw_message)
if domain
dkim = Postal::DKIMHeader.new(domain, raw_message)
headers << dkim.dkim_header
end
headers << "X-Postal-MsgID: #{self.token}"
headers << "X-Postal-MsgID: #{token}"
append_headers(*headers)
end
@@ -415,8 +408,8 @@ module Postal
#
def append_headers(*headers)
new_headers = headers.join("\r\n")
new_headers = "#{new_headers}\r\n#{self.raw_headers}"
@database.update(self.raw_table, {:data => new_headers}, :where => {:id => self.raw_headers_id})
new_headers = "#{new_headers}\r\n#{raw_headers}"
@database.update(raw_table, { data: new_headers }, where: { id: raw_headers_id })
@raw_headers = new_headers
@raw_message = nil
@headers = nil
@@ -427,16 +420,16 @@ module Postal
#
def webhook_hash
@webhook_hash ||= {
:id => self.id,
:token => self.token,
:direction => self.scope,
:message_id => self.message_id,
:to => self.rcpt_to,
:from => self.mail_from,
:subject => self.subject,
:timestamp => self.timestamp.to_f,
:spam_status => self.spam_status,
:tag => self.tag
id: id,
token: token,
direction: scope,
message_id: message_id,
to: rcpt_to,
from: mail_from,
subject: subject,
timestamp: timestamp.to_f,
spam_status: spam_status,
tag: tag
}
end
@@ -444,46 +437,47 @@ module Postal
# Mark this message as bounced
#
def bounce!(bounce_message)
create_delivery('Bounced', :details => "We've received a bounce message for this e-mail. See <msg:#{bounce_message.id}> for details.")
SendWebhookJob.queue(:main, :server_id => self.database.server_id, :event => "MessageBounced", :payload => {:_original_message => self.id, :_bounce => bounce_message.id})
create_delivery("Bounced", details: "We've received a bounce message for this e-mail. See <msg:#{bounce_message.id}> for details.")
SendWebhookJob.queue(:main, server_id: database.server_id, event: "MessageBounced", payload: { _original_message: id, _bounce: bounce_message.id })
end
#
# Should bounces be sent for this message?
#
def send_bounces?
self.bounce != 1 && self.mail_from.present?
bounce != 1 && mail_from.present?
end
#
# Add a load for this message
#
def create_load(request)
update('loaded' => Time.now.to_f) if loaded.nil?
database.insert(:loads, {:message_id => self.id, :ip_address => request.ip, :user_agent => request.user_agent, :timestamp => Time.now.to_f})
SendWebhookJob.queue(:main, :server_id => self.database.server_id, :event => 'MessageLoaded', :payload => {:_message => self.id, :ip_address => request.ip, :user_agent => request.user_agent})
update("loaded" => Time.now.to_f) if loaded.nil?
database.insert(:loads, { message_id: id, ip_address: request.ip, user_agent: request.user_agent, timestamp: Time.now.to_f })
SendWebhookJob.queue(:main, server_id: database.server_id, event: "MessageLoaded", payload: { _message: id, ip_address: request.ip, user_agent: request.user_agent })
end
#
# Create a new link
#  Create a new link
#
def create_link(url)
hash = Digest::SHA1.hexdigest(url.to_s)
token = Nifty::Utils::RandomString.generate(:length => 8)
database.insert(:links, {:message_id => self.id, :hash => hash, :url => url, :timestamp => Time.now.to_f, :token => token})
token = Nifty::Utils::RandomString.generate(length: 8)
database.insert(:links, { message_id: id, hash: hash, url: url, timestamp: Time.now.to_f, token: token })
token
end
#
# Return a message object that this message is a reply to
#  Return a message object that this message is a reply to
#
def original_messages
return nil unless self.bounce == 1
other_message_ids = raw_message.scan(/\X\-Postal\-MsgID\:\s*([a-z0-9]+)/i).flatten
return nil unless bounce == 1
other_message_ids = raw_message.scan(/\X-Postal-MsgID:\s*([a-z0-9]+)/i).flatten
if other_message_ids.empty?
[]
else
database.messages(:where => {:token => other_message_ids})
database.messages(where: { token: other_message_ids })
end
end
@@ -491,39 +485,39 @@ module Postal
# Was thsi message sent to a return path?
#
def rcpt_to_return_path?
!!(rcpt_to =~ /\@#{Regexp.escape(Postal.config.dns.custom_return_path_prefix)}\./)
!!(rcpt_to =~ /@#{Regexp.escape(Postal.config.dns.custom_return_path_prefix)}\./)
end
#
# Inspect this message
#
def inspect_message
result = MessageInspection.scan(self, self.scope&.to_sym)
result = MessageInspection.scan(self, scope&.to_sym)
# Update the messages table with the results of our inspection
update(:inspected => 1, :spam_score => result.spam_score, :threat => result.threat?, :threat_details => result.threat_message)
update(inspected: 1, spam_score: result.spam_score, threat: result.threat?, threat_details: result.threat_message)
# Add any spam details into the spam checks database
self.database.insert_multi(:spam_checks, [:message_id, :code, :score, :description], result.spam_checks.map { |d| [self.id, d.code, d.score, d.description] })
database.insert_multi(:spam_checks, [:message_id, :code, :score, :description], result.spam_checks.map { |d| [id, d.code, d.score, d.description] })
# Return the result
result
end
#
# Return all spam checks for this message
#  Return all spam checks for this message
#
def spam_checks
@spam_checks ||= self.database.select(:spam_checks, :where => {:message_id => self.id})
@spam_checks ||= database.select(:spam_checks, where: { message_id: id })
end
#
# Cancel the hold on this message
#
def cancel_hold
if self.status == 'Held'
create_delivery('HoldCancelled', :details => "The hold on this message has been removed without action.")
end
return unless status == "Held"
create_delivery("HoldCancelled", details: "The hold on this message has been removed without action.")
end
#
@@ -532,43 +526,43 @@ module Postal
def parse_content
parse_result = Postal::MessageParser.new(self)
if parse_result.actioned?
# Somethign was changed, update the raw message
@database.update(self.raw_table, {:data => parse_result.new_body}, :where => {:id => self.raw_body_id})
#  Somethign was changed, update the raw message
@database.update(raw_table, { data: parse_result.new_body }, where: { id: raw_body_id })
@raw = parse_result.new_body
@raw_message = nil
end
update('parsed' => 1, 'tracked_links' => parse_result.tracked_links, 'tracked_images' => parse_result.tracked_images)
update("parsed" => 1, "tracked_links" => parse_result.tracked_links, "tracked_images" => parse_result.tracked_images)
end
#
# Has this message been parsed?
#
def parsed?
self.parsed == 1
parsed == 1
end
#
# Should this message be parsed?
#
def should_parse?
parsed? == false && headers['x-amp'] != 'skip'
parsed? == false && headers["x-amp"] != "skip"
end
private
def _update
@database.update('messages', @attributes.reject {|k,v| k == :id }, :where => {:id => @attributes['id']})
@database.update("messages", @attributes.reject { |k, v| k == :id }, where: { id: @attributes["id"] })
end
def _create
self.timestamp = Time.now.to_f if self.timestamp.blank?
self.status = 'Pending' if self.status.blank?
self.token = Nifty::Utils::RandomString.generate(:length => 12) if self.token.blank?
last_id = @database.insert('messages', @attributes.reject {|k,v| k == :id })
@attributes['id'] = last_id
@database.statistics.increment_all(self.timestamp, self.scope)
self.timestamp = Time.now.to_f if timestamp.blank?
self.status = "Pending" if status.blank?
self.token = Nifty::Utils::RandomString.generate(length: 12) if token.blank?
last_id = @database.insert("messages", @attributes.reject { |k, v| k == :id })
@attributes["id"] = last_id
@database.statistics.increment_all(timestamp, scope)
Statistic.global.increment!(:total_messages)
Statistic.global.increment!("total_#{self.scope}".to_sym)
Statistic.global.increment!("total_#{scope}".to_sym)
add_to_message_queue
end
lib/postal/message_db/migration.rb
+8 -4
View File
@@ -10,8 +10,11 @@ module Postal
end
def self.run(database, start_from = database.schema_version)
files = Dir[Rails.root.join('lib', 'postal', 'message_db', 'migrations', '*.rb')]
files = files.map { |f| id, name = f.split('/').last.split('_', 2); [id.to_i, name] }.sort_by(&:first)
files = Dir[Rails.root.join("lib", "postal", "message_db", "migrations", "*.rb")]
files = files.map do |f|
id, name = f.split("/").last.split("_", 2)
[id.to_i, name]
end.sort_by(&:first)
latest_version = files.last.first
if latest_version > start_from
puts "\e[32mMigrating #{database.database_name} from version #{start_from} => #{files.last.first}\e[0m"
@@ -19,14 +22,15 @@ module Postal
puts "Nothing to do."
end
files.each do |version, file|
klass_name = file.gsub(/\.rb\z/, '').camelize
klass_name = file.gsub(/\.rb\z/, "").camelize
next if start_from >= version
puts "\e[45m++ Migrating #{klass_name} (#{version})\e[0m"
require "postal/message_db/migrations/#{version.to_s.rjust(2, '0')}_#{file}"
klass = Postal::MessageDB::Migrations.const_get(klass_name)
instance = klass.new(database)
instance.up
database.insert(:migrations, :version => version)
database.insert(:migrations, version: version)
end
end
lib/postal/message_db/migrations/01_create_migrations.rb
+6 -5
View File
@@ -2,14 +2,15 @@ module Postal
module MessageDB
module Migrations
class CreateMigrations < Postal::MessageDB::Migration
def up
@database.provisioner.create_table(:migrations,
:columns => {
:version => 'int(11) NOT NULL',
},
:primary_key => '`version`'
)
columns: {
version: "int(11) NOT NULL"
},
primary_key: "`version`")
end
end
end
end
lib/postal/message_db/migrations/02_create_messages.rb
+47 -46
View File
@@ -2,55 +2,56 @@ module Postal
module MessageDB
module Migrations
class CreateMessages < Postal::MessageDB::Migration
def up
@database.provisioner.create_table(:messages,
:columns => {
:id => 'int(11) NOT NULL AUTO_INCREMENT',
:token => 'varchar(255) DEFAULT NULL',
:scope => 'varchar(10) DEFAULT NULL',
:rcpt_to => 'varchar(255) DEFAULT NULL',
:mail_from => 'varchar(255) DEFAULT NULL',
:subject => 'varchar(255) DEFAULT NULL',
:message_id => 'varchar(255) DEFAULT NULL',
:timestamp => 'decimal(18,6) DEFAULT NULL',
:route_id => 'int(11) DEFAULT NULL',
:domain_id => 'int(11) DEFAULT NULL',
:credential_id => 'int(11) DEFAULT NULL',
:status => 'varchar(255) DEFAULT NULL',
:held => 'tinyint(1) DEFAULT 0',
:size => 'varchar(255) DEFAULT NULL',
:last_delivery_attempt => 'decimal(18,6) DEFAULT NULL',
:raw_table => 'varchar(255) DEFAULT NULL',
:raw_body_id => 'int(11) DEFAULT NULL',
:raw_headers_id => 'int(11) DEFAULT NULL',
:inspected => 'tinyint(1) DEFAULT 0',
:spam => 'tinyint(1) DEFAULT 0',
:spam_score => 'decimal(8,2) DEFAULT 0',
:threat => 'tinyint(1) DEFAULT 0',
:threat_details => 'varchar(255) DEFAULT NULL',
:bounce => 'tinyint(1) DEFAULT 0',
:bounce_for_id => 'int(11) DEFAULT 0',
:tag => 'varchar(255) DEFAULT NULL',
:loaded => 'decimal(18,6) DEFAULT NULL',
:clicked => 'decimal(18,6) DEFAULT NULL',
:received_with_ssl => 'tinyint(1) DEFAULT NULL',
},
:indexes => {
:on_message_id => '`message_id`(8)',
:on_token => '`token`(6)',
:on_bounce_for_id => '`bounce_for_id`',
:on_held => '`held`',
:on_scope_and_status => '`scope`(1), `spam`, `status`(6), `timestamp`',
:on_scope_and_tag => '`scope`(1), `spam`, `tag`(8), `timestamp`',
:on_scope_and_spam => '`scope`(1), `spam`, `timestamp`',
:on_scope_and_thr_status => '`scope`(1), `threat`, `status`(6), `timestamp`',
:on_scope_and_threat => '`scope`(1), `threat`, `timestamp`',
:on_rcpt_to => '`rcpt_to`(12), `timestamp`',
:on_mail_from => '`mail_from`(12), `timestamp`',
:on_raw_table => '`raw_table`(14)',
}
)
columns: {
id: "int(11) NOT NULL AUTO_INCREMENT",
token: "varchar(255) DEFAULT NULL",
scope: "varchar(10) DEFAULT NULL",
rcpt_to: "varchar(255) DEFAULT NULL",
mail_from: "varchar(255) DEFAULT NULL",
subject: "varchar(255) DEFAULT NULL",
message_id: "varchar(255) DEFAULT NULL",
timestamp: "decimal(18,6) DEFAULT NULL",
route_id: "int(11) DEFAULT NULL",
domain_id: "int(11) DEFAULT NULL",
credential_id: "int(11) DEFAULT NULL",
status: "varchar(255) DEFAULT NULL",
held: "tinyint(1) DEFAULT 0",
size: "varchar(255) DEFAULT NULL",
last_delivery_attempt: "decimal(18,6) DEFAULT NULL",
raw_table: "varchar(255) DEFAULT NULL",
raw_body_id: "int(11) DEFAULT NULL",
raw_headers_id: "int(11) DEFAULT NULL",
inspected: "tinyint(1) DEFAULT 0",
spam: "tinyint(1) DEFAULT 0",
spam_score: "decimal(8,2) DEFAULT 0",
threat: "tinyint(1) DEFAULT 0",
threat_details: "varchar(255) DEFAULT NULL",
bounce: "tinyint(1) DEFAULT 0",
bounce_for_id: "int(11) DEFAULT 0",
tag: "varchar(255) DEFAULT NULL",
loaded: "decimal(18,6) DEFAULT NULL",
clicked: "decimal(18,6) DEFAULT NULL",
received_with_ssl: "tinyint(1) DEFAULT NULL"
},
indexes: {
on_message_id: "`message_id`(8)",
on_token: "`token`(6)",
on_bounce_for_id: "`bounce_for_id`",
on_held: "`held`",
on_scope_and_status: "`scope`(1), `spam`, `status`(6), `timestamp`",
on_scope_and_tag: "`scope`(1), `spam`, `tag`(8), `timestamp`",
on_scope_and_spam: "`scope`(1), `spam`, `timestamp`",
on_scope_and_thr_status: "`scope`(1), `threat`, `status`(6), `timestamp`",
on_scope_and_threat: "`scope`(1), `threat`, `timestamp`",
on_rcpt_to: "`rcpt_to`(12), `timestamp`",
on_mail_from: "`mail_from`(12), `timestamp`",
on_raw_table: "`raw_table`(14)"
})
end
end
end
end
lib/postal/message_db/migrations/03_create_deliveries.rb
+16 -15
View File
@@ -2,24 +2,25 @@ module Postal
module MessageDB
module Migrations
class CreateDeliveries < Postal::MessageDB::Migration
def up
@database.provisioner.create_table(:deliveries,
:columns => {
:id => 'int(11) NOT NULL AUTO_INCREMENT',
:message_id => 'int(11) DEFAULT NULL',
:status => 'varchar(255) DEFAULT NULL',
:code => 'int(11) DEFAULT NULL',
:output => 'varchar(512) DEFAULT NULL',
:details => 'varchar(512) DEFAULT NULL',
:sent_with_ssl => 'tinyint(1) DEFAULT 0',
:log_id => 'varchar(100) DEFAULT NULL',
:timestamp => 'decimal(18,6) DEFAULT NULL'
},
:indexes => {
:on_message_id => '`message_id`'
}
)
columns: {
id: "int(11) NOT NULL AUTO_INCREMENT",
message_id: "int(11) DEFAULT NULL",
status: "varchar(255) DEFAULT NULL",
code: "int(11) DEFAULT NULL",
output: "varchar(512) DEFAULT NULL",
details: "varchar(512) DEFAULT NULL",
sent_with_ssl: "tinyint(1) DEFAULT 0",
log_id: "varchar(100) DEFAULT NULL",
timestamp: "decimal(18,6) DEFAULT NULL"
},
indexes: {
on_message_id: "`message_id`"
})
end
end
end
end
lib/postal/message_db/migrations/04_create_live_stats.rb
+9 -8
View File
@@ -2,17 +2,18 @@ module Postal
module MessageDB
module Migrations
class CreateLiveStats < Postal::MessageDB::Migration
def up
@database.provisioner.create_table(:live_stats,
:columns => {
:type => 'varchar(20) NOT NULL',
:minute => 'int(11) NOT NULL',
:count => 'int(11) DEFAULT NULL',
:timestamp => 'decimal(18,6) DEFAULT NULL',
},
:primary_key => '`minute`, `type`(8)'
)
columns: {
type: "varchar(20) NOT NULL",
minute: "int(11) NOT NULL",
count: "int(11) DEFAULT NULL",
timestamp: "decimal(18,6) DEFAULT NULL"
},
primary_key: "`minute`, `type`(8)")
end
end
end
end
lib/postal/message_db/migrations/05_create_raw_message_sizes.rb
+10 -9
View File
@@ -2,18 +2,19 @@ module Postal
module MessageDB
module Migrations
class CreateRawMessageSizes < Postal::MessageDB::Migration
def up
@database.provisioner.create_table(:raw_message_sizes,
:columns => {
:id => 'int(11) NOT NULL AUTO_INCREMENT',
:table_name => 'varchar(255) DEFAULT NULL',
:size => 'bigint DEFAULT NULL'
},
:indexes => {
:on_table_name => '`table_name`(14)'
}
)
columns: {
id: "int(11) NOT NULL AUTO_INCREMENT",
table_name: "varchar(255) DEFAULT NULL",
size: "bigint DEFAULT NULL"
},
indexes: {
on_table_name: "`table_name`(14)"
})
end
end
end
end
lib/postal/message_db/migrations/06_create_clicks.rb
+16 -15
View File
@@ -2,24 +2,25 @@ module Postal
module MessageDB
module Migrations
class CreateClicks < Postal::MessageDB::Migration
def up
@database.provisioner.create_table(:clicks,
:columns => {
:id => 'int(11) NOT NULL AUTO_INCREMENT',
:message_id => 'int(11) DEFAULT NULL',
:link_id => 'int(11) DEFAULT NULL',
:ip_address => 'varchar(255) DEFAULT NULL',
:country => 'varchar(255) DEFAULT NULL',
:city => 'varchar(255) DEFAULT NULL',
:user_agent => 'varchar(255) DEFAULT NULL',
:timestamp => 'decimal(18,6) DEFAULT NULL'
},
:indexes => {
:on_message_id => '`message_id`',
:on_link_id => '`link_id`'
}
)
columns: {
id: "int(11) NOT NULL AUTO_INCREMENT",
message_id: "int(11) DEFAULT NULL",
link_id: "int(11) DEFAULT NULL",
ip_address: "varchar(255) DEFAULT NULL",
country: "varchar(255) DEFAULT NULL",
city: "varchar(255) DEFAULT NULL",
user_agent: "varchar(255) DEFAULT NULL",
timestamp: "decimal(18,6) DEFAULT NULL"
},
indexes: {
on_message_id: "`message_id`",
on_link_id: "`link_id`"
})
end
end
end
end
lib/postal/message_db/migrations/07_create_loads.rb
+14 -13
View File
@@ -2,22 +2,23 @@ module Postal
module MessageDB
module Migrations
class CreateLoads < Postal::MessageDB::Migration
def up
@database.provisioner.create_table(:loads,
:columns => {
:id => 'int(11) NOT NULL AUTO_INCREMENT',
:message_id => 'int(11) DEFAULT NULL',
:ip_address => 'varchar(255) DEFAULT NULL',
:country => 'varchar(255) DEFAULT NULL',
:city => 'varchar(255) DEFAULT NULL',
:user_agent => 'varchar(255) DEFAULT NULL',
:timestamp => 'decimal(18,6) DEFAULT NULL'
},
:indexes => {
:on_message_id => '`message_id`'
}
)
columns: {
id: "int(11) NOT NULL AUTO_INCREMENT",
message_id: "int(11) DEFAULT NULL",
ip_address: "varchar(255) DEFAULT NULL",
country: "varchar(255) DEFAULT NULL",
city: "varchar(255) DEFAULT NULL",
user_agent: "varchar(255) DEFAULT NULL",
timestamp: "decimal(18,6) DEFAULT NULL"
},
indexes: {
on_message_id: "`message_id`"
})
end
end
end
end
lib/postal/message_db/migrations/08_create_stats.rb
+14 -13
View File
@@ -2,24 +2,25 @@ module Postal
module MessageDB
module Migrations
class CreateStats < Postal::MessageDB::Migration
def up
[:hourly, :daily, :monthly, :yearly].each do |table_name|
@database.provisioner.create_table("stats_#{table_name}",
:columns => {
:id => 'int(11) NOT NULL AUTO_INCREMENT',
:time => 'int(11) DEFAULT NULL',
:incoming => 'bigint DEFAULT NULL',
:outgoing => 'bigint DEFAULT NULL',
:spam => 'bigint DEFAULT NULL',
:bounces => 'bigint DEFAULT NULL',
:held => 'bigint DEFAULT NULL',
},
:unique_indexes => {
:on_time => '`time`'
}
)
columns: {
id: "int(11) NOT NULL AUTO_INCREMENT",
time: "int(11) DEFAULT NULL",
incoming: "bigint DEFAULT NULL",
outgoing: "bigint DEFAULT NULL",
spam: "bigint DEFAULT NULL",
bounces: "bigint DEFAULT NULL",
held: "bigint DEFAULT NULL"
},
unique_indexes: {
on_time: "`time`"
})
end
end
end
end
end
lib/postal/message_db/migrations/09_create_links.rb
+14 -13
View File
@@ -2,22 +2,23 @@ module Postal
module MessageDB
module Migrations
class CreateLinks < Postal::MessageDB::Migration
def up
@database.provisioner.create_table(:links,
:columns => {
:id => 'int(11) NOT NULL AUTO_INCREMENT',
:message_id => 'int(11) DEFAULT NULL',
:token => 'varchar(255) DEFAULT NULL',
:hash => 'varchar(255) DEFAULT NULL',
:url => 'varchar(255) DEFAULT NULL',
:timestamp => 'decimal(18,6) DEFAULT NULL'
},
:indexes => {
:on_message_id => '`message_id`',
:on_token => '`token`(8)',
}
)
columns: {
id: "int(11) NOT NULL AUTO_INCREMENT",
message_id: "int(11) DEFAULT NULL",
token: "varchar(255) DEFAULT NULL",
hash: "varchar(255) DEFAULT NULL",
url: "varchar(255) DEFAULT NULL",
timestamp: "decimal(18,6) DEFAULT NULL"
},
indexes: {
on_message_id: "`message_id`",
on_token: "`token`(8)"
})
end
end
end
end
lib/postal/message_db/migrations/10_create_spam_checks.rb
+13 -12
View File
@@ -2,21 +2,22 @@ module Postal
module MessageDB
module Migrations
class CreateSpamChecks < Postal::MessageDB::Migration
def up
@database.provisioner.create_table(:spam_checks,
:columns => {
:id => 'int(11) NOT NULL AUTO_INCREMENT',
:message_id => 'int(11) DEFAULT NULL',
:score => 'decimal(8,2) DEFAULT NULL',
:code => 'varchar(255) DEFAULT NULL',
:description => 'varchar(255) DEFAULT NULL'
},
:indexes => {
:on_message_id => '`message_id`',
:on_code => '`code`(8)',
}
)
columns: {
id: "int(11) NOT NULL AUTO_INCREMENT",
message_id: "int(11) DEFAULT NULL",
score: "decimal(8,2) DEFAULT NULL",
code: "varchar(255) DEFAULT NULL",
description: "varchar(255) DEFAULT NULL"
},
indexes: {
on_message_id: "`message_id`",
on_code: "`code`(8)"
})
end
end
end
end
lib/postal/message_db/migrations/11_add_time_to_deliveries.rb
+2
View File
@@ -2,9 +2,11 @@ module Postal
module MessageDB
module Migrations
class AddTimeToDeliveries < Postal::MessageDB::Migration
def up
@database.query("ALTER TABLE `#{@database.database_name}`.`deliveries` ADD COLUMN `time` decimal(8,2)")
end
end
end
end
lib/postal/message_db/migrations/12_add_hold_expiry.rb
+2
View File
@@ -2,9 +2,11 @@ module Postal
module MessageDB
module Migrations
class AddHoldExpiry < Postal::MessageDB::Migration
def up
@database.query("ALTER TABLE `#{@database.database_name}`.`messages` ADD COLUMN `hold_expiry` decimal(18,6)")
end
end
end
end
lib/postal/message_db/migrations/13_add_index_to_message_status.rb
+2
View File
@@ -2,9 +2,11 @@ module Postal
module MessageDB
module Migrations
class AddIndexToMessageStatus < Postal::MessageDB::Migration
def up
@database.query("ALTER TABLE `#{@database.database_name}`.`messages` ADD INDEX `on_status` (`status`(8)) USING BTREE")
end
end
end
end
lib/postal/message_db/migrations/14_create_suppressions.rb
+14 -13
View File
@@ -2,22 +2,23 @@ module Postal
module MessageDB
module Migrations
class CreateSuppressions < Postal::MessageDB::Migration
def up
@database.provisioner.create_table(:suppressions,
:columns => {
:id => 'int(11) NOT NULL AUTO_INCREMENT',
:type => 'varchar(255) DEFAULT NULL',
:address => 'varchar(255) DEFAULT NULL',
:reason => 'varchar(255) DEFAULT NULL',
:timestamp => 'decimal(18,6) DEFAULT NULL',
:keep_until => 'decimal(18,6) DEFAULT NULL',
},
:indexes => {
:on_address => '`address`(6)',
:on_keep_until => '`keep_until`',
}
)
columns: {
id: "int(11) NOT NULL AUTO_INCREMENT",
type: "varchar(255) DEFAULT NULL",
address: "varchar(255) DEFAULT NULL",
reason: "varchar(255) DEFAULT NULL",
timestamp: "decimal(18,6) DEFAULT NULL",
keep_until: "decimal(18,6) DEFAULT NULL"
},
indexes: {
on_address: "`address`(6)",
on_keep_until: "`keep_until`"
})
end
end
end
end
lib/postal/message_db/migrations/15_create_webhook_requests.rb
+18 -17
View File
@@ -2,26 +2,27 @@ module Postal
module MessageDB
module Migrations
class CreateWebhookRequests < Postal::MessageDB::Migration
def up
@database.provisioner.create_table(:webhook_requests,
:columns => {
:id => 'int(11) NOT NULL AUTO_INCREMENT',
:uuid => 'varchar(255) DEFAULT NULL',
:event => 'varchar(255) DEFAULT NULL',
:attempt => 'int(11) DEFAULT NULL',
:timestamp => 'decimal(18,6) DEFAULT NULL',
:status_code => 'int(1) DEFAULT NULL',
:body => 'text DEFAULT NULL',
:payload => 'text DEFAULT NULL',
:will_retry => 'tinyint DEFAULT NULL'
},
:indexes => {
:on_uuid => '`uuid`(8)',
:on_event => '`event`(8)',
:on_timestamp => '`timestamp`',
}
)
columns: {
id: "int(11) NOT NULL AUTO_INCREMENT",
uuid: "varchar(255) DEFAULT NULL",
event: "varchar(255) DEFAULT NULL",
attempt: "int(11) DEFAULT NULL",
timestamp: "decimal(18,6) DEFAULT NULL",
status_code: "int(1) DEFAULT NULL",
body: "text DEFAULT NULL",
payload: "text DEFAULT NULL",
will_retry: "tinyint DEFAULT NULL"
},
indexes: {
on_uuid: "`uuid`(8)",
on_event: "`event`(8)",
on_timestamp: "`timestamp`"
})
end
end
end
end
lib/postal/message_db/migrations/16_add_url_and_hook_to_webhooks.rb
+2
View File
@@ -2,11 +2,13 @@ module Postal
module MessageDB
module Migrations
class AddUrlAndHookToWebhooks < Postal::MessageDB::Migration
def up
@database.query("ALTER TABLE `#{@database.database_name}`.`webhook_requests` ADD COLUMN `url` varchar(255)")
@database.query("ALTER TABLE `#{@database.database_name}`.`webhook_requests` ADD COLUMN `webhook_id` int(11)")
@database.query("ALTER TABLE `#{@database.database_name}`.`webhook_requests` ADD INDEX `on_webhook_id` (`webhook_id`) USING BTREE")
end
end
end
end
lib/postal/message_db/migrations/17_add_replaced_link_count_to_messages.rb
+2
View File
@@ -2,11 +2,13 @@ module Postal
module MessageDB
module Migrations
class AddReplacedLinkCountToMessages < Postal::MessageDB::Migration
def up
@database.query("ALTER TABLE `#{@database.database_name}`.`messages` ADD COLUMN `tracked_links` int(11) DEFAULT 0")
@database.query("ALTER TABLE `#{@database.database_name}`.`messages` ADD COLUMN `tracked_images` int(11) DEFAULT 0")
@database.query("ALTER TABLE `#{@database.database_name}`.`messages` ADD COLUMN `parsed` tinyint DEFAULT 0")
end
end
end
end
lib/postal/message_db/migrations/18_add_endpoints_to_messages.rb
+2
View File
@@ -2,9 +2,11 @@ module Postal
module MessageDB
module Migrations
class AddEndpointsToMessages < Postal::MessageDB::Migration
def up
@database.query("ALTER TABLE `#{@database.database_name}`.`messages` ADD COLUMN `endpoint_id` int(11), ADD COLUMN `endpoint_type` varchar(255)")
end
end
end
end
lib/postal/message_db/migrations/19_convert_database_to_utf8mb4.rb
+2
View File
@@ -2,6 +2,7 @@ module Postal
module MessageDB
module Migrations
class ConvertDatabaseToUtf8mb4 < Postal::MessageDB::Migration
def up
@database.query("ALTER DATABASE `#{@database.database_name}` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
@database.query("ALTER TABLE `#{@database.database_name}`.`clicks` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
@@ -20,6 +21,7 @@ module Postal
@database.query("ALTER TABLE `#{@database.database_name}`.`suppressions` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
@database.query("ALTER TABLE `#{@database.database_name}`.`webhook_requests` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
end
end
end
end
lib/postal/message_db/migrations/20_increase_links_url_size.rb
+2
View File
@@ -2,9 +2,11 @@ module Postal
module MessageDB
module Migrations
class IncreaseLinksUrlSize < Postal::MessageDB::Migration
def up
@database.query("ALTER TABLE `#{@database.database_name}`.`links` MODIFY `url` TEXT")
end
end
end
end
lib/postal/message_db/mysql.rb
+5 -7
View File
@@ -7,23 +7,21 @@ module Postal
# it would be undesirable as we'd just end up with lots of connections.
def self.new_client
Mysql2::Client.new(:host => Postal.config.message_db.host, :username => Postal.config.message_db.username, :password => Postal.config.message_db.password, :port => Postal.config.message_db.port, :reconnect => true, :encoding => Postal.config.message_db.encoding || 'utf8mb4')
Mysql2::Client.new(host: Postal.config.message_db.host, username: Postal.config.message_db.username, password: Postal.config.message_db.password, port: Postal.config.message_db.port, reconnect: true, encoding: Postal.config.message_db.encoding || "utf8mb4")
end
@free_clients = []
def self.client(&block)
client = @free_clients.shift || self.new_client
client = @free_clients.shift || new_client
return_value = nil
tries = 2
begin
return_value = block.call(client)
rescue Mysql2::Error => e
if e.message =~ /(lost connection|gone away)/i && (tries -= 1) > 0
retry
else
raise
end
raise unless e.message =~ /(lost connection|gone away)/i && (tries -= 1) > 0
retry
ensure
@free_clients << client
end
lib/postal/message_db/provisioner.rb
+27 -31
View File
@@ -46,7 +46,7 @@ module Postal
@database.query("DROP DATABASE `#{@database.database_name}`;")
true
rescue Mysql2::Error => e
e.message =~ /doesn\'t exist/ ? false : raise
e.message =~ /doesn't exist/ ? false : raise
end
#
@@ -68,9 +68,9 @@ module Postal
# environment and can be quite dangerous in production.
#
def clean
['clicks', 'deliveries', 'links', 'live_stats', 'loads', 'messages',
'raw_message_sizes', 'spam_checks', 'stats_daily', 'stats_hourly',
'stats_monthly', 'stats_yearly', 'suppressions', 'webhook_requests'].each do |table|
["clicks", "deliveries", "links", "live_stats", "loads", "messages",
"raw_message_sizes", "spam_checks", "stats_daily", "stats_hourly",
"stats_monthly", "stats_yearly", "suppressions", "webhook_requests"].each do |table|
@database.query("TRUNCATE `#{@database.database_name}`.`#{table}`")
end
end
@@ -79,18 +79,15 @@ module Postal
# Creates a new empty raw message table for the given date. Returns nothing.
#
def create_raw_table(table)
begin
@database.query(create_table_query(table,:columns => {
:id => 'int(11) NOT NULL AUTO_INCREMENT',
:data => 'longblob DEFAULT NULL',
:next => 'int(11) DEFAULT NULL'
}
))
@database.query("INSERT INTO `#{@database.database_name}`.`raw_message_sizes` (table_name, size) VALUES ('#{table}', 0)")
rescue Mysql2::Error => e
# Don't worry if the table already exists, another thread has already run this code.
raise unless e.message =~ /already exists/
end
@database.query(create_table_query(table, columns: {
id: "int(11) NOT NULL AUTO_INCREMENT",
data: "longblob DEFAULT NULL",
next: "int(11) DEFAULT NULL"
}))
@database.query("INSERT INTO `#{@database.database_name}`.`raw_message_sizes` (table_name, size) VALUES ('#{table}', 0)")
rescue Mysql2::Error => e
# Don't worry if the table already exists, another thread has already run this code.
raise unless e.message =~ /already exists/
end
#
@@ -101,7 +98,7 @@ module Postal
[].tap do |tables|
@database.query("SHOW TABLES FROM `#{@database.database_name}` LIKE 'raw-%'").each do |tbl|
tbl_name = tbl.to_a.first.last
date = Date.parse(tbl_name.gsub(/\Araw\-/, ''))
date = Date.parse(tbl_name.gsub(/\Araw-/, ""))
if earliest_date.nil? || date < earliest_date
tables << tbl_name
end
@@ -128,25 +125,25 @@ module Postal
end
#
# Remove messages from the messages table that are too old to retain
#  Remove messages from the messages table that are too old to retain
#
def remove_messages(max_age = 60)
time = (Date.today - max_age.days).to_time.end_of_day
if newest_message_to_remove = @database.select(:messages, :where => {:timestamp => {:less_than_or_equal_to => time.to_f}}, :limit => 1, :order => :id, :direction => 'DESC', :fields => [:id]).first
id = newest_message_to_remove['id']
@database.query("DELETE FROM `#{@database.database_name}`.`clicks` WHERE `message_id` <= #{id}")
@database.query("DELETE FROM `#{@database.database_name}`.`loads` WHERE `message_id` <= #{id}")
@database.query("DELETE FROM `#{@database.database_name}`.`deliveries` WHERE `message_id` <= #{id}")
@database.query("DELETE FROM `#{@database.database_name}`.`spam_checks` WHERE `message_id` <= #{id}")
@database.query("DELETE FROM `#{@database.database_name}`.`messages` WHERE `id` <= #{id}")
end
return unless newest_message_to_remove = @database.select(:messages, where: { timestamp: { less_than_or_equal_to: time.to_f } }, limit: 1, order: :id, direction: "DESC", fields: [:id]).first
id = newest_message_to_remove["id"]
@database.query("DELETE FROM `#{@database.database_name}`.`clicks` WHERE `message_id` <= #{id}")
@database.query("DELETE FROM `#{@database.database_name}`.`loads` WHERE `message_id` <= #{id}")
@database.query("DELETE FROM `#{@database.database_name}`.`deliveries` WHERE `message_id` <= #{id}")
@database.query("DELETE FROM `#{@database.database_name}`.`spam_checks` WHERE `message_id` <= #{id}")
@database.query("DELETE FROM `#{@database.database_name}`.`messages` WHERE `id` <= #{id}")
end
#
# Remove raw message tables in order order until size is under the given size (given in MB)
#
def remove_raw_tables_until_less_than_size(size)
tables = self.raw_tables(nil)
tables = raw_tables(nil)
tables_removed = []
until @database.total_size <= size
table = tables.shift
@@ -166,18 +163,18 @@ module Postal
s << "CREATE TABLE `#{@database.database_name}`.`#{table_name}` ("
s << options[:columns].map do |column_name, column_options|
"`#{column_name}` #{column_options}"
end.join(', ')
end.join(", ")
if options[:indexes]
s << ", "
s << options[:indexes].map do |index_name, index_options|
"KEY `#{index_name}` (#{index_options}) USING BTREE"
end.join(', ')
end.join(", ")
end
if options[:unique_indexes]
s << ", "
s << options[:unique_indexes].map do |index_name, index_options|
"UNIQUE KEY `#{index_name}` (#{index_options})"
end.join(', ')
end.join(", ")
end
if options[:primary_key]
s << ", PRIMARY KEY (#{options[:primary_key]})"
@@ -189,7 +186,6 @@ module Postal
end
end
end
end
end
lib/postal/message_db/statistics.rb
+3 -3
View File
@@ -6,7 +6,7 @@ module Postal
@database = database
end
STATS_GAPS = {:hourly => :hour, :daily => :day, :monthly => :month, :yearly => :year}
STATS_GAPS = { hourly: :hour, daily: :day, monthly: :month, yearly: :year }
COUNTERS = [:incoming, :outgoing, :spam, :bounces, :held]
#
@@ -44,8 +44,8 @@ module Postal
h[c] = 0
end
end
@database.select("stats_#{type}", :where => {:time => items.keys.map(&:to_i)}, :fields => [:time] | counters).each do |data|
time = Time.zone.at(data.delete('time'))
@database.select("stats_#{type}", where: { time: items.keys.map(&:to_i) }, fields: [:time] | counters).each do |data|
time = Time.zone.at(data.delete("time"))
data.each do |key, value|
items[time][key.to_sym] = value
end
lib/postal/message_db/suppression_list.rb
+8 -8
View File
@@ -8,29 +8,29 @@ module Postal
def add(type, address, options = {})
keep_until = (options[:days] || Postal.config.general.suppression_list_removal_delay).days.from_now.to_f
if existing = @database.select('suppressions', :where => {:type => type, :address => address}, :limit =>1).first
reason = options[:reason] || existing['reason']
@database.update('suppressions', {:reason => reason, :keep_until => keep_until}, :where => {:id => existing['id']})
if existing = @database.select("suppressions", where: { type: type, address: address }, limit: 1).first
reason = options[:reason] || existing["reason"]
@database.update("suppressions", { reason: reason, keep_until: keep_until }, where: { id: existing["id"] })
else
@database.insert('suppressions', {:type => type, :address => address, :reason => options[:reason], :timestamp => Time.now.to_f, :keep_until => keep_until})
@database.insert("suppressions", { type: type, address: address, reason: options[:reason], timestamp: Time.now.to_f, keep_until: keep_until })
end
true
end
def get(type, address)
@database.select('suppressions', :where => {:type => type, :address => address, :keep_until => {:greater_than_or_equal_to => Time.now.to_f}}, :limit => 1).first
@database.select("suppressions", where: { type: type, address: address, keep_until: { greater_than_or_equal_to: Time.now.to_f } }, limit: 1).first
end
def all_with_pagination(page)
@database.select_with_pagination(:suppressions, page, :order => :timestamp, :direction => 'desc')
@database.select_with_pagination(:suppressions, page, order: :timestamp, direction: "desc")
end
def remove(type, address)
@database.delete('suppressions', :where => {:type => type, :address => address}) > 0
@database.delete("suppressions", where: { type: type, address: address }) > 0
end
def prune
@database.delete('suppressions', :where => {:keep_until => {:less_than => Time.now.to_f}}) || 0
@database.delete("suppressions", where: { keep_until: { less_than: Time.now.to_f } }) || 0
end
end
lib/postal/message_db/webhooks.rb
+17 -15
View File
@@ -11,26 +11,27 @@ module Postal
end
def list(page)
result = @database.select_with_pagination(:webhook_requests, page, :order => :timestamp, :direction => 'desc')
result = @database.select_with_pagination(:webhook_requests, page, order: :timestamp, direction: "desc")
result[:records] = result[:records].map { |i| Request.new(i) }
result
end
def find(uuid)
request = @database.select(:webhook_requests, :where => {:uuid => uuid}).first || raise(RequestNotFound, "No request found with UUID '#{uuid}'")
request = @database.select(:webhook_requests, where: { uuid: uuid }).first || raise(RequestNotFound, "No request found with UUID '#{uuid}'")
Request.new(request)
end
def prune
if last = @database.select(:webhook_requests, :where => {:timestamp => {:less_than => 10.days.ago.to_f}}, :order => 'timestamp', :direction => 'desc', :limit => 1, :fields => ['id']).first
@database.delete(:webhook_requests, :where => {:id => {:less_than_or_equal_to => last['id']}})
end
return unless last = @database.select(:webhook_requests, where: { timestamp: { less_than: 10.days.ago.to_f } }, order: "timestamp", direction: "desc", limit: 1, fields: ["id"]).first
@database.delete(:webhook_requests, where: { id: { less_than_or_equal_to: last["id"] } })
end
class RequestNotFound < Postal::Error
end
class Request
def initialize(attributes)
@attributes = attributes
end
@@ -40,47 +41,48 @@ module Postal
end
def timestamp
Time.zone.at(@attributes['timestamp'])
Time.zone.at(@attributes["timestamp"])
end
def event
@attributes['event']
@attributes["event"]
end
def status_code
@attributes['status_code']
@attributes["status_code"]
end
def url
@attributes['url']
@attributes["url"]
end
def uuid
@attributes['uuid']
@attributes["uuid"]
end
def payload
@attributes['payload']
@attributes["payload"]
end
def pretty_payload
@pretty_payload ||= begin
json = JSON.parse(self.payload)
json = JSON.parse(payload)
JSON.pretty_unparse(json)
end
end
def body
@attributes['body']
@attributes["body"]
end
def attempt
@attributes['attempt']
@attributes["attempt"]
end
def will_retry?
@attributes['will_retry'] == 1
@attributes["will_retry"] == 1
end
end
end
lib/postal/message_inspection.rb
+2
View File
@@ -31,11 +31,13 @@ module Postal
end
class << self
def scan(message, scope)
inspection = new(message, scope)
inspection.scan
inspection
end
end
end
lib/postal/message_inspector.rb
+3 -3
View File
@@ -17,11 +17,11 @@ module Postal
end
class << self
# Return an array of all inspectors that are available for this
# installation.
def inspectors
Array.new.tap do |inspectors|
[].tap do |inspectors|
if Postal.config.rspamd&.enabled
inspectors << MessageInspectors::Rspamd.new(Postal.config.rspamd)
elsif Postal.config.spamd&.enabled
@@ -31,9 +31,9 @@ module Postal
if Postal.config.clamav&.enabled
inspectors << MessageInspectors::Clamav.new(Postal.config.clamav)
end
end
end
end
end
lib/postal/message_inspectors.rb
+2
View File
@@ -1,10 +1,12 @@
module Postal
module MessageInspectors
extend ActiveSupport::Autoload
eager_autoload do
autoload :Clamav
autoload :Rspamd
autoload :SpamAssassin
end
end
end
lib/postal/message_inspectors/clamav.rb
+10 -6
View File
@@ -16,13 +16,13 @@ module Postal
data = tcp_socket.read
end
if data && data =~ /\Astream\:\s+(.*?)[\s\0]+?/
if $1.upcase == 'OK'
if data && data =~ /\Astream:\s+(.*?)[\s\0]+?/
if ::Regexp.last_match(1).upcase == "OK"
inspection.threat = false
inspection.threat_message = "No threats found"
else
inspection.threat = true
inspection.threat_message = $1
inspection.threat_message = ::Regexp.last_match(1)
end
else
inspection.threat = false
@@ -31,13 +31,17 @@ module Postal
rescue Timeout::Error
inspection.threat = false
inspection.threat_message = "Timed out scanning for threats"
rescue => e
rescue StandardError => e
logger.error "Error talking to clamav: #{e.class} (#{e.message})"
logger.error e.backtrace[0,5]
logger.error e.backtrace[0, 5]
inspection.threat = false
inspection.threat_message = "Error when scanning for threats"
ensure
tcp_socket.close rescue nil
begin
tcp_socket.close
rescue StandardError
nil
end
end
end
lib/postal/message_inspectors/rspamd.rb
+17 -17
View File
@@ -1,4 +1,4 @@
require 'net/http'
require "net/http"
module Postal
module MessageInspectors
@@ -10,12 +10,12 @@ module Postal
def inspect_message(inspection)
response = request(inspection.message, inspection.scope)
response = JSON.parse(response.body)
return unless response['symbols'].is_a?(Hash)
return unless response["symbols"].is_a?(Hash)
response['symbols'].values.each do |symbol|
next if symbol['description'].blank?
response["symbols"].values.each do |symbol|
next if symbol["description"].blank?
inspection.spam_checks << SpamCheck.new(symbol['name'], symbol['score'], symbol['description'])
inspection.spam_checks << SpamCheck.new(symbol["name"], symbol["score"], symbol["description"])
end
rescue Error => e
inspection.spam_checks << SpamCheck.new("ERROR", 0, e.message)
@@ -31,24 +31,24 @@ module Postal
raw_message = message.raw_message
request = Net::HTTP::Post.new('/checkv2')
request = Net::HTTP::Post.new("/checkv2")
request.body = raw_message
request['Content-Length'] = raw_message.bytesize.to_s
request['Password'] = @config.password if @config.password
request['Flags'] = @config.flags if @config.flags
request['User-Agent'] = 'Postal'
request['Deliver-To'] = message.rcpt_to
request['From'] = message.mail_from
request['Rcpt'] = message.rcpt_to
request['Queue-Id'] = message.token
request["Content-Length"] = raw_message.bytesize.to_s
request["Password"] = @config.password if @config.password
request["Flags"] = @config.flags if @config.flags
request["User-Agent"] = "Postal"
request["Deliver-To"] = message.rcpt_to
request["From"] = message.mail_from
request["Rcpt"] = message.rcpt_to
request["Queue-Id"] = message.token
if scope == :outgoing
request['User'] = ''
request["User"] = ""
# We don't actually know the IP but an empty input here will
# still trigger rspamd to treat this as an outbound email
# and disable certain checks.
# https://rspamd.com/doc/tutorials/scanning_outbound.html
request['Ip'] = ''
request["Ip"] = ""
end
response = nil
@@ -56,7 +56,7 @@ module Postal
response = http.request(request)
rescue Exception => e
logger.error "Error talking to rspamd: #{e.class} (#{e.message})"
logger.error e.backtrace[0,5]
logger.error e.backtrace[0, 5]
raise Error, "Error when scanning with rspamd (#{e.class})"
end
lib/postal/message_inspectors/spam_assassin.rb
+13 -12
View File
@@ -3,8 +3,8 @@ module Postal
class SpamAssassin < MessageInspector
EXCLUSIONS = {
:outgoing => ['NO_RECEIVED', 'NO_RELAYS', 'ALL_TRUSTED', 'FREEMAIL_FORGED_REPLYTO', 'RDNS_DYNAMIC', 'CK_HELO_GENERIC', /^SPF\_/, /^HELO\_/, /DKIM_/, /^RCVD_IN_/],
:incoming => []
outgoing: ["NO_RECEIVED", "NO_RELAYS", "ALL_TRUSTED", "FREEMAIL_FORGED_REPLYTO", "RDNS_DYNAMIC", "CK_HELO_GENERIC", /^SPF_/, /^HELO_/, /DKIM_/, /^RCVD_IN_/],
incoming: []
}
def inspect_message(inspection)
@@ -24,11 +24,11 @@ module Postal
total = 0.0
rules = data ? data.split(/^---(.*)\r?\n/).last.split(/\r?\n/) : []
while line = rules.shift
if line =~ /\A([\- ]?[\d\.]+)\s+(\w+)\s+(.*)/
total += $1.to_f
spam_checks << SpamCheck.new($2, $1.to_f, $3)
if line =~ /\A([- ]?[\d.]+)\s+(\w+)\s+(.*)/
total += ::Regexp.last_match(1).to_f
spam_checks << SpamCheck.new(::Regexp.last_match(2), ::Regexp.last_match(1).to_f, ::Regexp.last_match(3))
else
spam_checks.last.description << " " + line.strip
spam_checks.last.description << (" " + line.strip)
end
end
@@ -36,17 +36,18 @@ module Postal
checks.each do |check|
inspection.spam_checks << check
end
rescue Timeout::Error
inspection.spam_checks << SpamCheck.new("TIMEOUT", 0, "Timed out when scanning for spam")
rescue => e
rescue StandardError => e
logger.error "Error talking to spamd: #{e.class} (#{e.message})"
logger.error e.backtrace[0,5]
logger.error e.backtrace[0, 5]
inspection.spam_checks << SpamCheck.new("ERROR", 0, "Error when scanning for spam")
ensure
tcp_socket.close rescue nil
begin
tcp_socket.close
rescue StandardError
nil
end
end
end
lib/postal/message_parser.rb
+25 -27
View File
@@ -1,18 +1,18 @@
module Postal
class MessageParser
URL_REGEX = /(?<url>(?<protocol>https?)\:\/\/(?<domain>[A-Za-z0-9\-\.\:]+)(?<path>\/[A-Za-z0-9\.\/\+\?\&\-\_\%\=\~\:\;\(\)\[\]#]*)?+)/
URL_REGEX = /(?<url>(?<protocol>https?):\/\/(?<domain>[A-Za-z0-9\-.:]+)(?<path>\/[A-Za-z0-9.\/+?&\-_%=~:;()\[\]#]*)?+)/
def initialize(message)
@message = message
@actioned = false
@tracked_links = 0
@tracked_images = 0
@domain = @message.server.track_domains.where(:domain => @message.domain, :dns_status => "OK").first
@domain = @message.server.track_domains.where(domain: @message.domain, dns_status: "OK").first
if @domain
@parsed_output = generate
end
return unless @domain
@parsed_output = generate
end
attr_reader :tracked_links
@@ -36,29 +36,27 @@ module Postal
if @mail.mime_type =~ /text\/plain/
@mail.body = parse(@mail.body.decoded.dup, :text)
@mail.content_transfer_encoding = nil
@mail.charset = 'UTF-8'
@mail.charset = "UTF-8"
elsif @mail.mime_type =~ /text\/html/
@mail.body = parse(@mail.body.decoded.dup, :html)
@mail.content_transfer_encoding = nil
@mail.charset = 'UTF-8'
@mail.charset = "UTF-8"
end
end
else
parse_parts(@mail.parts)
end
@mail.to_s
rescue => e
if Rails.env.development?
raise
else
if defined?(Raven)
Raven.capture_exception(e)
end
@actioned = false
@tracked_links = 0
@tracked_images = 0
@original_message
rescue StandardError => e
raise if Rails.env.development?
if defined?(Raven)
Raven.capture_exception(e)
end
@actioned = false
@tracked_links = 0
@tracked_images = 0
@original_message
end
def parse_parts(parts)
@@ -66,11 +64,11 @@ module Postal
if part.content_type =~ /text\/html/
part.body = parse(part.body.decoded.dup, :html)
part.content_transfer_encoding = nil
part.charset = 'UTF-8'
part.charset = "UTF-8"
elsif part.content_type =~ /text\/plain/
part.body = parse(part.body.decoded.dup, :text)
part.content_transfer_encoding = nil
part.charset = 'UTF-8'
part.charset = "UTF-8"
elsif part.content_type =~ /multipart\/(alternative|related)/
unless part.parts.empty?
parse_parts(part.parts)
@@ -97,33 +95,33 @@ module Postal
if track_domain?($~[:domain])
@tracked_links += 1
url = $~[:url]
while url =~ /[^\w]$/ do
while url =~ /[^\w]$/
theend = url.size - 2
url = url[0..theend]
end
token = @message.create_link(url)
"#{domain}/#{@message.server.token}/#{token}"
else
$&
::Regexp.last_match(0)
end
end
end
if type == :html
part.gsub!(/href=([\'\"])(#{URL_REGEX})[\'\"]/) do
part.gsub!(/href=(['"])(#{URL_REGEX})['"]/) do
if track_domain?($~[:domain])
@tracked_links += 1
token = @message.create_link($~[:url])
"href='#{domain}/#{@message.server.token}/#{token}'"
else
$&
::Regexp.last_match(0)
end
end
end
part.gsub!(/(https?)\+notrack\:\/\//) do
part.gsub!(/(https?)\+notrack:\/\//) do
@actioned = true
"#{$1}://"
"#{::Regexp.last_match(1)}://"
end
part
@@ -132,7 +130,7 @@ module Postal
def insert_tracking_image(part)
@tracked_images += 1
container = "<p class='ampimg' style='display:none;visibility:none;margin:0;padding:0;line-height:0;'><img src='#{domain}/img/#{@message.server.token}/#{@message.token}' alt=''></p>"
if part =~ /\<\/body\>/
if part =~ /<\/body>/
part.gsub("</body>", "#{container}</body>")
else
part + container
lib/postal/message_requeuer.rb
+4 -4
View File
@@ -22,10 +22,10 @@ module Postal
end
def check_exit
if @exit
log "Exiting"
Process.exit(0)
end
return unless @exit
log "Exiting"
Process.exit(0)
end
end
lib/postal/mx_lookup.rb
+1 -1
View File
@@ -23,7 +23,7 @@ module Postal
def resolve(domain)
Resolv::DNS.open do |dns|
dns.timeouts = [10,5]
dns.timeouts = [10, 5]
dns.getresources(domain, Resolv::DNS::Resource::IN::MX).map { |m| [m.preference.to_i, m.exchange.to_s] }
end
end
lib/postal/query_string.rb
+2 -4
View File
@@ -9,12 +9,10 @@ module Postal
to_hash[value.to_s]
end
def empty?
to_hash.empty?
end
delegate :empty?, to: :to_hash
def to_hash
@hash ||= @string.scan(/([a-z]+)\:\s*(?:(\d{2,4}\-\d{2}-\d{2}\s\d{2}\:\d{2})|\"(.*?)\"|(.*?))(\s|\z)/).each_with_object({}) do |(key, date, string_with_spaces, value), hash|
@hash ||= @string.scan(/([a-z]+):\s*(?:(\d{2,4}-\d{2}-\d{2}\s\d{2}:\d{2})|"(.*?)"|(.*?))(\s|\z)/).each_with_object({}) do |(key, date, string_with_spaces, value), hash|
if date
actual_value = date
elsif string_with_spaces
lib/postal/rabbit_mq.rb
+13 -13
View File
@@ -1,34 +1,34 @@
require 'postal/config'
require 'bunny'
require "postal/config"
require "bunny"
module Postal
module RabbitMQ
def self.create_connection
bunny_host = [ 'localhost' ]
bunny_host = ["localhost"]
if Postal.config.rabbitmq&.host.is_a?(Array)
bunny_host = Postal.config.rabbitmq&.host
elsif Postal.config.rabbitmq&.host.is_a?(String)
bunny_host = [ Postal.config.rabbitmq&.host ]
bunny_host = [Postal.config.rabbitmq&.host]
end
conn = Bunny.new(
:hosts => bunny_host,
:port => Postal.config.rabbitmq&.port || 5672,
:tls => Postal.config.rabbitmq&.tls || false,
:verify_peer => Postal.config.rabbitmq&.verify_peer || true,
:tls_ca_certificates => Postal.config.rabbitmq&.tls_ca_certificates || [ "/etc/ssl/certs/ca-certificates.crt" ],
:username => Postal.config.rabbitmq&.username || 'guest',
:password => Postal.config.rabbitmq&.password || 'guest',
:vhost => Postal.config.rabbitmq&.vhost || nil
hosts: bunny_host,
port: Postal.config.rabbitmq&.port || 5672,
tls: Postal.config.rabbitmq&.tls || false,
verify_peer: Postal.config.rabbitmq&.verify_peer || true,
tls_ca_certificates: Postal.config.rabbitmq&.tls_ca_certificates || ["/etc/ssl/certs/ca-certificates.crt"],
username: Postal.config.rabbitmq&.username || "guest",
password: Postal.config.rabbitmq&.password || "guest",
vhost: Postal.config.rabbitmq&.vhost || nil
)
conn.start
conn
end
def self.create_channel
conn = self.create_connection
conn = create_connection
conn.create_channel(nil, Postal.config.workers.threads)
end
lib/postal/reply_separator.rb
+16 -15
View File
@@ -3,30 +3,31 @@ module Postal
RULES = [
/^-{2,10} $.*/m,
/^\>*\s*----- ?Original Message ?-----.*/m,
/^\>*\s*From\:[^\r\n]*[\r\n]+Sent\:.*/m,
/^\>*\s*From\:[^\r\n]*[\r\n]+Date\:.*/m,
/^\>*\s*-----Urspr.ngliche Nachricht----- .*/m,
/^\>*\s*Le[^\r\n]{10,200}a .crit ?\:\s*$.*/,
/^\>*\s*__________________.*/m,
/^\>*\s*On.{10,200}wrote:\s*$.*/m,
/^\>*\s*Sent from my.*/m,
/^\>*\s*=== Please reply above this line ===.*/m,
/(^\>.*\n?){10,}/,
/^>*\s*----- ?Original Message ?-----.*/m,
/^>*\s*From:[^\r\n]*[\r\n]+Sent:.*/m,
/^>*\s*From:[^\r\n]*[\r\n]+Date:.*/m,
/^>*\s*-----Urspr.ngliche Nachricht----- .*/m,
/^>*\s*Le[^\r\n]{10,200}a .crit ?:\s*$.*/,
/^>*\s*__________________.*/m,
/^>*\s*On.{10,200}wrote:\s*$.*/m,
/^>*\s*Sent from my.*/m,
/^>*\s*=== Please reply above this line ===.*/m,
/(^>.*\n?){10,}/
]
def self.separate(text)
return '' unless text.is_a?(String)
text = text.gsub("\r", '')
return "" unless text.is_a?(String)
text = text.gsub("\r", "")
stripped = ""
RULES.each do |rule|
text.gsub!(rule) do
stripped = $&.to_s + "\n" + stripped
''
stripped = ::Regexp.last_match(0).to_s + "\n" + stripped
""
end
end
stripped = stripped.strip
[text.strip, stripped.blank? ? nil : stripped]
[text.strip, stripped.presence]
end
end
lib/postal/rspec_helpers.rb
+4 -4
View File
@@ -8,12 +8,12 @@ module Postal
server.message_db.provisioner.clean
end
def create_plain_text_message(server, text, to = 'test@example.com', override_attributes = {})
domain = create(:domain, :owner => server)
attributes = {:from => "test@#{domain.name}", :subject => "Test Plain Text Message"}.merge(override_attributes)
def create_plain_text_message(server, text, to = "test@example.com", override_attributes = {})
domain = create(:domain, owner: server)
attributes = { from: "test@#{domain.name}", subject: "Test Plain Text Message" }.merge(override_attributes)
attributes[:to] = to
attributes[:plain_body] = text
message = OutgoingMessagePrototype.new(server, '127.0.0.1', 'testsuite', attributes)
message = OutgoingMessagePrototype.new(server, "127.0.0.1", "testsuite", attributes)
result = message.create_message(to)
server.message_db.message(result[:id])
end
lib/postal/send_result.rb
+2
View File
@@ -1,5 +1,6 @@
module Postal
class SendResult
attr_accessor :type
attr_accessor :details
attr_accessor :retry
@@ -9,5 +10,6 @@ module Postal
attr_accessor :log_id
attr_accessor :time
attr_accessor :suppress_bounce
end
end
lib/postal/sender.rb
+2
View File
@@ -1,5 +1,6 @@
module Postal
class Sender
def start
end
@@ -8,5 +9,6 @@ module Postal
def finish
end
end
end
lib/postal/smtp_sender.rb
+50 -47
View File
@@ -1,4 +1,4 @@
require 'resolv'
require "resolv"
module Postal
class SMTPSender < Sender
@@ -10,7 +10,7 @@ module Postal
@smtp_client = nil
@connection_errors = []
@hostnames = []
@log_id = Nifty::Utils::RandomString.generate(:length => 8).upcase
@log_id = Nifty::Utils::RandomString.generate(length: 8).upcase
end
def start
@@ -22,16 +22,15 @@ module Postal
elsif server.is_a?(Hash)
hostname = server[:hostname]
port = server[:port] || 25
ssl_mode = server[:ssl_mode] || 'Auto'
ssl_mode = server[:ssl_mode] || "Auto"
else
hostname = server
port = 25
ssl_mode = 'Auto'
ssl_mode = "Auto"
end
@hostnames << hostname
[:aaaa, :a].each do |ip_type|
if @source_ip_address && @source_ip_address.ipv6.blank? && ip_type == :aaaa
# Don't try to use IPv6 if the IP address we're sending from doesn't support it.
next
@@ -57,11 +56,11 @@ module Postal
end
case ssl_mode
when 'Auto'
when "Auto"
smtp_client.enable_starttls_auto(self.class.ssl_context_without_verify)
when 'STARTTLS'
when "STARTTLS"
smtp_client.enable_starttls(self.class.ssl_context_with_verify)
when 'TLS'
when "TLS"
smtp_client.enable_tls(self.class.ssl_context_with_verify)
else
# Nothing
@@ -69,9 +68,8 @@ module Postal
smtp_client.start(@source_ip_address ? @source_ip_address.hostname : self.class.default_helo_hostname)
log "Connected to #{@remote_ip}:#{port} (#{hostname})"
rescue => e
if e.is_a?(OpenSSL::SSL::SSLError) && ssl_mode == 'Auto'
rescue StandardError => e
if e.is_a?(OpenSSL::SSL::SSLError) && ssl_mode == "Auto"
log "SSL error (#{e.message}), retrying without SSL"
ssl_mode = nil
retry
@@ -79,7 +77,11 @@ module Postal
log "Cannot connect to #{@remote_ip}:#{port} (#{hostname}) (#{e.class}: #{e.message})"
@connection_errors << e.message unless @connection_errors.include?(e.message)
smtp_client.disconnect rescue nil
begin
smtp_client.disconnect
rescue StandardError
nil
end
smtp_client = nil
end
@@ -95,17 +97,24 @@ module Postal
def reconnect
log "Reconnecting"
@smtp_client&.finish rescue nil
begin
@smtp_client&.finish
rescue StandardError
nil
end
start
end
def safe_rset
# Something went wrong sending the last email. Reset the connection if possible, else disconnect.
@smtp_client.rset
rescue StandardError
# Don't reconnect, this would be rather rude if we don't have any more emails to send.
begin
@smtp_client.rset
rescue
# Don't reconnect, this would be rather rude if we don't have any more emails to send.
@smtp_client.finish rescue nil
@smtp_client.finish
rescue StandardError
nil
end
end
@@ -126,7 +135,7 @@ module Postal
begin
if message.bounce == 1
mail_from = ""
elsif message.domain.return_path_status == 'OK'
elsif message.domain.return_path_status == "OK"
mail_from = "#{message.server.token}@#{message.domain.return_path_domain}"
else
mail_from = "#{message.server.token}@#{Postal.config.dns.return_path}"
@@ -138,10 +147,10 @@ module Postal
log "-> No SMTP server available for #{@domain}"
log "-> Hostnames: #{@hostnames.inspect}"
log "-> Errors: #{@connection_errors.inspect}"
result.type = 'SoftFail'
result.type = "SoftFail"
result.retry = true
result.details = "No SMTP servers were available for #{@domain}. Tried #{@hostnames.to_sentence}"
result.output = @connection_errors.join(', ')
result.output = @connection_errors.join(", ")
result.connect_error = true
return result
else
@@ -151,46 +160,43 @@ module Postal
smtp_result = @smtp_client.send_message(raw_message, mail_from, [rcpt_to])
end
rescue Errno::ECONNRESET, Errno::EPIPE, OpenSSL::SSL::SSLError
if (tries += 1) < 2
reconnect
retry
else
raise
end
raise unless (tries += 1) < 2
reconnect
retry
end
result.type = 'Sent'
result.type = "Sent"
result.details = "Message for #{rcpt_to} accepted by #{destination_host_description}"
if @smtp_client.source_address
result.details += " (from #{@smtp_client.source_address})"
end
result.output = smtp_result.string
log "Message sent ##{message.id} to #{destination_host_description} for #{rcpt_to}"
rescue Net::SMTPServerBusy, Net::SMTPAuthenticationError, Net::SMTPSyntaxError, Net::SMTPUnknownError, Net::ReadTimeout => e
log "#{e.class}: #{e.message}"
result.type = 'SoftFail'
result.type = "SoftFail"
result.retry = true
result.details = "Temporary SMTP delivery error when sending to #{destination_host_description}"
result.output = e.message
if e.to_s =~ /(\d+) seconds/
result.retry = $1.to_i + 10
result.retry = ::Regexp.last_match(1).to_i + 10
elsif e.to_s =~ /(\d+) minutes/
result.retry = ($1.to_i * 60) + 10
result.retry = (::Regexp.last_match(1).to_i * 60) + 10
end
safe_rset
rescue Net::SMTPFatalError => e
log "#{e.class}: #{e.message}"
result.type = 'HardFail'
result.type = "HardFail"
result.details = "Permanent SMTP delivery error when sending to #{destination_host_description}"
result.output = e.message
safe_rset
rescue => e
rescue StandardError => e
log "#{e.class}: #{e.message}"
if defined?(Raven)
Raven.capture_exception(e, :extra => {:log_id => @log_id, :server_id => message.server.id, :message_id => message.id})
Raven.capture_exception(e, extra: { log_id: @log_id, server_id: message.server.id, message_id: message.id })
end
result.type = 'SoftFail'
result.type = "SoftFail"
result.retry = true
result.details = "An error occurred while sending the message to #{destination_host_description}"
result.output = e.message
@@ -198,8 +204,7 @@ module Postal
end
result.time = (Time.now - start_time).to_f.round(2)
return result
ensure
result
end
def finish
@@ -230,7 +235,7 @@ module Postal
def lookup_ip_address(type, hostname)
records = []
Resolv::DNS.open do |dns|
dns.timeouts = [10,5]
dns.timeouts = [10, 5]
case type
when :a
records = dns.getresources(hostname, Resolv::DNS::Resource::IN::A)
@@ -265,15 +270,13 @@ module Postal
def self.relay_hosts
hosts = Postal.config.smtp_relays.map do |relay|
if relay.hostname.present?
{
:hostname => relay.hostname,
:port => relay.port,
:ssl_mode => relay.ssl_mode
}
else
nil
end
next unless relay.hostname.present?
{
hostname: relay.hostname,
port: relay.port,
ssl_mode: relay.ssl_mode
}
end.compact
hosts.empty? ? nil : hosts
end
lib/postal/smtp_server.rb
+2
View File
@@ -1,9 +1,11 @@
module Postal
module SMTPServer
extend ActiveSupport::Autoload
eager_autoload do
autoload :Client
autoload :Server
end
end
end
lib/postal/smtp_server/client.rb
+100 -95
View File
@@ -1,11 +1,11 @@
require 'resolv'
require 'nifty/utils/random_string'
require "resolv"
require "nifty/utils/random_string"
module Postal
module SMTPServer
class Client
CRAM_MD5_DIGEST = OpenSSL::Digest.new('md5')
CRAM_MD5_DIGEST = OpenSSL::Digest.new("md5")
LOG_REDACTION_STRING = "[redacted]".freeze
attr_reader :logging_enabled
@@ -23,9 +23,9 @@ module Postal
end
def check_ip_address
if @ip_address && Postal.config.smtp_server.log_exclude_ips && @ip_address =~ Regexp.new(Postal.config.smtp_server.log_exclude_ips)
@logging_enabled = false
end
return unless @ip_address && Postal.config.smtp_server.log_exclude_ips && @ip_address =~ Regexp.new(Postal.config.smtp_server.log_exclude_ips)
@logging_enabled = false
end
def transaction_reset
@@ -36,7 +36,7 @@ module Postal
end
def id
@id ||= Nifty::Utils::RandomString.generate(:length => 6).upcase
@id ||= Nifty::Utils::RandomString.generate(length: 6).upcase
end
def handle(data)
@@ -56,13 +56,13 @@ module Postal
def sanitize_input_for_log(data)
if @password_expected_next
@password_expected_next = false
if data =~ /\A[a-z0-9]{3,}\=*\z/i
if data =~ /\A[a-z0-9]{3,}=*\z/i
return LOG_REDACTION_STRING
end
end
data = data.dup
data.gsub!(/(.*AUTH \w+) (.*)\z/i) { "#{$1} #{LOG_REDACTION_STRING}" }
data.gsub!(/(.*AUTH \w+) (.*)\z/i) { "#{::Regexp.last_match(1)} #{LOG_REDACTION_STRING}" }
data
end
@@ -74,9 +74,7 @@ module Postal
@start_tls || false
end
def start_tls=(value)
@start_tls = value
end
attr_writer :start_tls
def handle_command(data)
case data
@@ -93,12 +91,13 @@ module Postal
when /^RCPT TO/i then rcpt_to(data)
when /^DATA/i then data(data)
else
'502 Invalid/unsupported command'
"502 Invalid/unsupported command"
end
end
def log(text)
return false unless @logging_enabled
Postal.logger_for(:smtp_server).debug "[#{id}] #{text}"
end
@@ -106,8 +105,12 @@ module Postal
def resolve_hostname
Resolv::DNS.open do |dns|
dns.timeouts = [10,5]
@hostname = dns.getname(@ip_address) rescue @ip_address
dns.timeouts = [10, 5]
@hostname = begin
dns.getname(@ip_address)
rescue StandardError
@ip_address
end
end
end
@@ -120,7 +123,7 @@ module Postal
"220 #{Postal.config.dns.smtp_server_hostname} ESMTP Postal/#{id}"
else
@finished = true
'502 Proxy Error'
"502 Proxy Error"
end
end
@@ -141,15 +144,15 @@ module Postal
def ehlo(data)
resolve_hostname
@helo_name = data.strip.split(' ', 2)[1]
@helo_name = data.strip.split(" ", 2)[1]
transaction_reset
@state = :welcomed
["250-My capabilities are", Postal.config.smtp_server.tls_enabled? && !@tls ? "250-STARTTLS" : nil, "250 AUTH CRAM-MD5 PLAIN LOGIN", ]
["250-My capabilities are", Postal.config.smtp_server.tls_enabled? && !@tls ? "250-STARTTLS" : nil, "250 AUTH CRAM-MD5 PLAIN LOGIN"]
end
def helo(data)
resolve_hostname
@helo_name = data.strip.split(' ', 2)[1]
@helo_name = data.strip.split(" ", 2)[1]
transaction_reset
@state = :welcomed
"250 #{Postal.config.dns.smtp_server_hostname}"
@@ -158,59 +161,61 @@ module Postal
def rset
transaction_reset
@state = :welcomed
'250 OK'
"250 OK"
end
def noop
'250 OK'
"250 OK"
end
def auth_plain(data)
handler = Proc.new do |data|
handler = proc do |data|
@proc = nil
data = Base64.decode64(data)
parts = data.split("\0")
username, password = parts[-2], parts[-1]
username = parts[-2]
password = parts[-1]
unless username && password
next '535 Authenticated failed - protocol error'
next "535 Authenticated failed - protocol error"
end
authenticate(password)
end
data = data.gsub(/AUTH PLAIN ?/i, '')
if data.strip == ''
data = data.gsub(/AUTH PLAIN ?/i, "")
if data.strip == ""
@proc = handler
@password_expected_next = true
'334'
"334"
else
handler.call(data)
end
end
def auth_login(data)
password_handler = Proc.new do |data|
password_handler = proc do |data|
@proc = nil
password = Base64.decode64(data)
authenticate(password)
end
username_handler = Proc.new do |data|
username_handler = proc do |data|
@proc = password_handler
@password_expected_next = true
'334 UGFzc3dvcmQ6' # "Password:"
"334 UGFzc3dvcmQ6" # "Password:"
end
data = data.gsub!(/AUTH LOGIN ?/i, '')
if data.strip == ''
data = data.gsub!(/AUTH LOGIN ?/i, "")
if data.strip == ""
@proc = username_handler
'334 VXNlcm5hbWU6' # "Username:"
"334 VXNlcm5hbWU6" # "Username:"
else
username_handler.call(nil)
end
end
def authenticate(password)
if @credential = Credential.where(:type => 'SMTP', :key => password).first
if @credential = Credential.where(type: "SMTP", key: password).first
@credential.use
"235 Granted for #{@credential.server.organization.permalink}/#{@credential.server.permalink}"
else
@@ -220,27 +225,27 @@ module Postal
end
def auth_cram_md5(data)
challenge = Digest::SHA1.hexdigest(Time.now.to_i.to_s + rand(100000).to_s)
challenge = "<#{challenge[0,20]}@#{Postal.config.dns.smtp_server_hostname}>"
challenge = Digest::SHA1.hexdigest(Time.now.to_i.to_s + rand(100_000).to_s)
challenge = "<#{challenge[0, 20]}@#{Postal.config.dns.smtp_server_hostname}>"
handler = Proc.new do |data|
handler = proc do |data|
@proc = nil
username, password = Base64.decode64(data).split(' ', 2).map{ |a| a.chomp }
org_permlink, server_permalink = username.split(/[\/\_]/, 2)
server = ::Server.includes(:organization).where(:organizations => {:permalink => org_permlink}, :permalink => server_permalink).first
username, password = Base64.decode64(data).split(" ", 2).map { |a| a.chomp }
org_permlink, server_permalink = username.split(/[\/_]/, 2)
server = ::Server.includes(:organization).where(organizations: { permalink: org_permlink }, permalink: server_permalink).first
if server.nil?
log "\e[33m WARN: AUTH failure for #{@ip_address}\e[0m"
next '535 Denied'
next "535 Denied"
end
grant = nil
server.credentials.where(:type => 'SMTP').each do |credential|
server.credentials.where(type: "SMTP").each do |credential|
correct_response = OpenSSL::HMAC.hexdigest(CRAM_MD5_DIGEST, credential.key, challenge)
if password == correct_response
@credential = credential
@credential.use
grant = "235 Granted for #{credential.server.organization.permalink}/#{credential.server.permalink}"
break
end
next unless password == correct_response
@credential = credential
@credential.use
grant = "235 Granted for #{credential.server.organization.permalink}/#{credential.server.permalink}"
break
end
if grant.nil?
log "\e[33m WARN: AUTH failure for #{@ip_address}\e[0m"
@@ -250,12 +255,12 @@ module Postal
end
@proc = handler
"334 " + Base64.encode64(challenge).gsub(/[\r\n]/, '')
"334 " + Base64.encode64(challenge).gsub(/[\r\n]/, "")
end
def mail_from(data)
unless in_state(:welcomed, :mail_from_received)
return '503 EHLO/HELO first please'
return "503 EHLO/HELO first please"
end
@state = :mail_from_received
@@ -263,93 +268,93 @@ module Postal
if data =~ /AUTH=/
# Discard AUTH= parameter and anything that follows.
# We don't need this parameter as we don't trust any client to set it
mail_from_line = data.sub(/ *AUTH=.*/, '')
mail_from_line = data.sub(/ *AUTH=.*/, "")
else
mail_from_line = data
end
@mail_from = mail_from_line.gsub(/MAIL FROM\s*:\s*/i, '').gsub(/.*</, '').gsub(/>.*/, '').strip
'250 OK'
@mail_from = mail_from_line.gsub(/MAIL FROM\s*:\s*/i, "").gsub(/.*</, "").gsub(/>.*/, "").strip
"250 OK"
end
def rcpt_to(data)
unless in_state(:mail_from_received, :rcpt_to_received)
return '503 EHLO/HELO and MAIL FROM first please'
return "503 EHLO/HELO and MAIL FROM first please"
end
rcpt_to = data.gsub(/RCPT TO\s*:\s*/i, '').gsub(/.*</, '').gsub(/>.*/, '').strip
rcpt_to = data.gsub(/RCPT TO\s*:\s*/i, "").gsub(/.*</, "").gsub(/>.*/, "").strip
if rcpt_to.blank?
return '501 RCPT TO should not be empty'
return "501 RCPT TO should not be empty"
end
uname, domain = rcpt_to.split('@', 2)
uname, domain = rcpt_to.split("@", 2)
if domain.blank?
return '501 Invalid RCPT TO'
return "501 Invalid RCPT TO"
end
uname, tag = uname.split('+', 2)
uname, tag = uname.split("+", 2)
if domain == Postal.config.dns.return_path || domain =~ /\A#{Regexp.escape(Postal.config.dns.custom_return_path_prefix)}\./
# This is a return path
@state = :rcpt_to_received
if server = ::Server.where(:token => uname).first
if server = ::Server.where(token: uname).first
if server.suspended?
'535 Mail server has been suspended'
"535 Mail server has been suspended"
else
log "Added bounce on server #{server.id}"
@recipients << [:bounce, rcpt_to, server]
'250 OK'
"250 OK"
end
else
'550 Invalid server token'
"550 Invalid server token"
end
elsif domain == Postal.config.dns.route_domain
# This is an email direct to a route. This isn't actually supported yet.
@state = :rcpt_to_received
if route = Route.where(:token => uname).first
if route = Route.where(token: uname).first
if route.server.suspended?
'535 Mail server has been suspended'
elsif route.mode == 'Reject'
'550 Route does not accept incoming messages'
"535 Mail server has been suspended"
elsif route.mode == "Reject"
"550 Route does not accept incoming messages"
else
log "Added route #{route.id} to recipients (tag: #{tag.inspect})"
actual_rcpt_to = "#{route.name}" + (tag ? "+#{tag}" : "") + "@#{route.domain.name}"
@recipients << [:route, actual_rcpt_to, route.server, :route => route]
'250 OK'
@recipients << [:route, actual_rcpt_to, route.server, { route: route }]
"250 OK"
end
else
'550 Invalid route token'
"550 Invalid route token"
end
elsif @credential
# This is outgoing mail for an authenticated user
@state = :rcpt_to_received
if @credential.server.suspended?
'535 Mail server has been suspended'
"535 Mail server has been suspended"
else
log "Added external address '#{rcpt_to}'"
@recipients << [:credential, rcpt_to, @credential.server]
'250 OK'
"250 OK"
end
elsif uname && domain && route = Route.find_by_name_and_domain(uname, domain)
# This is incoming mail for a route
@state = :rcpt_to_received
if route.server.suspended?
'535 Mail server has been suspended'
elsif route.mode == 'Reject'
'550 Route does not accept incoming messages'
"535 Mail server has been suspended"
elsif route.mode == "Reject"
"550 Route does not accept incoming messages"
else
log "Added route #{route.id} to recipients (tag: #{tag.inspect})"
@recipients << [:route, rcpt_to, route.server, :route => route]
'250 OK'
@recipients << [:route, rcpt_to, route.server, { route: route }]
"250 OK"
end
else
# User is trying to relay but is not authenticated. Try to authenticate by IP address
@credential = Credential.where(:type => 'SMTP-IP').all.sort_by { |c| c.ipaddr&.prefix || 0 }.reverse.find do |credential|
@credential = Credential.where(type: "SMTP-IP").all.sort_by { |c| c.ipaddr&.prefix || 0 }.reverse.find do |credential|
credential.ipaddr.include?(@ip_address)
end
@@ -358,33 +363,33 @@ module Postal
@credential.use
rcpt_to(data)
else
'530 Authentication required'
"530 Authentication required"
end
end
end
def data(data)
unless in_state(:rcpt_to_received)
return '503 HELO/EHLO, MAIL FROM and RCPT TO before sending data'
return "503 HELO/EHLO, MAIL FROM and RCPT TO before sending data"
end
@data = "".force_encoding("BINARY")
@headers = {}
@receiving_headers = true
received_header_content = "from #{@helo_name} (#{@hostname} [#{@ip_address}]) by #{Postal.config.dns.smtp_server_hostname} with SMTP; #{Time.now.utc.rfc2822.to_s}".force_encoding('BINARY')
if !Postal.config.smtp_server.strip_received_headers?
received_header_content = "from #{@helo_name} (#{@hostname} [#{@ip_address}]) by #{Postal.config.dns.smtp_server_hostname} with SMTP; #{Time.now.utc.rfc2822}".force_encoding("BINARY")
unless Postal.config.smtp_server.strip_received_headers?
@data << "Received: #{received_header_content}\r\n"
end
@headers['received'] = [received_header_content]
@headers["received"] = [received_header_content]
handler = Proc.new do |data|
if data == '.'
handler = proc do |data|
if data == "."
@logging_enabled = true
@proc = nil
finished
else
data = data.to_s.sub(/\A\.\./, '.')
data = data.to_s.sub(/\A\.\./, ".")
if @credential && @credential.server.log_smtp_data?
# We want to log if enabled
@@ -405,7 +410,7 @@ module Postal
# skip the append methods at the bottom of this loop.
next if Postal.config.smtp_server.strip_received_headers? && @header_key && @header_key.downcase == "received"
else
@header_key, value = data.split(/\:\s*/, 2)
@header_key, value = data.split(/:\s*/, 2)
@headers[@header_key.downcase] ||= []
@headers[@header_key.downcase] << value
# As above
@@ -419,7 +424,7 @@ module Postal
end
@proc = handler
'354 Go ahead'
"354 Go ahead"
end
def finished
@@ -429,10 +434,10 @@ module Postal
return "552 Message too large (maximum size %dMB)" % Postal.config.smtp_server.max_message_size
end
if @headers['received'].select { |r| r =~ /by #{Postal.config.dns.smtp_server_hostname}/ }.count > 4
if @headers["received"].select { |r| r =~ /by #{Postal.config.dns.smtp_server_hostname}/ }.count > 4
transaction_reset
@state = :welcomed
return '550 Loop detected'
return "550 Loop detected"
end
authenticated_domain = nil
@@ -441,7 +446,7 @@ module Postal
if authenticated_domain.nil?
transaction_reset
@state = :welcomed
return '530 From/Sender name is not valid'
return "530 From/Sender name is not valid"
end
end
@@ -456,13 +461,13 @@ module Postal
message.mail_from = @mail_from
message.raw_message = @data
message.received_with_ssl = @tls
message.scope = 'outgoing'
message.scope = "outgoing"
message.domain_id = authenticated_domain&.id
message.credential_id = @credential.id
message.save
when :bounce
if rp_route = server.routes.where(:name => "__returnpath__").first
if rp_route = server.routes.where(name: "__returnpath__").first
# If there's a return path route, we can use this to create the message
rp_route.create_messages do |message|
message.rcpt_to = rcpt_to
@@ -478,7 +483,7 @@ module Postal
message.mail_from = @mail_from
message.raw_message = @data
message.received_with_ssl = @tls
message.scope = 'incoming'
message.scope = "incoming"
message.bounce = 1
message.save
end
@@ -493,7 +498,7 @@ module Postal
end
transaction_reset
@state = :welcomed
'250 OK'
"250 OK"
end
def in_state(*states)
lib/postal/smtp_server/server.rb
+68 -51
View File
@@ -1,5 +1,5 @@
require 'ipaddr'
require 'nio'
require "ipaddr"
require "nio"
module Postal
module SMTPServer
@@ -18,11 +18,11 @@ module Postal
trap("USR1") do
STDOUT.puts "Received USR1 signal, respawning."
fork do
if ENV['APP_ROOT']
Dir.chdir(ENV['APP_ROOT'])
if ENV["APP_ROOT"]
Dir.chdir(ENV["APP_ROOT"])
end
ENV.delete('BUNDLE_GEMFILE')
exec("bundle exec --keep-file-descriptors rake postal:smtp_server", :close_others => false)
ENV.delete("BUNDLE_GEMFILE")
exec("bundle exec --keep-file-descriptors rake postal:smtp_server", close_others: false)
end
end
@@ -30,7 +30,6 @@ module Postal
STDOUT.puts "Received TERM signal, shutting down."
unlisten
end
end
def ssl_context
@@ -38,7 +37,7 @@ module Postal
ssl_context = OpenSSL::SSL::SSLContext.new
ssl_context.cert = Postal.smtp_certificates[0]
ssl_context.extra_chain_cert = Postal.smtp_certificates[1..-1]
ssl_context.key = Postal.smtp_private_key
ssl_context.key = Postal.smtp_private_key
ssl_context.ssl_version = Postal.config.smtp_server.ssl_version if Postal.config.smtp_server.ssl_version
ssl_context.ciphers = Postal.config.smtp_server.tls_ciphers if Postal.config.smtp_server.tls_ciphers
ssl_context
@@ -46,8 +45,8 @@ module Postal
end
def listen
if ENV['SERVER_FD']
@server = TCPServer.for_fd(ENV['SERVER_FD'].to_i)
if ENV["SERVER_FD"]
@server = TCPServer.for_fd(ENV["SERVER_FD"].to_i)
else
@server = TCPServer.open(Postal.config.smtp_server.bind_address, Postal.config.smtp_server.port)
end
@@ -61,7 +60,7 @@ module Postal
@server.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPINTVL, 10)
@server.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPCNT, 5)
end
ENV['SERVER_FD'] = @server.to_i.to_s
ENV["SERVER_FD"] = @server.to_i.to_s
logger.info "Listening on #{Postal.config.smtp_server.bind_address}:#{Postal.config.smtp_server.port}"
end
@@ -72,7 +71,7 @@ module Postal
end
def kill_parent
Process.kill('TERM', Process.ppid)
Process.kill("TERM", Process.ppid)
end
def run_event_loop
@@ -81,7 +80,7 @@ module Postal
# Register the SMTP listener
@io_selector.register(@server, :r)
# Create a hash to contain a buffer for each client.
buffers = Hash.new { |h, k| h[k] = String.new.force_encoding('BINARY') }
buffers = Hash.new { |h, k| h[k] = String.new.force_encoding("BINARY") }
loop do
# Wait for an event to occur
@io_selector.select do |monitor|
@@ -112,17 +111,25 @@ module Postal
# Register the client and its socket with nio4r
monitor = @io_selector.register(new_io, :r)
monitor.value = client
rescue => e
rescue StandardError => e
# If something goes wrong, log as appropriate and disconnect the client
if defined?(Raven)
Raven.capture_exception(e, :extra => {:log_id => (client.id rescue nil)})
Raven.capture_exception(e, extra: { log_id: begin
client.id
rescue StandardError
nil
end })
end
logger.error "An error occurred while accepting a new client."
logger.error "#{e.class}: #{e.message}"
e.backtrace.each do |line|
logger.error line
end
new_io.close rescue nil
begin
new_io.close
rescue StandardError
nil
end
end
else
# This event is not an incoming connection so it must be data from a client
@@ -135,7 +142,7 @@ module Postal
if client.start_tls?
begin
# Can we accept the TLS connection at this time?
io.accept_nonblock()
io.accept_nonblock
# We were able to accept the connection, the client is no longer handshaking
client.start_tls = false
rescue IO::WaitReadable, IO::WaitWritable => e
@@ -153,12 +160,10 @@ module Postal
# There is an extra step for SSL sockets
case io
when OpenSSL::SSL::SSLSocket
buffers[io] << io.readpartial(10240)
while(io.pending > 0)
buffers[io] << io.readpartial(10240)
end
buffers[io] << io.readpartial(10_240)
buffers[io] << io.readpartial(10_240) while io.pending > 0
else
buffers[io] << io.readpartial(10240)
buffers[io] << io.readpartial(10_240)
end
rescue EOFError, Errno::ECONNRESET, Errno::ETIMEDOUT
# Client went away
@@ -167,7 +172,7 @@ module Postal
# Normalize all \r\n and \n to \r\n, but ignore only \r.
# A \r\n may be split in 2 buffers (\n in one buffer and \r in the other)
buffers[io] = buffers[io].gsub(/\r/,"").encode(buffers[io].encoding, crlf_newline: true)
buffers[io] = buffers[io].gsub(/\r/, "").encode(buffers[io].encoding, crlf_newline: true)
# We line buffer, so look to see if we have received a newline
# and keep doing so until all buffered lines have been processed.
@@ -178,17 +183,17 @@ module Postal
# Send the received line to the client object for processing
result = client.handle(line)
# If the client object returned some data, write it back to the client
unless result.nil?
result = [result] unless result.is_a?(Array)
result.compact.each do |line|
client.log "\e[34m=> #{line.strip}\e[0m"
begin
io.write(line.to_s + "\r\n")
io.flush
rescue Errno::ECONNRESET
# Client disconnected before we could write response
eof = true
end
next if result.nil?
result = [result] unless result.is_a?(Array)
result.compact.each do |line|
client.log "\e[34m=> #{line.strip}\e[0m"
begin
io.write(line.to_s + "\r\n")
io.flush
rescue Errno::ECONNRESET
# Client disconnected before we could write response
eof = true
end
end
end
@@ -219,11 +224,15 @@ module Postal
Process.exit(0)
end
end
rescue => e
rescue StandardError => e
# Something went wrong, log as appropriate
client_id = client ? client.id : '------'
client_id = client ? client.id : "------"
if defined?(Raven)
Raven.capture_exception(e, :extra => {:log_id => (client.id rescue nil)})
Raven.capture_exception(e, extra: { log_id: begin
client.id
rescue StandardError
nil
end })
end
logger.error "[#{client_id}] An error occurred while processing data from a client."
logger.error "[#{client_id}] #{e.class}: #{e.message}"
@@ -231,9 +240,17 @@ module Postal
logger.error "[#{client_id}] #{line}"
end
# Close all IO and forget this client
@io_selector.deregister(io) rescue nil
begin
@io_selector.deregister(io)
rescue StandardError
nil
end
buffers.delete(io)
io.close rescue nil
begin
io.close
rescue StandardError
nil
end
if @io_selector.empty?
Process.exit(0)
end
@@ -241,27 +258,27 @@ module Postal
end
end
# If unlisten has been called, stop listening
if $unlisten
@io_selector.deregister(@server)
@server.close
# If there's nothing left to do, shut down the process
if @io_selector.empty?
Process.exit(0)
end
# Clear the request
$unlisten = false
next unless $unlisten
@io_selector.deregister(@server)
@server.close
# If there's nothing left to do, shut down the process
if @io_selector.empty?
Process.exit(0)
end
# Clear the request
$unlisten = false
end
end
def run
# Write PID to file if path specified
if ENV['PID_FILE']
File.open(ENV['PID_FILE'], 'w') { |f| f.write(Process.pid.to_s + "\n") }
if ENV["PID_FILE"]
File.write(ENV["PID_FILE"], Process.pid.to_s + "\n")
end
# If we have been spawned to replace an existing processm shut down the
# parent after listening.
if ENV['SERVER_FD']
if ENV["SERVER_FD"]
listen
kill_parent
else
lib/postal/spam_check.rb
+3 -3
View File
@@ -11,9 +11,9 @@ module Postal
def to_hash
{
:code => code,
:score => score,
:description => description
code: code,
score: score,
description: description
}
end
lib/postal/tracking_middleware.rb
+39 -42
View File
@@ -1,27 +1,27 @@
module Postal
class TrackingMiddleware
TRACKING_PIXEL = File.read(Rails.root.join('app', 'assets', 'images', 'tracking_pixel.png'))
TRACKING_PIXEL = File.read(Rails.root.join("app", "assets", "images", "tracking_pixel.png"))
def initialize(app = nil)
@app = app
end
def call(env)
unless env['HTTP_X_POSTAL_TRACK_HOST'].to_i == 1
unless env["HTTP_X_POSTAL_TRACK_HOST"].to_i == 1
return @app.call(env)
end
request = Rack::Request.new(env)
case request.path
when /\A\/img\/([a-z0-9\-]+)\/([a-z0-9\-]+)/i
server_token = $1
message_token = $2
when /\A\/img\/([a-z0-9-]+)\/([a-z0-9-]+)/i
server_token = ::Regexp.last_match(1)
message_token = ::Regexp.last_match(2)
dispatch_image_request(request, server_token, message_token)
when /\A\/([a-z0-9\-]+)\/([a-z0-9\-]+)/i
server_token = $1
link_token = $2
when /\A\/([a-z0-9-]+)\/([a-z0-9-]+)/i
server_token = ::Regexp.last_match(1)
link_token = ::Regexp.last_match(2)
dispatch_redirect_request(request, server_token, link_token)
else
[200, {}, ["Hello."]]
@@ -33,72 +33,69 @@ module Postal
def dispatch_image_request(request, server_token, message_token)
message_db = get_message_db_from_server_token(server_token)
if message_db.nil?
return [404, {}, ['Invalid Server Token']]
return [404, {}, ["Invalid Server Token"]]
end
begin
message = message_db.message(:token => message_token)
message = message_db.message(token: message_token)
message.create_load(request)
rescue Postal::MessageDB::Message::NotFound
# This message has been removed, we'll just continue to serve the image
rescue => e
rescue StandardError => e
# Somethign else went wrong. We don't want to stop the image loading though because
# this is our problem. Log this exception though.
Raven.capture_exception(e) if defined?(Raven)
end
source_image = request.params['src']
source_image = request.params["src"]
case source_image
when nil
headers = {}
headers['Content-Type'] = "image/png"
headers['Content-Length'] = TRACKING_PIXEL.bytesize.to_s
return [200, headers, [TRACKING_PIXEL]]
when /\Ahttps?\:\/\//
response = Postal::HTTP.get(source_image, :timeout => 3)
if response[:code] == 200
headers = {}
headers['Content-Type'] = response[:headers]['content-type']&.first
headers['Last-Modified'] = response[:headers]['last-modified']&.first
headers['Cache-Control'] = response[:headers]['cache-control']&.first
headers['Etag'] = response[:headers]['etag']&.first
headers['Content-Length'] = response[:body].bytesize.to_s
return [200, headers, [response[:body]]]
else
return [404, {}, ['Not found']]
end
headers["Content-Type"] = "image/png"
headers["Content-Length"] = TRACKING_PIXEL.bytesize.to_s
[200, headers, [TRACKING_PIXEL]]
when /\Ahttps?:\/\//
response = Postal::HTTP.get(source_image, timeout: 3)
return [404, {}, ["Not found"]] unless response[:code] == 200
headers = {}
headers["Content-Type"] = response[:headers]["content-type"]&.first
headers["Last-Modified"] = response[:headers]["last-modified"]&.first
headers["Cache-Control"] = response[:headers]["cache-control"]&.first
headers["Etag"] = response[:headers]["etag"]&.first
headers["Content-Length"] = response[:body].bytesize.to_s
[200, headers, [response[:body]]]
else
return [400, {}, ['Invalid/missing source image']]
[400, {}, ["Invalid/missing source image"]]
end
end
def dispatch_redirect_request(request, server_token, link_token)
message_db = get_message_db_from_server_token(server_token)
if message_db.nil?
return [404, {}, ['Invalid Server Token']]
return [404, {}, ["Invalid Server Token"]]
end
link = message_db.select(:links, :where => {:token => link_token}, :limit => 1).first
link = message_db.select(:links, where: { token: link_token }, limit: 1).first
if link.nil?
return [404, {}, ['Link not found']]
return [404, {}, ["Link not found"]]
end
time = Time.now.to_f
if link['message_id']
message_db.update(:messages, {:clicked => time}, :where => {:id => link['message_id']})
message_db.insert(:clicks, {:message_id => link['message_id'], :link_id => link['id'], :ip_address => request.ip, :user_agent => request.user_agent, :timestamp => time})
SendWebhookJob.queue(:main, :server_id => message_db.server_id, :event => 'MessageLinkClicked', :payload => {:_message => link['message_id'], :url => link['url'], :token => link['token'], :ip_address => request.ip, :user_agent => request.user_agent})
if link["message_id"]
message_db.update(:messages, { clicked: time }, where: { id: link["message_id"] })
message_db.insert(:clicks, { message_id: link["message_id"], link_id: link["id"], ip_address: request.ip, user_agent: request.user_agent, timestamp: time })
SendWebhookJob.queue(:main, server_id: message_db.server_id, event: "MessageLinkClicked", payload: { _message: link["message_id"], url: link["url"], token: link["token"], ip_address: request.ip, user_agent: request.user_agent })
end
return [307, {'Location' => link['url']}, ["Redirected to: #{link['url']}"]]
[307, { "Location" => link["url"] }, ["Redirected to: #{link['url']}"]]
end
def get_message_db_from_server_token(token)
if server = ::Server.find_by_token(token)
server.message_db
else
nil
end
return unless server = ::Server.find_by_token(token)
server.message_db
end
end
lib/postal/user_creator.rb
+6 -6
View File
@@ -1,4 +1,4 @@
require 'highline'
require "highline"
module Postal
module UserCreator
@@ -10,10 +10,10 @@ module Postal
puts "This tool is usually only used to create your initial admin user."
puts
user = User.new
user.email_address = cli.ask("E-Mail Address".ljust(20, ' ') + ": ")
user.first_name = cli.ask("First Name".ljust(20, ' ') + ": ")
user.last_name = cli.ask("Last Name".ljust(20, ' ') + ": ")
user.password = cli.ask("Initial Password".ljust(20, ' ') + ": ") { |value| value.echo = '*' }
user.email_address = cli.ask("E-Mail Address".ljust(20, " ") + ": ")
user.first_name = cli.ask("First Name".ljust(20, " ") + ": ")
user.last_name = cli.ask("Last Name".ljust(20, " ") + ": ")
user.password = cli.ask("Initial Password".ljust(20, " ") + ": ") { |value| value.echo = "*" }
block.call(user) if block_given?
if user.save
@@ -28,7 +28,7 @@ module Postal
end
puts
end
end
end
end
lib/postal/version.rb
+8 -6
View File
@@ -1,12 +1,14 @@
module Postal
VERSION_PATH = File.expand_path('../../VERSION', __dir__)
VERSION = if File.file?(VERSION_PATH)
File.read(VERSION_PATH).strip.delete_prefix('v')
else
'0.0.0-dev'
end
VERSION_PATH = File.expand_path("../../VERSION", __dir__)
if File.file?(VERSION_PATH)
VERSION = File.read(VERSION_PATH).strip.delete_prefix("v")
else
VERSION = "0.0.0-dev"
end
def self.version
VERSION
end
end
lib/postal/worker.rb
+59 -55
View File
@@ -11,11 +11,17 @@ module Postal
def work
logger.info "Worker running with #{Postal.config.workers.threads} threads"
Signal.trap("INT") { @exit = true; set_process_name }
Signal.trap("TERM") { @exit = true; set_process_name }
Signal.trap("INT") do
@exit = true
set_process_name
end
Signal.trap("TERM") do
@exit = true
set_process_name
end
self.class.job_channel.prefetch(Postal.config.workers.threads)
@initial_queues.each { |queue | join_queue(queue) }
@initial_queues.each { |queue| join_queue(queue) }
exit_checks = 0
loop do
@@ -42,49 +48,51 @@ module Postal
private
def receive_job(delivery_info, properties, body)
begin
message = JSON.parse(body) rescue nil
if message && message['class_name']
@running_jobs << message['id']
set_process_name
start_time = Time.now
Thread.current[:job_id] = message['id']
logger.info "[#{message['id']}] Started processing \e[34m#{message['class_name']}\e[0m job"
begin
klass = message['class_name'].constantize.new(message['id'], message['params'])
klass.perform
GC.start
rescue => e
klass.on_error(e) if defined?(klass)
logger.warn "[#{message['id']}] \e[31m#{e.class}: #{e.message}\e[0m"
e.backtrace.each do |line|
logger.warn "[#{message['id']}] " + line
end
if defined?(Raven)
Raven.capture_exception(e, :extra => {:job_id => message['id']})
end
ensure
logger.info "[#{message['id']}] Finished processing \e[34m#{message['class_name']}\e[0m job in #{Time.now - start_time}s"
end
end
ensure
Thread.current[:job_id] = nil
self.class.job_channel.ack(delivery_info.delivery_tag)
@running_jobs.delete(message['id']) if message['id']
message = begin
JSON.parse(body)
rescue StandardError
nil
end
if message && message["class_name"]
@running_jobs << message["id"]
set_process_name
if @exit && @running_jobs.empty?
logger.info "Exiting because all jobs have finished."
exit 0
start_time = Time.now
Thread.current[:job_id] = message["id"]
logger.info "[#{message['id']}] Started processing \e[34m#{message['class_name']}\e[0m job"
begin
klass = message["class_name"].constantize.new(message["id"], message["params"])
klass.perform
GC.start
rescue StandardError => e
klass.on_error(e) if defined?(klass)
logger.warn "[#{message['id']}] \e[31m#{e.class}: #{e.message}\e[0m"
e.backtrace.each do |line|
logger.warn "[#{message['id']}] " + line
end
if defined?(Raven)
Raven.capture_exception(e, extra: { job_id: message["id"] })
end
ensure
logger.info "[#{message['id']}] Finished processing \e[34m#{message['class_name']}\e[0m job in #{Time.now - start_time}s"
end
end
ensure
Thread.current[:job_id] = nil
self.class.job_channel.ack(delivery_info.delivery_tag)
@running_jobs.delete(message["id"]) if message["id"]
set_process_name
if @exit && @running_jobs.empty?
logger.info "Exiting because all jobs have finished."
exit 0
end
end
def join_queue(queue)
if @active_queues[queue]
logger.info "Attempted to join queue #{queue} but already joined."
else
consumer = self.class.job_queue(queue).subscribe(:manual_ack => true) do |delivery_info, properties, body|
consumer = self.class.job_queue(queue).subscribe(manual_ack: true) do |delivery_info, properties, body|
receive_job(delivery_info, properties, body)
end
@active_queues[queue] = consumer
@@ -129,24 +137,22 @@ module Postal
need = id
elsif @unassigned_ips.include?(ip)
# We know this IP isn't valid. We don't need to do anything
else
elsif !self.class.local_ip?(ip) && ip_address = IPAddress.where("ipv4 = ? OR ipv6 = ?", ip, ip).first
# We need to look this up
if !self.class.local_ip?(ip) && ip_address = IPAddress.where("ipv4 = ? OR ipv6 = ?", ip, ip).first
@pairs[ip_address.ipv4] = ip_address.ipv6
@ip_to_id_mapping[ip] = ip_address.id
need = ip_address.id
else
@unassigned_ips << ip
end
@pairs[ip_address.ipv4] = ip_address.ipv6
@ip_to_id_mapping[ip] = ip_address.id
need = ip_address.id
else
@unassigned_ips << ip
end
if need
pair = @pairs[ip] || @pairs.key(ip)
if pair.nil? || current_ip_addresses.include?(pair)
needed_ip_ids << @ip_to_id_mapping[ip]
else
logger.info "Host has '#{ip}' but its pair (#{pair}) isn't here. Cannot add now."
end
next unless need
pair = @pairs[ip] || @pairs.key(ip)
if pair.nil? || current_ip_addresses.include?(pair)
needed_ip_ids << @ip_to_id_mapping[ip]
else
logger.info "Host has '#{ip}' but its pair (#{pair}) isn't here. Cannot add now."
end
end
@@ -197,9 +203,7 @@ module Postal
def self.job_queue(name)
@job_queues ||= {}
@job_queues[name] ||= begin
job_channel.queue("deliver-jobs-#{name}", :durable => true, :arguments => {'x-message-ttl' => 60000})
end
@job_queues[name] ||= job_channel.queue("deliver-jobs-#{name}", durable: true, arguments: { "x-message-ttl" => 60_000 })
end
def self.local_ip?(ip)
lib/tasks/auto_annotate_models.rake
+35 -35
View File
@@ -6,41 +6,41 @@ if Rails.env.development?
# You can override any of these by setting an environment variable of the
# same name.
Annotate.set_defaults(
'routes' => 'false',
'position_in_routes' => 'before',
'position_in_class' => 'before',
'position_in_test' => 'before',
'position_in_fixture' => 'before',
'position_in_factory' => 'before',
'position_in_serializer' => 'before',
'show_foreign_keys' => 'true',
'show_indexes' => 'true',
'simple_indexes' => 'false',
'model_dir' => 'app/models',
'root_dir' => '',
'include_version' => 'false',
'require' => '',
'exclude_tests' => 'false',
'exclude_fixtures' => 'false',
'exclude_factories' => 'false',
'exclude_serializers' => 'false',
'exclude_scaffolds' => 'true',
'exclude_controllers' => 'true',
'exclude_helpers' => 'true',
'ignore_model_sub_dir' => 'false',
'ignore_columns' => nil,
'ignore_routes' => nil,
'ignore_unknown_models' => 'false',
'hide_limit_column_types' => 'integer,boolean',
'skip_on_db_migrate' => 'false',
'format_bare' => 'true',
'format_rdoc' => 'false',
'format_markdown' => 'false',
'sort' => 'false',
'force' => 'false',
'trace' => 'false',
'wrapper_open' => nil,
'wrapper_close' => nil
"routes" => "false",
"position_in_routes" => "before",
"position_in_class" => "before",
"position_in_test" => "before",
"position_in_fixture" => "before",
"position_in_factory" => "before",
"position_in_serializer" => "before",
"show_foreign_keys" => "true",
"show_indexes" => "true",
"simple_indexes" => "false",
"model_dir" => "app/models",
"root_dir" => "",
"include_version" => "false",
"require" => "",
"exclude_tests" => "false",
"exclude_fixtures" => "false",
"exclude_factories" => "false",
"exclude_serializers" => "false",
"exclude_scaffolds" => "true",
"exclude_controllers" => "true",
"exclude_helpers" => "true",
"ignore_model_sub_dir" => "false",
"ignore_columns" => nil,
"ignore_routes" => nil,
"ignore_unknown_models" => "false",
"hide_limit_column_types" => "integer,boolean",
"skip_on_db_migrate" => "false",
"format_bare" => "true",
"format_rdoc" => "false",
"format_markdown" => "false",
"sort" => "false",
"force" => "false",
"trace" => "false",
"wrapper_open" => nil,
"wrapper_close" => nil
)
end
lib/tasks/postal.rake
+16 -15
View File
@@ -1,25 +1,27 @@
namespace :postal do
desc "Start the cron worker"
task :cron => :environment do
require 'clockwork'
require Rails.root.join('config', 'cron')
trap('TERM') { puts "Exiting..."; Process.exit(0) }
task cron: :environment do
require "clockwork"
require Rails.root.join("config", "cron")
trap("TERM") do
puts "Exiting..."
Process.exit(0)
end
Clockwork.run
end
desc 'Start SMTP Server'
task :smtp_server => :environment do
Postal::SMTPServer::Server.new(:debug => true).run
desc "Start SMTP Server"
task smtp_server: :environment do
Postal::SMTPServer::Server.new(debug: true).run
end
desc 'Start the message requeuer'
task :requeuer => :environment do
desc "Start the message requeuer"
task requeuer: :environment do
Postal::MessageRequeuer.new.run
end
desc 'Run all migrations on message databases'
task :migrate_message_databases => :environment do
desc "Run all migrations on message databases"
task migrate_message_databases: :environment do
Server.all.each do |server|
puts "\e[35m-------------------------------------------------------------------\e[0m"
puts "\e[35m#{server.id}: #{server.name} (#{server.permalink})\e[0m"
@@ -27,9 +29,8 @@ namespace :postal do
server.message_db.provisioner.migrate
end
end
end
Rake::Task['db:migrate'].enhance do
Rake::Task['postal:migrate_message_databases'].invoke
Rake::Task["db:migrate"].enhance do
Rake::Task["postal:migrate_message_databases"].invoke
end