feat: new configuration system (and schema) (#2819)

spec/lib/message_dequeuer/single_message_processor_spec.rb

@@ -42,7 +42,7 @@ module MessageDequeuer
     end
 
     context "when the number of attempts is more than the maximum" do
-      let(:queued_message) { create(:queued_message, :locked, message: message, attempts: Postal.config.general.maximum_delivery_attempts + 1) }
+      let(:queued_message) { create(:queued_message, :locked, message: message, attempts: Postal::Config.postal.default_maximum_delivery_attempts + 1) }
 
       it "logs" do
         processor.process

spec/lib/postal/legacy_config_source_spec.rb

@@ -0,0 +1,346 @@
+# frozen_string_literal: true
+
+require "rails_helper"
+
+module Postal
+
+  SOURCE_CONFIG = YAML.safe_load(File.read(Rails.root.join("spec/examples/full_legacy_config_file.yml")))
+
+  # Rather than actuall test the LegacyConfigSource directly, I have decided
+  # to test this source via. the Konfig::Config system to ensure it works as
+  # expected in practice rather than just in theory. Testing '#get' would be
+  # fairly easy (and mostly pointless) where as testing the values we actually
+  # want are correct is preferred.
+  RSpec.describe LegacyConfigSource do
+    before do
+      # For the purposes of testing, we want to ignore any defaults provided
+      # by the schema itself. Otherwise, we might see a value returned that
+      # looks correct but is actually the default rather than the value from
+      # config file.
+      allow_any_instance_of(Konfig::SchemaAttribute).to receive(:default).and_return(nil)
+    end
+
+    let(:source) { described_class.new(SOURCE_CONFIG) }
+    subject(:config) { Konfig::Config.build(ConfigSchema, sources: [source]) }
+
+    describe "the 'postal' group" do
+      it "returns a value for postal.web_hostname" do
+        expect(config.postal.web_hostname).to eq "postal.llamas.com"
+      end
+
+      it "returns a value for postal.web_protocol" do
+        expect(config.postal.web_protocol).to eq "https"
+      end
+
+      it "returns a value for postal.smtp_hostname" do
+        expect(config.postal.smtp_hostname).to eq "smtp.postal.llamas.com"
+      end
+
+      it "returns a value for postal.use_ip_pools?" do
+        expect(config.postal.use_ip_pools?).to eq false
+      end
+
+      it "returns a value for postal.default_maximum_delivery_attempts" do
+        expect(config.postal.default_maximum_delivery_attempts).to eq 20
+      end
+
+      it "returns a value for postal.default_maximum_hold_expiry_days" do
+        expect(config.postal.default_maximum_hold_expiry_days).to eq 10
+      end
+
+      it "returns a value for postal.default_suppression_list_automatic_removal_days" do
+        expect(config.postal.default_suppression_list_automatic_removal_days).to eq 60
+      end
+
+      it "returns a value for postal.use_local_ns_for_domain_verification?" do
+        expect(config.postal.use_local_ns_for_domain_verification?).to eq true
+      end
+
+      it "returns a value for postal.default_spam_threshold" do
+        expect(config.postal.default_spam_threshold).to eq 10
+      end
+
+      it "returns a value for postal.default_spam_failure_threshold" do
+        expect(config.postal.default_spam_failure_threshold).to eq 25
+      end
+
+      it "returns a value for postal.use_resent_sender_header?" do
+        expect(config.postal.use_resent_sender_header?).to eq true
+      end
+
+      it "returns a value for postal.smtp_relays" do
+        expect(config.postal.smtp_relays).to eq [
+          { "host" => "1.2.3.4", "port" => 25, "ssl_mode" => "Auto" },
+          { "host" => "2.2.2.2", "port" => 2525, "ssl_mode" => "None" }
+        ]
+      end
+    end
+
+    describe "the 'web_server' group" do
+      it "returns a value for web_server.default_bind_address" do
+        expect(config.web_server.default_bind_address).to eq "127.0.0.1"
+      end
+
+      it "returns a value for web_server.default_port" do
+        expect(config.web_server.default_port).to eq 6000
+      end
+
+      it "returns a value for web_server.max_threads" do
+        expect(config.web_server.max_threads).to eq 10
+      end
+    end
+
+    describe "the 'main_db' group" do
+      it "returns a value for main_db.host" do
+        expect(config.main_db.host).to eq "localhost"
+      end
+
+      it "returns a value for main_db.port" do
+        expect(config.main_db.port).to eq 3306
+      end
+
+      it "returns a value for main_db.username" do
+        expect(config.main_db.username).to eq "postal"
+      end
+
+      it "returns a value for main_db.password" do
+        expect(config.main_db.password).to eq "t35tpassword"
+      end
+
+      it "returns a value for main_db.database" do
+        expect(config.main_db.database).to eq "postal"
+      end
+
+      it "returns a value for main_db.pool_size" do
+        expect(config.main_db.pool_size).to eq 20
+      end
+
+      it "returns a value for main_db.encoding" do
+        expect(config.main_db.encoding).to eq "utf8mb4"
+      end
+    end
+
+    describe "the 'message_db' group" do
+      it "returns a value for message_db.host" do
+        expect(config.message_db.host).to eq "localhost"
+      end
+
+      it "returns a value for message_db.port" do
+        expect(config.message_db.port).to eq 3306
+      end
+
+      it "returns a value for message_db.username" do
+        expect(config.message_db.username).to eq "postal"
+      end
+
+      it "returns a value for message_db.password" do
+        expect(config.message_db.password).to eq "p05t41"
+      end
+
+      it "returns a value for message_db.database_name_prefix" do
+        expect(config.message_db.database_name_prefix).to eq "postal"
+      end
+    end
+
+    describe "the 'logging' group" do
+      it "returns a value for logging.rails_log_enabled" do
+        expect(config.logging.rails_log_enabled).to eq true
+      end
+    end
+
+    describe "the 'gelf' group" do
+      it "returns a value for gelf.host" do
+        expect(config.gelf.host).to eq "logs.llamas.com"
+      end
+
+      it "returns a value for gelf.port" do
+        expect(config.gelf.port).to eq 12_201
+      end
+
+      it "returns a value for gelf.facility" do
+        expect(config.gelf.facility).to eq "mailer"
+      end
+    end
+
+    describe "the 'smtp_server' group" do
+      it "returns a value for smtp_server.default_port" do
+        expect(config.smtp_server.default_port).to eq 25
+      end
+
+      it "returns a value for smtp_server.default_bind_address" do
+        expect(config.smtp_server.default_bind_address).to eq "127.0.0.1"
+      end
+
+      it "returns a value for smtp_server.tls_enabled" do
+        expect(config.smtp_server.tls_enabled).to eq true
+      end
+
+      it "returns a value for smtp_server.tls_certificate_path" do
+        expect(config.smtp_server.tls_certificate_path).to eq "config/smtp.cert"
+      end
+
+      it "returns a value for smtp_server.tls_private_key_path" do
+        expect(config.smtp_server.tls_private_key_path).to eq "config/smtp.key"
+      end
+
+      it "returns a value for smtp_server.tls_ciphers" do
+        expect(config.smtp_server.tls_ciphers).to eq "abc"
+      end
+
+      it "returns a value for smtp_server.ssl_version" do
+        expect(config.smtp_server.ssl_version).to eq "SSLv23"
+      end
+
+      it "returns a value for smtp_server.proxy_protocol" do
+        expect(config.smtp_server.proxy_protocol).to eq false
+      end
+
+      it "returns a value for smtp_server.log_connections" do
+        expect(config.smtp_server.log_connections).to eq true
+      end
+
+      it "returns a value for smtp_server.max_message_size" do
+        expect(config.smtp_server.max_message_size).to eq 10
+      end
+    end
+
+    describe "the 'dns' group" do
+      it "returns a value for dns.mx_records" do
+        expect(config.dns.mx_records).to eq ["mx1.postal.llamas.com", "mx2.postal.llamas.com"]
+      end
+
+      it "returns a value for dns.spf_include" do
+        expect(config.dns.spf_include).to eq "spf.postal.llamas.com"
+      end
+
+      it "returns a value for dns.return_path_domain" do
+        expect(config.dns.return_path_domain).to eq "rp.postal.llamas.com"
+      end
+
+      it "returns a value for dns.route_domain" do
+        expect(config.dns.route_domain).to eq "routes.postal.llamas.com"
+      end
+
+      it "returns a value for dns.track_domain" do
+        expect(config.dns.track_domain).to eq "track.postal.llamas.com"
+      end
+
+      it "returns a value for dns.helo_hostname" do
+        expect(config.dns.helo_hostname).to eq "helo.postal.llamas.com"
+      end
+
+      it "returns a value for dns.dkim_identifier" do
+        expect(config.dns.dkim_identifier).to eq "postal"
+      end
+
+      it "returns a value for dns.domain_verify_prefix" do
+        expect(config.dns.domain_verify_prefix).to eq "postal-verification"
+      end
+
+      it "returns a value for dns.custom_return_path_prefix" do
+        expect(config.dns.custom_return_path_prefix).to eq "psrp"
+      end
+    end
+
+    describe "the 'smtp' group" do
+      it "returns a value for smtp.host" do
+        expect(config.smtp.host).to eq "127.0.0.1"
+      end
+
+      it "returns a value for smtp.port" do
+        expect(config.smtp.port).to eq 25
+      end
+
+      it "returns a value for smtp.username" do
+        expect(config.smtp.username).to eq "postalserver"
+      end
+
+      it "returns a value for smtp.password" do
+        expect(config.smtp.password).to eq "llama"
+      end
+
+      it "returns a value for smtp.from_name" do
+        expect(config.smtp.from_name).to eq "Postal"
+      end
+
+      it "returns a value for smtp.from_address" do
+        expect(config.smtp.from_address).to eq "postal@llamas.com"
+      end
+    end
+
+    describe "the 'rails' group" do
+      it "returns a value for rails.environment" do
+        expect(config.rails.environment).to eq "production"
+      end
+
+      it "returns a value for rails.secret_key" do
+        expect(config.rails.secret_key).to eq "abcdef123123123123123"
+      end
+    end
+
+    describe "the 'rspamd' group" do
+      it "returns a value for rspamd.enabled" do
+        expect(config.rspamd.enabled).to eq true
+      end
+
+      it "returns a value for rspamd.host" do
+        expect(config.rspamd.host).to eq "rspamd.llamas.com"
+      end
+
+      it "returns a value for rspamd.port" do
+        expect(config.rspamd.port).to eq 11_334
+      end
+
+      it "returns a value for rspamd.ssl?" do
+        expect(config.rspamd.ssl?).to eq false
+      end
+
+      it "returns a value for rspamd.password" do
+        expect(config.rspamd.password).to eq "llama"
+      end
+
+      it "returns a value for rspamd.flags" do
+        expect(config.rspamd.flags).to eq "abc"
+      end
+    end
+
+    describe "the 'spamd' group" do
+      it "returns a value for spamd.enabled" do
+        expect(config.spamd.enabled).to eq false
+      end
+
+      it "returns a value for spamd.host" do
+        expect(config.spamd.host).to eq "spamd.llamas.com"
+      end
+
+      it "returns a value for spamd.port" do
+        expect(config.spamd.port).to eq 783
+      end
+    end
+
+    describe "the 'clamav' group" do
+      it "returns a value for clamav.enabled" do
+        expect(config.clamav.enabled).to eq false
+      end
+
+      it "returns a value for clamav.host" do
+        expect(config.clamav.host).to eq "clamav.llamas.com"
+      end
+
+      it "returns a value for clamav.port" do
+        expect(config.clamav.port).to eq 2000
+      end
+    end
+
+    describe "the 'smtp_client' group" do
+      it "returns a value for smtp_client.open_timeout" do
+        expect(config.smtp_client.open_timeout).to eq 60
+      end
+
+      it "returns a value for smtp_client.read_timeout" do
+        expect(config.smtp_client.read_timeout).to eq 120
+      end
+    end
+  end
+
+end

spec/lib/received_header_spec.rb

@@ -12,7 +12,7 @@ describe ReceivedHeader do
       it "returns the correct string" do
         result = described_class.generate(nil, "testhelo", "1.1.1.1", :smtp)
         expect(result).to eq "from testhelo (hostname.com [1.1.1.1]) " \
-                             "by #{Postal.config.dns.smtp_server_hostname} " \
+                             "by #{Postal::Config.postal.smtp_hostname} " \
                              "with SMTP; #{Time.now.utc.rfc2822}"
       end
     end
@@ -21,7 +21,7 @@ describe ReceivedHeader do
       it "returns the correct string" do
         server = Server.new(privacy_mode: true)
         result = described_class.generate(server, "testhelo", "1.1.1.1", :smtp)
-        expect(result).to eq "by #{Postal.config.dns.smtp_server_hostname} " \
+        expect(result).to eq "by #{Postal::Config.postal.smtp_hostname} " \
                              "with SMTP; #{Time.now.utc.rfc2822}"
       end
     end
@@ -31,7 +31,7 @@ describe ReceivedHeader do
         server = Server.new(privacy_mode: false)
         result = described_class.generate(server, "testhelo", "1.1.1.1", :smtp)
         expect(result).to eq "from testhelo (hostname.com [1.1.1.1]) " \
-                             "by #{Postal.config.dns.smtp_server_hostname} " \
+                             "by #{Postal::Config.postal.smtp_hostname} " \
                              "with SMTP; #{Time.now.utc.rfc2822}"
       end
     end
@@ -40,7 +40,7 @@ describe ReceivedHeader do
       it "returns the correct string" do
         result = described_class.generate(nil, "web-ui", "1.1.1.1", :http)
         expect(result).to eq "from web-ui (hostname.com [1.1.1.1]) " \
-                             "by #{Postal.config.web.host} " \
+                             "by #{Postal::Config.postal.web_hostname} " \
                              "with HTTP; #{Time.now.utc.rfc2822}"
       end
     end

spec/lib/smtp_server/client/data_spec.rb

@@ -39,7 +39,7 @@ module SMTPServer
         client.handle("RCPT TO: #{route.name}@#{route.domain.name}")
         Timecop.freeze do
           client.handle("DATA")
-          expect(client.headers["received"]).to include "from test.example.com (1.2.3.4 [1.2.3.4]) by postal.example.com with SMTP; #{Time.now.utc.rfc2822}"
+          expect(client.headers["received"]).to include "from test.example.com (1.2.3.4 [1.2.3.4]) by #{Postal::Config.postal.smtp_hostname} with SMTP; #{Time.now.utc.rfc2822}"
         end
       end
 
@@ -72,7 +72,7 @@ module SMTPServer
             client.handle("This is some content for the message.")
             client.handle("It will keep going.")
             expect(client.instance_variable_get("@data")).to eq <<~DATA
-              Received: from test.example.com (1.2.3.4 [1.2.3.4]) by #{Postal.config.dns.smtp_server_hostname} with SMTP; #{Time.now.utc.rfc2822}\r
+              Received: from test.example.com (1.2.3.4 [1.2.3.4]) by #{Postal::Config.postal.smtp_hostname} with SMTP; #{Time.now.utc.rfc2822}\r
               Subject: Test\r
               \r
               This is some content for the message.\r

spec/lib/smtp_server/client/finished_spec.rb

@@ -24,7 +24,7 @@ module SMTPServer
     describe "when finished sending data" do
       context "when the data is larger than the maximum message size" do
         it "returns an error and resets the state" do
-          allow(Postal.config.smtp_server).to receive(:max_message_size).and_return(1)
+          allow(Postal::Config.smtp_server).to receive(:max_message_size).and_return(1)
           client.handle("DATA")
           client.handle("a" * 1024 * 1024 * 10)
           expect(client.handle(".")).to eq "552 Message too large (maximum size 1MB)"
@@ -34,10 +34,10 @@ module SMTPServer
       context "when a loop is detected" do
         it "returns an error and resets the state" do
           client.handle("DATA")
-          client.handle("Received: from example1.com by #{Postal.config.dns.smtp_server_hostname}")
-          client.handle("Received: from example2.com by #{Postal.config.dns.smtp_server_hostname}")
-          client.handle("Received: from example1.com by #{Postal.config.dns.smtp_server_hostname}")
-          client.handle("Received: from example2.com by #{Postal.config.dns.smtp_server_hostname}")
+          client.handle("Received: from example1.com by #{Postal::Config.postal.smtp_hostname}")
+          client.handle("Received: from example2.com by #{Postal::Config.postal.smtp_hostname}")
+          client.handle("Received: from example1.com by #{Postal::Config.postal.smtp_hostname}")
+          client.handle("Received: from example2.com by #{Postal::Config.postal.smtp_hostname}")
           client.handle("Subject: Test")
           client.handle("From: #{mail_from}")
           client.handle("To: #{rcpt_to}")
@@ -93,7 +93,7 @@ module SMTPServer
 
       context "when sending a bounce message" do
         let(:credential) { nil }
-        let(:rcpt_to) { "#{server.token}@#{Postal.config.dns.return_path}" }
+        let(:rcpt_to) { "#{server.token}@#{Postal::Config.dns.return_path_domain}" }
 
         context "when there is a return path route" do
           let(:domain) { create(:domain, owner: server) }
@@ -114,7 +114,7 @@ module SMTPServer
 
             queued_message = QueuedMessage.first
             expect(queued_message).to have_attributes(
-              domain: Postal.config.dns.return_path,
+              domain: Postal::Config.dns.return_path_domain,
               server: server
             )
 
@@ -145,7 +145,7 @@ module SMTPServer
 
             queued_message = QueuedMessage.first
             expect(queued_message).to have_attributes(
-              domain: Postal.config.dns.return_path,
+              domain: Postal::Config.dns.return_path_domain,
               server: server
             )
 

spec/lib/smtp_server/client/helo_spec.rb

@@ -11,7 +11,7 @@ module SMTPServer
     describe "HELO" do
       it "returns the hostname" do
         expect(client.state).to eq :welcome
-        expect(client.handle("HELO: test.example.com")).to eq "250 #{Postal.config.dns.smtp_server_hostname}"
+        expect(client.handle("HELO: test.example.com")).to eq "250 #{Postal::Config.postal.smtp_hostname}"
         expect(client.state).to eq :welcomed
       end
     end
@@ -24,7 +24,7 @@ module SMTPServer
 
       context "when TLS is enabled" do
         it "returns capabilities include starttls" do
-          allow(Postal.config.smtp_server).to receive(:tls_enabled?).and_return(true)
+          allow(Postal::Config.smtp_server).to receive(:tls_enabled?).and_return(true)
           expect(client.handle("EHLO test.example.com")).to eq ["250-My capabilities are",
                                                                 "250-STARTTLS",
                                                                 "250 AUTH CRAM-MD5 PLAIN LOGIN"]

spec/lib/smtp_server/client/rcpt_to_spec.rb

@@ -36,18 +36,18 @@ module SMTPServer
 
       context "when the RCPT TO address is the system return path host" do
         it "returns an error if the server does not exist" do
-          expect(client.handle("RCPT TO: nothing@#{Postal.config.dns.return_path}")).to eq "550 Invalid server token"
+          expect(client.handle("RCPT TO: nothing@#{Postal::Config.dns.return_path_domain}")).to eq "550 Invalid server token"
         end
 
         it "returns an error if the server is suspended" do
           server = create(:server, :suspended)
-          expect(client.handle("RCPT TO: #{server.token}@#{Postal.config.dns.return_path}"))
+          expect(client.handle("RCPT TO: #{server.token}@#{Postal::Config.dns.return_path_domain}"))
             .to eq "535 Mail server has been suspended"
         end
 
         it "adds a recipient if all OK" do
           server = create(:server)
-          address = "#{server.token}@#{Postal.config.dns.return_path}"
+          address = "#{server.token}@#{Postal::Config.dns.return_path_domain}"
           expect(client.handle("RCPT TO: #{address}")).to eq "250 OK"
           expect(client.recipients).to eq [[:bounce, address, server]]
           expect(client.state).to eq :rcpt_to_received
@@ -56,19 +56,19 @@ module SMTPServer
 
       context "when the RCPT TO address is on a host using the return path prefix" do
         it "returns an error if the server does not exist" do
-          address = "nothing@#{Postal.config.dns.custom_return_path_prefix}.example.com"
+          address = "nothing@#{Postal::Config.dns.custom_return_path_prefix}.example.com"
           expect(client.handle("RCPT TO: #{address}")).to eq "550 Invalid server token"
         end
 
         it "returns an error if the server is suspended" do
           server = create(:server, :suspended)
-          address = "#{server.token}@#{Postal.config.dns.custom_return_path_prefix}.example.com"
+          address = "#{server.token}@#{Postal::Config.dns.custom_return_path_prefix}.example.com"
           expect(client.handle("RCPT TO: #{address}")).to eq "535 Mail server has been suspended"
         end
 
         it "adds a recipient if all OK" do
           server = create(:server)
-          address = "#{server.token}@#{Postal.config.dns.custom_return_path_prefix}.example.com"
+          address = "#{server.token}@#{Postal::Config.dns.custom_return_path_prefix}.example.com"
           expect(client.handle("RCPT TO: #{address}")).to eq "250 OK"
           expect(client.recipients).to eq [[:bounce, address, server]]
           expect(client.state).to eq :rcpt_to_received
@@ -77,28 +77,28 @@ module SMTPServer
 
       context "when the RCPT TO address is within the route domain" do
         it "returns an error if the route token is invalid" do
-          address = "nothing@#{Postal.config.dns.route_domain}"
+          address = "nothing@#{Postal::Config.dns.route_domain}"
           expect(client.handle("RCPT TO: #{address}")).to eq "550 Invalid route token"
         end
 
         it "returns an error if the server is suspended" do
           server = create(:server, :suspended)
           route = create(:route, server: server)
-          address = "#{route.token}@#{Postal.config.dns.route_domain}"
+          address = "#{route.token}@#{Postal::Config.dns.route_domain}"
           expect(client.handle("RCPT TO: #{address}")).to eq "535 Mail server has been suspended"
         end
 
         it "returns an error if the route is set to Reject mail" do
           server = create(:server)
           route = create(:route, server: server, mode: "Reject")
-          address = "#{route.token}@#{Postal.config.dns.route_domain}"
+          address = "#{route.token}@#{Postal::Config.dns.route_domain}"
           expect(client.handle("RCPT TO: #{address}")).to eq "550 Route does not accept incoming messages"
         end
 
         it "adds a recipient if all OK" do
           server = create(:server)
           route = create(:route, server: server)
-          address = "#{route.token}+tag1@#{Postal.config.dns.route_domain}"
+          address = "#{route.token}+tag1@#{Postal::Config.dns.route_domain}"
           expect(client.handle("RCPT TO: #{address}")).to eq "250 OK"
           expect(client.recipients).to eq [[:route, "#{route.name}+tag1@#{route.domain.name}", server, { route: route }]]
           expect(client.state).to eq :rcpt_to_received