diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1f0dd0..e38fb83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: name: CI Image Build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: docker/setup-buildx-action@v2 @@ -37,13 +37,14 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max target: ci + platforms: linux/amd64 test: name: Test Suite runs-on: ubuntu-latest needs: build steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - uses: docker/login-action@v2 @@ -64,7 +65,8 @@ jobs: needs: [build] if: startsWith(github.ref, 'refs/heads/') steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + - uses: docker/setup-qemu-action@v3 - uses: docker/setup-buildx-action@v2 - uses: docker/login-action@v2 with: @@ -86,6 +88,7 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max target: full + platforms: linux/amd64 publish-image: name: Publish Image @@ -93,7 +96,8 @@ jobs: needs: [build, test, release-please] if: ${{ needs.release-please.outputs.release_created }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + - uses: docker/setup-qemu-action@v3 - uses: docker/setup-buildx-action@v2 - uses: docker/login-action@v2 with: @@ -111,3 +115,4 @@ jobs: target: full build-args: | VERSION=${{ needs.release-please.outputs.version }} + platforms: linux/amd64,linux/arm64/v8 diff --git a/.gitignore b/.gitignore index 7f537dc..21277fb 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ Procfile.local VERSION .rubocop-https* +.env* + diff --git a/.rubocop.yml b/.rubocop.yml index 25bb950..17ab0f2 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -6,6 +6,7 @@ AllCops: - "db/schema.rb" # Fixes missing gem exception when running Rubocop on GitHub Actions. - "vendor/bundle/**/*" + - lib/tasks/auto_annotate_models.rake # Always use double quotes Style/StringLiterals: diff --git a/.ruby-version b/.ruby-version index 667b8b1..be94e6f 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1,2 +1 @@ -3.2.1 - +3.2.2 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..c6befd2 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,53 @@ +# Contributing to Postal + +This doc explains how to go about running Postal in development to allow you to make contributions to the project. + +## Dependencies + +You will need a MySQL database server to get started. Postal needs to be able to make databases within that server whenever new mail servers are created so the permissions that you use should be suitable for that. + +You'll also need Ruby. Postal currently uses Ruby 3.2.2. Install that using whichever version manager takes your fancy - rbenv, asdf, rvm etc. + +## Clone + +You'll need to clone the repository + +``` +git clone git@github.com:postalserver/postal +``` + +Once cloned, you can install the Ruby dependencies using bundler. + +``` +bundle install +``` + +## Configuration + +Configuration is handled using a config file. This lives in `config/postal/postal.yml`. An example configuration file is provided in `config/examples/development.yml`. This example is for development use only and not an example for production use. + +You'll also need a key for signing. You can generate one of these like this: + +``` +openssl genrsa -out config/postal/signing.key 2048 +``` + +If you're running the tests (and you probably should be), you'll find an example file for test configuration in `config/examples/test.yml`. This should be placed in `config/postal/postal.test.yml` with the appropriate values. + +If you prefer, you can configure Postal using environment variables. These should be placed in `.env` or `.env.test` as apprpriate. + +## Running + +The neatest way to run postal is to ensure that `./bin` is your `$PATH` and then use one of the following commands. + +* `bin/dev` - will run all components of the application using Foreman +* `bin/postal` - will run the Postal binary providing access to running individual components or other tools. + +## Database initialization + +Use the commands below to initialize your database and make your first user. + +``` +postal initialize +postal make-user +``` diff --git a/Dockerfile b/Dockerfile index b1b17c1..a0a209e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ -FROM ruby:3.2.1-bullseye AS base +FROM ruby:3.2.2-bullseye AS base SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN apt-get update \ && apt-get install -y --no-install-recommends \ software-properties-common dirmngr apt-transport-https \ - && (curl -sL https://deb.nodesource.com/setup_14.x | bash -) \ + && (curl -sL https://deb.nodesource.com/setup_20.x | bash -) \ && rm -rf /var/lib/apt/lists/* # Install main dependencies @@ -31,12 +31,12 @@ RUN mkdir -p /opt/postal/app /opt/postal/config WORKDIR /opt/postal/app # Install bundler -RUN gem install bundler -v 2.4.9 --no-doc +RUN gem install bundler -v 2.5.6 --no-doc # Install the latest and active gem dependencies and re-run # the appropriate commands to handle installs. -COPY Gemfile Gemfile.lock ./ -RUN bundle config set force_ruby_platform true && bundle install -j 4 +COPY --chown=postal Gemfile Gemfile.lock ./ +RUN bundle install # Copy the application (and set permissions) COPY ./docker/wait-for.sh /docker-entrypoint.sh @@ -46,8 +46,11 @@ COPY --chown=postal . . ARG VERSION=unspecified RUN echo $VERSION > VERSION -# Set the path to the config -ENV POSTAL_CONFIG_ROOT=/config +# Set paths for when running in a container +ENV POSTAL_CONFIG_FILE_PATH=/config/postal.yml +ENV POSTAL_SIGNING_KEY_PATH=/config/signing.key +ENV SMTP_SERVER_TLS_CERTIFICATE_PATH=/config/smtp.cert +ENV SMTP_SERVER_TLS_PRIVATE_KEY_PATH=/config/smtp.key # Set the CMD ENTRYPOINT [ "/docker-entrypoint.sh" ] @@ -59,5 +62,5 @@ FROM base AS ci # full target - default if no --target option is given FROM base AS full -RUN POSTAL_SKIP_CONFIG_CHECK=1 RAILS_GROUPS=assets bundle exec rake assets:precompile +RUN RAILS_GROUPS=assets bundle exec rake assets:precompile RUN touch /opt/postal/app/public/assets/.prebuilt diff --git a/Gemfile b/Gemfile index 31b5bef..46eca03 100644 --- a/Gemfile +++ b/Gemfile @@ -3,36 +3,34 @@ source "https://rubygems.org" gem "authie" gem "autoprefixer-rails" -gem "basic_ssl" gem "bcrypt" -gem "bunny" -gem "changey" gem "chronic" -gem "clockwork" -gem "dotenv-rails" +gem "domain_name" +gem "dotenv" gem "dynamic_form" gem "encrypto_signo" gem "execjs", "~> 2.7", "< 2.8" -gem "foreman" gem "gelf" gem "haml" gem "hashie" gem "highline", require: false -gem "jwt" gem "kaminari" +gem "klogger-logger" +gem "konfig-config", "~> 2.0" gem "mail" gem "moonrope" gem "mysql2" gem "nifty-utils" gem "nilify_blanks" gem "nio4r" +gem "prometheus-client" gem "puma" -gem "rails", "= 6.1.7.6" -gem "resolv", "~> 0.2.1" +gem "rails", "= 7.0.8.1" +gem "resolv" gem "secure_headers" gem "sentry-rails" -gem "sentry-ruby" gem "turbolinks", "~> 5" +gem "webrick" group :development, :assets do gem "coffee-rails", "~> 5.0" @@ -41,10 +39,6 @@ group :development, :assets do gem "uglifier", ">= 1.3.0" end -group :development, :test do - gem "byebug" -end - group :development do gem "annotate" gem "database_cleaner", require: false @@ -53,5 +47,7 @@ group :development do gem "rspec-rails", require: false gem "rubocop" gem "rubocop-rails" + gem "shoulda-matchers" gem "timecop" + gem "webmock" end diff --git a/Gemfile.lock b/Gemfile.lock index 83cf9a0..172a522 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,87 +1,85 @@ GEM remote: https://rubygems.org/ specs: - actioncable (6.1.7.6) - actionpack (= 6.1.7.6) - activesupport (= 6.1.7.6) + actioncable (7.0.8.1) + actionpack (= 7.0.8.1) + activesupport (= 7.0.8.1) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailbox (6.1.7.6) - actionpack (= 6.1.7.6) - activejob (= 6.1.7.6) - activerecord (= 6.1.7.6) - activestorage (= 6.1.7.6) - activesupport (= 6.1.7.6) + actionmailbox (7.0.8.1) + actionpack (= 7.0.8.1) + activejob (= 7.0.8.1) + activerecord (= 7.0.8.1) + activestorage (= 7.0.8.1) + activesupport (= 7.0.8.1) mail (>= 2.7.1) - actionmailer (6.1.7.6) - actionpack (= 6.1.7.6) - actionview (= 6.1.7.6) - activejob (= 6.1.7.6) - activesupport (= 6.1.7.6) + net-imap + net-pop + net-smtp + actionmailer (7.0.8.1) + actionpack (= 7.0.8.1) + actionview (= 7.0.8.1) + activejob (= 7.0.8.1) + activesupport (= 7.0.8.1) mail (~> 2.5, >= 2.5.4) + net-imap + net-pop + net-smtp rails-dom-testing (~> 2.0) - actionpack (6.1.7.6) - actionview (= 6.1.7.6) - activesupport (= 6.1.7.6) - rack (~> 2.0, >= 2.0.9) + actionpack (7.0.8.1) + actionview (= 7.0.8.1) + activesupport (= 7.0.8.1) + rack (~> 2.0, >= 2.2.4) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.2.0) - actiontext (6.1.7.6) - actionpack (= 6.1.7.6) - activerecord (= 6.1.7.6) - activestorage (= 6.1.7.6) - activesupport (= 6.1.7.6) + actiontext (7.0.8.1) + actionpack (= 7.0.8.1) + activerecord (= 7.0.8.1) + activestorage (= 7.0.8.1) + activesupport (= 7.0.8.1) + globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (6.1.7.6) - activesupport (= 6.1.7.6) + actionview (7.0.8.1) + activesupport (= 7.0.8.1) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.1, >= 1.2.0) - activejob (6.1.7.6) - activesupport (= 6.1.7.6) + activejob (7.0.8.1) + activesupport (= 7.0.8.1) globalid (>= 0.3.6) - activemodel (6.1.7.6) - activesupport (= 6.1.7.6) - activerecord (6.1.7.6) - activemodel (= 6.1.7.6) - activesupport (= 6.1.7.6) - activestorage (6.1.7.6) - actionpack (= 6.1.7.6) - activejob (= 6.1.7.6) - activerecord (= 6.1.7.6) - activesupport (= 6.1.7.6) + activemodel (7.0.8.1) + activesupport (= 7.0.8.1) + activerecord (7.0.8.1) + activemodel (= 7.0.8.1) + activesupport (= 7.0.8.1) + activestorage (7.0.8.1) + actionpack (= 7.0.8.1) + activejob (= 7.0.8.1) + activerecord (= 7.0.8.1) + activesupport (= 7.0.8.1) marcel (~> 1.0) mini_mime (>= 1.1.0) - activesupport (6.1.7.6) + activesupport (7.0.8.1) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) - zeitwerk (~> 2.3) - amq-protocol (2.3.2) + addressable (2.8.6) + public_suffix (>= 2.0.2, < 6.0) annotate (3.2.0) activerecord (>= 3.2, < 8.0) rake (>= 10.4, < 14.0) ast (2.4.2) - authie (3.4.0) - secure_random_string + authie (4.1.3) + activerecord (>= 6.1, < 8.0) autoprefixer-rails (10.4.13.0) execjs (~> 2) - basic_ssl (1.0.3) - bcrypt (3.1.18) + bcrypt (3.1.20) + bigdecimal (3.1.6) builder (3.2.4) - bunny (2.20.3) - amq-protocol (~> 2.3, >= 2.3.1) - sorted_set (~> 1, >= 1.0.2) - byebug (11.1.3) - changey (1.1.0) - activerecord (>= 4.2, < 7) chronic (0.10.2) - clockwork (3.0.2) - activesupport - tzinfo coffee-rails (5.0.0) coffee-script (>= 2.2.0) railties (>= 5.2.0) @@ -90,6 +88,9 @@ GEM execjs coffee-script-source (1.12.2) concurrent-ruby (1.2.3) + crack (1.0.0) + bigdecimal + rexml crass (1.0.6) database_cleaner (2.0.2) database_cleaner-active_record (>= 2, < 3) @@ -97,13 +98,11 @@ GEM activerecord (>= 5.a) database_cleaner-core (~> 2.0.0) database_cleaner-core (2.0.1) - date (3.3.3) + date (3.3.4) deep_merge (1.2.2) diff-lcs (1.5.0) - dotenv (2.8.1) - dotenv-rails (2.8.1) - dotenv (= 2.8.1) - railties (>= 3.2) + domain_name (0.6.20240107) + dotenv (3.0.2) dynamic_form (1.3.1) actionview (> 5.2.0) activemodel (> 5.2.0) @@ -116,15 +115,15 @@ GEM factory_bot (~> 6.4) railties (>= 5.0.0) ffi (1.15.5) - foreman (0.87.2) gelf (3.1.0) json globalid (1.2.1) activesupport (>= 6.1) - haml (6.1.1) + haml (6.3.0) temple (>= 0.8.2) thor tilt + hashdiff (1.1.0) hashie (5.0.0) highline (2.1.0) i18n (1.14.1) @@ -133,8 +132,7 @@ GEM rails-dom-testing (>= 1, < 3) railties (>= 4.2.0) thor (>= 0.14, < 2.0) - json (2.6.3) - jwt (2.7.0) + json (2.7.1) kaminari (1.2.2) activesupport (>= 4.1.0) kaminari-actionview (= 1.2.2) @@ -147,6 +145,12 @@ GEM activerecord kaminari-core (= 1.2.2) kaminari-core (1.2.2) + klogger-logger (1.4.0) + concurrent-ruby (>= 1.0, < 2.0) + json + rouge (>= 3.30, < 5.0) + konfig-config (2.1.1) + hashie loofah (2.22.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) @@ -157,22 +161,22 @@ GEM net-smtp marcel (1.0.2) method_source (1.0.0) - mini_mime (1.1.2) + mini_mime (1.1.5) mini_portile2 (2.8.5) minitest (5.22.2) moonrope (2.0.2) deep_merge (~> 1.0) json rack (>= 1.4) - mysql2 (0.5.5) - net-imap (0.3.4) + mysql2 (0.5.6) + net-imap (0.4.10) date net-protocol net-pop (0.1.2) net-protocol - net-protocol (0.2.1) + net-protocol (0.2.2) timeout - net-smtp (0.3.3) + net-smtp (0.4.0.1) net-protocol nifty-utils (1.1.7) nilify_blanks (1.4.0) @@ -182,34 +186,39 @@ GEM nokogiri (1.16.2) mini_portile2 (~> 2.8.2) racc (~> 1.4) + nokogiri (1.16.2-aarch64-linux) + racc (~> 1.4) nokogiri (1.16.2-arm64-darwin) racc (~> 1.4) + nokogiri (1.16.2-x86_64-darwin) + racc (~> 1.4) nokogiri (1.16.2-x86_64-linux) racc (~> 1.4) parallel (1.22.1) parser (3.2.1.1) ast (~> 2.4.1) + prometheus-client (4.2.2) + public_suffix (5.0.4) puma (6.4.2) nio4r (~> 2.0) racc (1.7.3) - rack (2.2.8) + rack (2.2.8.1) rack-test (2.1.0) rack (>= 1.3) - rails (6.1.7.6) - actioncable (= 6.1.7.6) - actionmailbox (= 6.1.7.6) - actionmailer (= 6.1.7.6) - actionpack (= 6.1.7.6) - actiontext (= 6.1.7.6) - actionview (= 6.1.7.6) - activejob (= 6.1.7.6) - activemodel (= 6.1.7.6) - activerecord (= 6.1.7.6) - activestorage (= 6.1.7.6) - activesupport (= 6.1.7.6) + rails (7.0.8.1) + actioncable (= 7.0.8.1) + actionmailbox (= 7.0.8.1) + actionmailer (= 7.0.8.1) + actionpack (= 7.0.8.1) + actiontext (= 7.0.8.1) + actionview (= 7.0.8.1) + activejob (= 7.0.8.1) + activemodel (= 7.0.8.1) + activerecord (= 7.0.8.1) + activestorage (= 7.0.8.1) + activesupport (= 7.0.8.1) bundler (>= 1.15.0) - railties (= 6.1.7.6) - sprockets-rails (>= 2.0.0) + railties (= 7.0.8.1) rails-dom-testing (2.2.0) activesupport (>= 5.0.0) minitest @@ -217,18 +226,19 @@ GEM rails-html-sanitizer (1.6.0) loofah (~> 2.21) nokogiri (~> 1.14) - railties (6.1.7.6) - actionpack (= 6.1.7.6) - activesupport (= 6.1.7.6) + railties (7.0.8.1) + actionpack (= 7.0.8.1) + activesupport (= 7.0.8.1) method_source rake (>= 12.2) thor (~> 1.0) + zeitwerk (~> 2.5) rainbow (3.1.1) rake (13.1.0) - rbtree (0.4.6) regexp_parser (2.7.0) - resolv (0.2.2) + resolv (0.3.0) rexml (3.2.5) + rouge (4.2.0) rspec (3.12.0) rspec-core (~> 3.12.0) rspec-expectations (~> 3.12.0) @@ -278,16 +288,13 @@ GEM sprockets-rails tilt secure_headers (6.5.0) - secure_random_string (1.0.0) - sentry-rails (5.8.0) + sentry-rails (5.16.1) railties (>= 5.0) - sentry-ruby (~> 5.8.0) - sentry-ruby (5.8.0) + sentry-ruby (~> 5.16.1) + sentry-ruby (5.16.1) concurrent-ruby (~> 1.0, >= 1.0.2) - set (1.0.3) - sorted_set (1.0.3) - rbtree - set (~> 1.0) + shoulda-matchers (6.1.0) + activesupport (>= 5.2.0) sprockets (4.2.0) concurrent-ruby (~> 1.0) rack (>= 2.2.4, < 4) @@ -295,11 +302,11 @@ GEM actionpack (>= 5.2) activesupport (>= 5.2) sprockets (>= 3.0.0) - temple (0.10.0) + temple (0.10.3) thor (1.3.0) - tilt (2.1.0) + tilt (2.3.0) timecop (0.9.8) - timeout (0.3.2) + timeout (0.4.1) turbolinks (5.2.1) turbolinks-source (~> 5.2) turbolinks-source (5.2.0) @@ -308,52 +315,55 @@ GEM uglifier (4.2.0) execjs (>= 0.3.0, < 3) unicode-display_width (2.4.2) + webmock (3.20.0) + addressable (>= 2.8.0) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) + webrick (1.8.1) websocket-driver (0.7.6) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) zeitwerk (2.6.13) PLATFORMS - arm64-darwin-22 - arm64-darwin-23 + aarch64-linux + arm64-darwin ruby + x86_64-darwin x86_64-linux DEPENDENCIES annotate authie autoprefixer-rails - basic_ssl bcrypt - bunny - byebug - changey chronic - clockwork coffee-rails (~> 5.0) database_cleaner - dotenv-rails + domain_name + dotenv dynamic_form encrypto_signo execjs (~> 2.7, < 2.8) factory_bot_rails - foreman gelf haml hashie highline jquery-rails - jwt kaminari + klogger-logger + konfig-config (~> 2.0) mail moonrope mysql2 nifty-utils nilify_blanks nio4r + prometheus-client puma - rails (= 6.1.7.6) - resolv (~> 0.2.1) + rails (= 7.0.8.1) + resolv rspec rspec-rails rubocop @@ -361,10 +371,12 @@ DEPENDENCIES sass-rails secure_headers sentry-rails - sentry-ruby + shoulda-matchers timecop turbolinks (~> 5) uglifier (>= 1.3.0) + webmock + webrick BUNDLED WITH - 2.4.9 + 2.5.6 diff --git a/Procfile.dev b/Procfile.dev index 9f0e14a..4269444 100644 --- a/Procfile.dev +++ b/Procfile.dev @@ -1,5 +1,3 @@ -web: bundle exec puma -C config/puma.rb +web: unset PORT; bundle exec puma -C config/puma.rb worker: bundle exec ruby script/worker.rb -cron: bundle exec rake postal:cron -smtp: bundle exec rake postal:smtp_server -requeuer: bundle exec rake postal:requeuer +smtp: unset PORT; bundle exec ruby script/smtp_server.rb diff --git a/README.md b/README.md index 3996ee5..0e6a943 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,7 @@ - +![welcome](https://github.com/postalserver/postal/assets/4765/37e6c405-058a-43c6-be30-7149d873561d) **Postal** is a complete and fully featured mail server for use by websites & web servers. Think Sendgrid, Mailgun or Postmark but open source and ready for you to run on your own servers. Postal is developed by [Krystal](https://k.io) to serve its own mail processing requirements and we have since decided that it should be released as an open source project for the community. -![Screenshot](https://share.adam.ac/17/k4lA5OuPlU2.png) - * [Documentation](https://docs.postalserver.io) * [Installation Instructions](https://docs.postalserver.io/install/prerequisites) * [FAQs](https://docs.postalserver.io/welcome/faqs) & [Features](https://docs.postalserver.io/welcome/feature-list) diff --git a/SECURITY.md b/SECURITY.md index f75f979..7936c08 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,12 +2,12 @@ ## Supported Versions -We only support updates to the 2.x versions of Postal. +We only support updates to the 3.x versions of Postal. | Version | Supported | | ------- | ------------------ | -| 2.x.x | :white_check_mark: | -| < 2.0 | :x: | +| 3.x.x | :white_check_mark: | +| < 3.0 | :x: | ## Reporting a Vulnerability diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 3053575..d2c36cd 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -109,7 +109,8 @@ class ApplicationController < ActionController::Base auth_session.invalidate! reset_session end - Authie::Session.start(self, user: user) + + create_auth_session(user) @current_user = user end diff --git a/app/controllers/domains_controller.rb b/app/controllers/domains_controller.rb index f1ef258..146df1d 100644 --- a/app/controllers/domains_controller.rb +++ b/app/controllers/domains_controller.rb @@ -71,7 +71,7 @@ class DomainsController < ApplicationController when "Email" if params[:code] if @domain.verification_token == params[:code].to_s.strip - @domain.verify + @domain.mark_as_verified redirect_to_with_json [:setup, organization, @server, @domain], notice: "#{@domain.name} has been verified successfully. You now need to configure your DNS records." else respond_to do |wants| diff --git a/app/controllers/messages_controller.rb b/app/controllers/messages_controller.rb index 6a6c2f1..ea0e940 100644 --- a/app/controllers/messages_controller.rb +++ b/app/controllers/messages_controller.rb @@ -19,7 +19,7 @@ class MessagesController < ApplicationController @message.from = "test@#{domain.name}" end end - @message.subject = "Test Message at #{Time.zone.now.to_s(:long)}" + @message.subject = "Test Message at #{Time.zone.now.to_fs(:long)}" @message.plain_body = "This is a message to test the delivery of messages through Postal." end @@ -116,7 +116,7 @@ class MessagesController < ApplicationController def retry if @message.raw_message? if @message.queued_message - @message.queued_message.queue! + @message.queued_message.retry_now flash[:notice] = "This message will be retried shortly." elsif @message.held? @message.add_to_message_queue(manual: true) @@ -161,7 +161,7 @@ class MessagesController < ApplicationController if @query = (params[:query] || session["msg_query_#{@server.id}_#{scope}"]).presence session["msg_query_#{@server.id}_#{scope}"] = @query - qs = Postal::QueryString.new(@query) + qs = QueryString.new(@query) if qs.empty? flash.now[:alert] = "It doesn't appear you entered anything to filter on. Please double check your query." else diff --git a/app/controllers/sessions_controller.rb b/app/controllers/sessions_controller.rb index e873d7d..99a80f2 100644 --- a/app/controllers/sessions_controller.rb +++ b/app/controllers/sessions_controller.rb @@ -4,7 +4,7 @@ class SessionsController < ApplicationController layout "sub" - skip_before_action :login_required, only: [:new, :create, :create_with_token, :begin_password_reset, :finish_password_reset, :ip, :raise_error] + skip_before_action :login_required, only: [:new, :create, :begin_password_reset, :finish_password_reset, :ip, :raise_error] def create login(User.authenticate(params[:email_address], params[:password])) @@ -15,18 +15,6 @@ class SessionsController < ApplicationController render "new" end - def create_with_token - result = JWT.decode(params[:token], Postal.signing_key.to_s, "HS256")[0] - if result["timestamp"] > 1.minute.ago.to_f - login(User.find(result["user"].to_i)) - redirect_to root_path - else - destroy - end - rescue JWT::VerificationError - destroy - end - def destroy auth_session.invalidate! if logged_in? reset_session diff --git a/app/jobs/action_deletion_job.rb b/app/jobs/action_deletion_job.rb deleted file mode 100644 index e598a85..0000000 --- a/app/jobs/action_deletion_job.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -class ActionDeletionJob < Postal::Job - - def perform - object = params["type"].constantize.deleted.find_by_id(params["id"]) - if object - log "Deleting #{params['type']}##{params['id']}" - object.destroy - log "Deleted #{params['type']}##{params['id']}" - else - log "Couldn't find deleted object #{params['type']}##{params['id']}" - end - end - -end diff --git a/app/jobs/action_deletions_job.rb b/app/jobs/action_deletions_job.rb deleted file mode 100644 index f68b1d9..0000000 --- a/app/jobs/action_deletions_job.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -class ActionDeletionsJob < Postal::Job - - def perform - Organization.deleted.each do |org| - log "Permanently removing organization #{org.id} (#{org.permalink})" - org.destroy - end - - Server.deleted.each do |server| - log "Permanently removing server #{server.id} (#{server.full_permalink})" - server.destroy - end - end - -end diff --git a/app/jobs/prune_suppression_lists_job.rb b/app/jobs/prune_suppression_lists_job.rb deleted file mode 100644 index ece9db7..0000000 --- a/app/jobs/prune_suppression_lists_job.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -class PruneSuppressionListsJob < Postal::Job - - def perform - Server.all.each do |s| - log "Pruning suppression lists for server #{s.id}" - s.message_db.suppression_list.prune - end - end - -end diff --git a/app/jobs/prune_webhook_requests_job.rb b/app/jobs/prune_webhook_requests_job.rb deleted file mode 100644 index 632bc4a..0000000 --- a/app/jobs/prune_webhook_requests_job.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -class PruneWebhookRequestsJob < Postal::Job - - def perform - Server.all.each do |s| - log "Pruning webhook requests for server #{s.id}" - s.message_db.webhooks.prune - end - end - -end diff --git a/app/jobs/requeue_webhooks_job.rb b/app/jobs/requeue_webhooks_job.rb deleted file mode 100644 index 736f0b6..0000000 --- a/app/jobs/requeue_webhooks_job.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -class RequeueWebhooksJob < Postal::Job - - def perform - WebhookRequest.requeue_all - end - -end diff --git a/app/jobs/send_notifications_job.rb b/app/jobs/send_notifications_job.rb deleted file mode 100644 index 9d519c6..0000000 --- a/app/jobs/send_notifications_job.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -class SendNotificationsJob < Postal::Job - - def perform - Server.send_send_limit_notifications - end - -end diff --git a/app/jobs/send_webhook_job.rb b/app/jobs/send_webhook_job.rb deleted file mode 100644 index 8228563..0000000 --- a/app/jobs/send_webhook_job.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -class SendWebhookJob < Postal::Job - - def perform - if server = Server.find(params["server_id"]) - new_items = {} - params["payload"]&.each do |key, value| - next unless key.to_s =~ /\A_(\w+)/ - - begin - new_items[::Regexp.last_match(1)] = server.message_db.message(value.to_i).webhook_hash - rescue Postal::MessageDB::Message::NotFound - # No message found, don't do any replacement - end - end - - new_items.each do |key, value| - params["payload"].delete("_#{key}") - params["payload"][key] = value - end - - WebhookRequest.trigger(server, params["event"], params["payload"]) - else - log "Couldn't find server with ID #{params['server_id']}" - end - end - -end diff --git a/app/jobs/sleep_job.rb b/app/jobs/sleep_job.rb deleted file mode 100644 index 9604b76..0000000 --- a/app/jobs/sleep_job.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -class SleepJob < Postal::Job - - def perform - sleep 5 - end - -end diff --git a/app/jobs/tidy_raw_messages_job.rb b/app/jobs/tidy_raw_messages_job.rb deleted file mode 100644 index 151188f..0000000 --- a/app/jobs/tidy_raw_messages_job.rb +++ /dev/null @@ -1,8 +0,0 @@ -# frozen_string_literal: true - -class TidyRawMessagesJob < Postal::Job - - def perform - end - -end diff --git a/app/jobs/unqueue_message_job.rb b/app/jobs/unqueue_message_job.rb deleted file mode 100644 index 809ecfb..0000000 --- a/app/jobs/unqueue_message_job.rb +++ /dev/null @@ -1,468 +0,0 @@ -# frozen_string_literal: true - -class UnqueueMessageJob < Postal::Job - - # rubocop:disable Layout/LineLength - def perform - if original_message = QueuedMessage.find_by_id(params["id"]) - if original_message.acquire_lock - - log "Lock acquired for queued message #{original_message.id}" - - begin - original_message.message - rescue Postal::MessageDB::Message::NotFound - log "Unqueue #{original_message.id} because backend message has been removed." - original_message.destroy - return - end - - unless original_message.retriable? - log "Skipping because retry after isn't reached" - original_message.unlock - return - end - - begin - other_messages = original_message.batchable_messages(100) - log "Found #{other_messages.size} associated messages to process at the same time (batch key: #{original_message.batch_key})" - rescue StandardError - original_message.unlock - raise - end - - ([original_message] + other_messages).each do |queued_message| - log_prefix = "[#{queued_message.server_id}::#{queued_message.message_id} #{queued_message.id}]" - begin - log "#{log_prefix} Got queued message with exclusive lock" - - begin - queued_message.message - rescue Postal::MessageDB::Message::NotFound - log "#{log_prefix} Unqueueing #{queued_message.id} because backend message has been removed" - queued_message.destroy - next - end - - # - # If the server is suspended, hold all messages - # - if queued_message.server.suspended? - log "#{log_prefix} Server is suspended. Holding message." - queued_message.message.create_delivery("Held", details: "Mail server has been suspended. No e-mails can be processed at present. Contact support for assistance.") - queued_message.destroy - next - end - - # We might not be able to send this any more, check the attempts - if queued_message.attempts >= Postal.config.general.maximum_delivery_attempts - details = "Maximum number of delivery attempts (#{queued_message.attempts}) has been reached." - if queued_message.message.scope == "incoming" - # Send bounces to incoming e-mails when they are hard failed - if bounce_id = queued_message.send_bounce - details += " Bounce sent to sender (see message )" - end - elsif queued_message.message.scope == "outgoing" - # Add the recipient to the suppression list - if queued_message.server.message_db.suppression_list.add(:recipient, queued_message.message.rcpt_to, reason: "too many soft fails") - log "Added #{queued_message.message.rcpt_to} to suppression list because maximum attempts has been reached" - details += " Added #{queued_message.message.rcpt_to} to suppression list because delivery has failed #{queued_message.attempts} times." - end - end - queued_message.message.create_delivery("HardFail", details: details) - queued_message.destroy - log "#{log_prefix} Message has reached maximum number of attempts. Hard failing." - next - end - - # If the raw message has been removed (removed by retention) - unless queued_message.message.raw_message? - log "#{log_prefix} Raw message has been removed. Not sending." - queued_message.message.create_delivery("HardFail", details: "Raw message has been removed. Cannot send message.") - queued_message.destroy - next - end - - # - #  Handle Incoming Messages - # - if queued_message.message.scope == "incoming" - # - # If this is a bounce, we need to handle it as such - # - if queued_message.message.bounce - log "#{log_prefix} Message is a bounce" - original_messages = queued_message.message.original_messages - unless original_messages.empty? - queued_message.message.original_messages.each do |orig_msg| - queued_message.message.update(bounce_for_id: orig_msg.id, domain_id: orig_msg.domain_id) - queued_message.message.create_delivery("Processed", details: "This has been detected as a bounce message for .") - orig_msg.bounce!(queued_message.message) - log "#{log_prefix} Bounce linked with message #{orig_msg.id}" - end - queued_message.destroy - next - end - - # This message was sent to the return path but hasn't been matched - #  to an original message. If we have a route for this, route it - #  otherwise we'll drop at this point. - if queued_message.message.route_id.nil? - log "#{log_prefix} No source messages found. Hard failing." - queued_message.message.create_delivery("HardFail", details: "This message was a bounce but we couldn't link it with any outgoing message and there was no route for it.") - queued_message.destroy - next - end - end - - # - # Update live stats - # - queued_message.message.database.live_stats.increment(queued_message.message.scope) - - # - # Inspect incoming messages - # - unless queued_message.message.inspected - log "#{log_prefix} Inspecting message" - queued_message.message.inspect_message - if queued_message.message.inspected - is_spam = queued_message.message.spam_score > queued_message.server.spam_threshold - queued_message.message.update(spam: true) if is_spam - queued_message.message.append_headers( - "X-Postal-Spam: #{queued_message.message.spam ? 'yes' : 'no'}", - "X-Postal-Spam-Threshold: #{queued_message.server.spam_threshold}", - "X-Postal-Spam-Score: #{queued_message.message.spam_score}", - "X-Postal-Threat: #{queued_message.message.threat ? 'yes' : 'no'}" - ) - log "#{log_prefix} Message inspected successfully. Headers added." - end - end - - # - # If this message has a SPAM score higher than is permitted - # - if queued_message.message.spam_score >= queued_message.server.spam_failure_threshold - log "#{log_prefix} Message has a spam score higher than the server's maxmimum. Hard failing." - queued_message.message.create_delivery("HardFail", details: "Message's spam score is higher than the failure threshold for this server. Threshold is currently #{queued_message.server.spam_failure_threshold}.") - queued_message.destroy - next - end - - # If the server is in development mode, hold it - if queued_message.server.mode == "Development" && !queued_message.manual? - log "Server is in development mode so holding." - queued_message.message.create_delivery("Held", details: "Server is in development mode.") - queued_message.destroy - log "#{log_prefix} Server is in development mode. Holding." - next - end - - # - # Find out what sort of message we're supposed to be sending and dispatch this request over to - # the sender. - # - if route = queued_message.message.route - - # If the route says we're holding quananteed mail and this is spam, we'll hold this - if route.spam_mode == "Quarantine" && queued_message.message.spam && !queued_message.manual? - queued_message.message.create_delivery("Held", details: "Message placed into quarantine.") - queued_message.destroy - log "#{log_prefix} Route says to quarantine spam message. Holding." - next - end - - # If the route says we're holding quananteed mail and this is spam, we'll hold this - if route.spam_mode == "Fail" && queued_message.message.spam && !queued_message.manual? - queued_message.message.create_delivery("HardFail", details: "Message is spam and the route specifies it should be failed.") - queued_message.destroy - log "#{log_prefix} Route says to fail spam message. Hard failing." - next - end - - # - # Messages that should be blindly accepted are blindly accepted - # - if route.mode == "Accept" - queued_message.message.create_delivery("Processed", details: "Message has been accepted but not sent to any endpoints.") - queued_message.destroy - log "#{log_prefix} Route says to accept without endpoint. Marking as processed." - next - end - - # - # Messages that should be accepted and held should be held - # - if route.mode == "Hold" - log "#{log_prefix} Route says to hold message." - if queued_message.manual? - log "#{log_prefix} Message was queued manually. Marking as processed." - queued_message.message.create_delivery("Processed", details: "Message has been processed.") - else - log "#{log_prefix} Message was not queued manually. Holding." - queued_message.message.create_delivery("Held", details: "Message has been accepted but not sent to any endpoints.") - end - queued_message.destroy - next - end - - # - # Messages that should be bounced should be bounced (or rejected if they got this far) - # - if route.mode == "Bounce" || route.mode == "Reject" - if id = queued_message.send_bounce - queued_message.message.create_delivery("HardFail", details: "Message has been bounced because the route asks for this. See message ") - log "#{log_prefix} Route says to bounce. Hard failing and sent bounce (#{id})." - end - queued_message.destroy - next - end - - if @fixed_result - result = @fixed_result - else - case queued_message.message.endpoint - when SMTPEndpoint - sender = cached_sender(Postal::SMTPSender, queued_message.message.recipient_domain, nil, servers: [queued_message.message.endpoint]) - when HTTPEndpoint - sender = cached_sender(Postal::HTTPSender, queued_message.message.endpoint) - when AddressEndpoint - sender = cached_sender(Postal::SMTPSender, queued_message.message.endpoint.domain, nil, force_rcpt_to: queued_message.message.endpoint.address) - else - log "#{log_prefix} Invalid endpoint for route (#{queued_message.message.endpoint_type})" - queued_message.message.create_delivery("HardFail", details: "Invalid endpoint for route.") - queued_message.destroy - next - end - result = sender.send_message(queued_message.message) - if result.connect_error - @fixed_result = result - end - end - - # Log the result - log_details = result.details - if result.type == "HardFail" && result.suppress_bounce - # The delivery hard failed, but requested that no bounce be sent - log "#{log_prefix} Suppressing bounce message after hard fail" - elsif result.type == "HardFail" && queued_message.message.send_bounces? - # If the message is a hard fail, send a bounce message for this message. - log "#{log_prefix} Sending a bounce because message hard failed" - if bounce_id = queued_message.send_bounce - log_details += ". " unless log_details =~ /\.\z/ - log_details += " Sent bounce message to sender (see message )" - end - end - - queued_message.message.create_delivery(result.type, details: log_details, output: result.output&.strip, sent_with_ssl: result.secure, log_id: result.log_id, time: result.time) - - if result.retry - log "#{log_prefix} Message requeued for trying later." - queued_message.retry_later(result.retry.is_a?(Integer) ? result.retry : nil) - queued_message.allocate_ip_address - queued_message.update_column(:ip_address_id, queued_message.ip_address&.id) - else - log "#{log_prefix} Message processing completed." - queued_message.message.endpoint.mark_as_used - queued_message.destroy - end - else - log "#{log_prefix} No route and/or endpoint available for processing. Hard failing." - queued_message.message.create_delivery("HardFail", details: "Message does not have a route and/or endpoint available for delivery.") - queued_message.destroy - next - end - end - - # - # Handle Outgoing Messages - # - if queued_message.message.scope == "outgoing" - if queued_message.message.domain.nil? - log "#{log_prefix} Message has no domain. Hard failing." - queued_message.message.create_delivery("HardFail", details: "Message's domain no longer exist") - queued_message.destroy - next - end - - # - # If there's no to address, we can't do much. Fail it. - # - if queued_message.message.rcpt_to.blank? - log "#{log_prefix} Message has no to address. Hard failing." - queued_message.message.create_delivery("HardFail", details: "Message doesn't have an RCPT to") - queued_message.destroy - next - end - - # Extract a tag and add it to the message if one doesn't exist - if queued_message.message.tag.nil? && tag = queued_message.message.headers["x-postal-tag"] - log "#{log_prefix} Added tag #{tag.last}" - queued_message.message.update(tag: tag.last) - end - - # - # If the credentials for this message is marked as holding and this isn't manual, hold it - # - if !queued_message.manual? && queued_message.message.credential && queued_message.message.credential.hold? - log "#{log_prefix} Credential wants us to hold messages. Holding." - queued_message.message.create_delivery("Held", details: "Credential is configured to hold all messages authenticated by it.") - queued_message.destroy - next - end - - # - # If the recipient is on the suppression list and this isn't a manual queueing block sending - # - if !queued_message.manual? && sl = queued_message.server.message_db.suppression_list.get(:recipient, queued_message.message.rcpt_to) - log "#{log_prefix} Recipient is on the suppression list. Holding." - queued_message.message.create_delivery("Held", details: "Recipient (#{queued_message.message.rcpt_to}) is on the suppression list (reason: #{sl['reason']})") - queued_message.destroy - next - end - - # Parse the content of the message as appropriate - if queued_message.message.should_parse? - log "#{log_prefix} Parsing message content as it hasn't been parsed before" - queued_message.message.parse_content - end - - # Inspect outgoing messages when there's a threshold set for the server - if !queued_message.message.inspected && queued_message.server.outbound_spam_threshold - log "#{log_prefix} Inspecting message" - queued_message.message.inspect_message - if queued_message.message.inspected - if queued_message.message.spam_score >= queued_message.server.outbound_spam_threshold - queued_message.message.update(spam: true) - end - log "#{log_prefix} Message inspected successfully" - end - end - - if queued_message.message.spam - queued_message.message.create_delivery("HardFail", details: "Message is likely spam. Threshold is #{queued_message.server.outbound_spam_threshold} and the message scored #{queued_message.message.spam_score}.") - queued_message.destroy - log "#{log_prefix} Message is spam (#{queued_message.message.spam_score}). Hard failing." - next - end - - # Add outgoing headers - unless queued_message.message.has_outgoing_headers? - queued_message.message.add_outgoing_headers - end - - # Check send limits - if queued_message.server.send_limit_exceeded? - # If we're over the limit, we're going to be holding this message - queued_message.server.update_columns(send_limit_exceeded_at: Time.now, send_limit_approaching_at: nil) - queued_message.message.create_delivery("Held", details: "Message held because send limit (#{queued_message.server.send_limit}) has been reached.") - queued_message.destroy - log "#{log_prefix} Server send limit has been exceeded. Holding." - next - elsif queued_message.server.send_limit_approaching? - # If we're approaching the limit, just say we are but continue to process the message - queued_message.server.update_columns(send_limit_approaching_at: Time.now, send_limit_exceeded_at: nil) - else - queued_message.server.update_columns(send_limit_approaching_at: nil, send_limit_exceeded_at: nil) - end - - # Update the live stats for this message. - queued_message.message.database.live_stats.increment(queued_message.message.scope) - - # If the server is in development mode, hold it - if queued_message.server.mode == "Development" && !queued_message.manual? - log "Server is in development mode so holding." - queued_message.message.create_delivery("Held", details: "Server is in development mode.") - queued_message.destroy - log "#{log_prefix} Server is in development mode. Holding." - next - end - - # Send the outgoing message to the SMTP sender - - if @fixed_result - result = @fixed_result - else - sender = cached_sender(Postal::SMTPSender, queued_message.message.recipient_domain, queued_message.ip_address) - result = sender.send_message(queued_message.message) - if result.connect_error - @fixed_result = result - end - end - - # - # If the message has been hard failed, check to see how many other recent hard fails we've had for the address - # and if there are more than 2, suppress the address for 30 days. - # - if result.type == "HardFail" - recent_hard_fails = queued_message.server.message_db.select(:messages, where: { rcpt_to: queued_message.message.rcpt_to, status: "HardFail", timestamp: { greater_than: 24.hours.ago.to_f } }, count: true) - if recent_hard_fails >= 1 && queued_message.server.message_db.suppression_list.add(:recipient, queued_message.message.rcpt_to, reason: "too many hard fails") - log "#{log_prefix} Added #{queued_message.message.rcpt_to} to suppression list because #{recent_hard_fails} hard fails in 24 hours" - result.details += "." if result.details =~ /\.\z/ - result.details += " Recipient added to suppression list (too many hard fails)." - end - end - - # - # If a message is sent successfully, remove the users from the suppression list - # - if result.type == "Sent" && queued_message.server.message_db.suppression_list.remove(:recipient, queued_message.message.rcpt_to) - log "#{log_prefix} Removed #{queued_message.message.rcpt_to} from suppression list because success" - result.details += "." if result.details =~ /\.\z/ - result.details += " Recipient removed from suppression list." - end - - # Log the result - queued_message.message.create_delivery(result.type, details: result.details, output: result.output, sent_with_ssl: result.secure, log_id: result.log_id, time: result.time) - if result.retry - log "#{log_prefix} Message requeued for trying later." - queued_message.retry_later(result.retry.is_a?(Integer) ? result.retry : nil) - else - log "#{log_prefix} Processing complete" - queued_message.destroy - end - end - rescue StandardError => e - log "#{log_prefix} Internal error: #{e.class}: #{e.message}" - e.backtrace.each { |line| log("#{log_prefix} #{line}") } - queued_message.retry_later - log "#{log_prefix} Queued message was unlocked" - if defined?(Sentry) - Sentry.capture_exception(e, extra: { job_id: self.id, server_id: queued_message.server_id, message_id: queued_message.message_id }) - end - queued_message.message&.create_delivery("Error", - details: "An internal error occurred while sending " \ - "this message. This message will be retried " \ - "automatically.", - output: "#{e.class}: #{e.message}", log_id: "J-#{self.id}") - end - end - - else - log "Couldn't get lock for message #{params['id']}. I won't do this." - end - else - log "No queued message with ID #{params['id']} was available for processing." - end - ensure - begin - @sender&.finish - rescue StandardError - nil - end - end - # rubocop:enable Layout/LineLength - - private - - # rubocop:disable Naming/MemoizedInstanceVariableName - def cached_sender(klass, *args) - @sender ||= begin - sender = klass.new(*args) - sender.start - sender - end - end - # rubocop:enable Naming/MemoizedInstanceVariableName - -end diff --git a/app/jobs/webhook_delivery_job.rb b/app/jobs/webhook_delivery_job.rb deleted file mode 100644 index a7369f5..0000000 --- a/app/jobs/webhook_delivery_job.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -class WebhookDeliveryJob < Postal::Job - - def perform - if webhook_request = WebhookRequest.find_by_id(params["id"]) - if webhook_request.deliver - log "Succesfully delivered" - else - log "Delivery failed" - end - else - log "No webhook request found with ID '#{params['id']}'" - end - end - -end diff --git a/app/lib/dkim_header.rb b/app/lib/dkim_header.rb new file mode 100644 index 0000000..2aad2bb --- /dev/null +++ b/app/lib/dkim_header.rb @@ -0,0 +1,130 @@ +# frozen_string_literal: true + +class DKIMHeader + + def initialize(domain, message) + if domain && domain.dkim_status == "OK" + @domain_name = domain.name + @dkim_key = domain.dkim_key + @dkim_identifier = domain.dkim_identifier + else + @domain_name = Postal::Config.dns.return_path_domain + @dkim_key = Postal.signing_key + @dkim_identifier = Postal::Config.dns.dkim_identifier + end + @domain = domain + @message = message + @raw_headers, @raw_body = @message.gsub(/\r?\n/, "\r\n").split(/\r\n\r\n/, 2) + end + + def dkim_header + "DKIM-Signature: v=1; " + dkim_properties.join("\r\n\t") + signature.scan(/.{1,72}/).join("\r\n\t") + end + + private + + def headers + @headers ||= @raw_headers.to_s.gsub(/\r?\n\s/, " ").split(/\r?\n/) + end + + def header_names + normalized_headers.map { |h| h.split(":")[0].strip } + end + + def normalized_headers + [].tap do |new_headers| + dkim_headers = headers.select do |h| + h.match(/ + ^( + from|sender|reply-to|subject|date|message-id|to|cc|mime-version|content-type|content-transfer-encoding| + resent-to|resent-cc|resent-from|resent-sender|resent-message-id|in-reply-to|references|list-id|list-help| + list-owner|list-unsubscribe|list-subscribe|list-post + ):/ix) + end + dkim_headers.each do |h| + new_headers << normalize_header(h) + end + end + end + + def normalize_header(content) + content = content.dup + + # From the DKIM RFC6376 + # https://datatracker.ietf.org/doc/html/rfc6376#section-3.4.2 + + # Split the key and value. + key, value = content.split(":", 2) + + # Convert all header field names (not the header field values) to + # lowercase. For example, convert "SUBJect: AbC" to "subject: AbC". + key.downcase! + + # Unfold all header field continuation lines as described in [RFC5322] + value.gsub!(/\r?\n[ \t]+/, " ") + + # Convert all sequences of one or more WSP characters to a single SP character. + value.gsub!(/[ \t]+/, " ") + + # Delete all WSP characters at the end of each unfolded header field value. + value.gsub!(/[ \t]*\z/, "") + + # Delete any WSP characters remaining after the colon separating the header field name from the header field value. + value.gsub!(/\A[ \t]*/, "") + + # Join together + key + ":" + value + end + + def normalized_body + @normalized_body ||= begin + content = @raw_body.dup + + # From the DKIM RFC6376 + # https://datatracker.ietf.org/doc/html/rfc6376#section-3.4.4 + + # a. Reduce whitespace + # + # * Reduce all sequences of WSP within a line to a single SP character. + content.gsub!(/[ \t]+/, " ") + + # * Ignore all whitespace at the end of lines. Implementations MUST NOT + # remove the CRLF at the end of the line. + content.gsub!(/ \r\n/, "\r\n") + + # b. Ignore all empty lines at the end of the message body. + content.gsub!(/[ \r\n]*\z/, "") + + content += "\r\n" + content + end + end + + def body_hash + @body_hash ||= Base64.encode64(Digest::SHA256.digest(normalized_body)).strip + end + + def dkim_properties + @dkim_properties ||= [].tap do |header| + header << "a=rsa-sha256; c=relaxed/relaxed;" + header << "d=#{@domain_name};" + header << "s=#{@dkim_identifier}; t=#{Time.now.utc.to_i};" + header << "bh=#{body_hash};" + header << "h=#{header_names.join(':')};" + header << "b=" + end + end + + def dkim_header_for_signing + "dkim-signature:v=1; #{dkim_properties.join(' ')}" + end + + def signable_header_string + (normalized_headers + [dkim_header_for_signing]).join("\r\n") + end + + def signature + Base64.encode64(@dkim_key.sign(OpenSSL::Digest.new("SHA256"), signable_header_string)).gsub("\n", "") + end + +end diff --git a/app/lib/dns_resolver.rb b/app/lib/dns_resolver.rb new file mode 100644 index 0000000..1f7189a --- /dev/null +++ b/app/lib/dns_resolver.rb @@ -0,0 +1,161 @@ +# frozen_string_literal: true + +require "resolv" + +class DNSResolver + + class LocalResolversUnavailableError < StandardError + end + + attr_reader :nameservers + attr_reader :timeout + + def initialize(nameservers) + @nameservers = nameservers + end + + # Return all A records for the given name + # + # @param [String] name + # @return [Array] + def a(name, **options) + get_resources(name, Resolv::DNS::Resource::IN::A, **options).map do |s| + s.address.to_s + end + end + + # Return all AAAA records for the given name + # + # @param [String] name + # @return [Array] + def aaaa(name, **options) + get_resources(name, Resolv::DNS::Resource::IN::AAAA, **options).map do |s| + s.address.to_s + end + end + + # Return all TXT records for the given name + # + # @param [String] name + # @return [Array] + def txt(name, **options) + get_resources(name, Resolv::DNS::Resource::IN::TXT, **options).map do |s| + s.data.to_s.strip + end + end + + # Return all CNAME records for the given name + # + # @param [String] name + # @return [Array] + def cname(name, **options) + get_resources(name, Resolv::DNS::Resource::IN::CNAME, **options).map do |s| + s.name.to_s.downcase + end + end + + # Return all MX records for the given name + # + # @param [String] name + # @return [Array>] + def mx(name, **options) + records = get_resources(name, Resolv::DNS::Resource::IN::MX, **options).map do |m| + [m.preference.to_i, m.exchange.to_s] + end + records.sort do |a, b| + if a[0] == b[0] + [-1, 1].sample + else + a[0] <=> b[0] + end + end + end + + # Return the effective nameserver names for a given domain name. + # + # @param [String] name + # @return [Array] + def effective_ns(name, **options) + records = [] + parts = name.split(".") + (parts.size - 1).times do |n| + d = parts[n, parts.size - n + 1].join(".") + + records = get_resources(d, Resolv::DNS::Resource::IN::NS, **options).map do |s| + s.name.to_s + end + + break if records.present? + end + + records + end + + # Return the hostname for a given IP address. + # Returns the IP address itself if no hostname can be determined. + # + # @param [String] ip_address + # @return [String] + def ip_to_hostname(ip_address, **options) + dns(**options) do |dns| + dns.getname(ip_address)&.to_s + end + rescue Resolv::ResolvError => e + raise if e.message =~ /timeout/ && options[:raise_timeout_errors] + + ip_address + end + + private + + def dns(raise_timeout_errors: false) + Resolv::DNS.open(nameserver: @nameservers, + raise_timeout_errors: raise_timeout_errors) do |dns| + dns.timeouts = [Postal::Config.dns.timeout, + Postal::Config.dns.timeout / 2, + Postal::Config.dns.timeout / 2] + yield dns + end + end + + def get_resources(name, type, **options) + encoded_name = DomainName::Punycode.encode_hostname(name) + dns(**options) do |dns| + dns.getresources(encoded_name, type) + end + end + + class << self + + # Return a resolver which will use the nameservers for the given domain + # + # @param [String] name + # @return [DNSResolver] + def for_domain(name) + nameservers = local.effective_ns(name) + ips = nameservers.map do |ns| + local.a(ns) + end.flatten.uniq + new(ips) + end + + # Return a local resolver to use for lookups + # + # @return [DNSResolver] + def local + @local ||= begin + resolv_conf_path = Postal::Config.dns.resolv_conf_path + raise LocalResolversUnavailableError, "No resolver config found at #{resolv_conf_path}" unless File.file?(resolv_conf_path) + + resolv_conf = Resolv::DNS::Config.parse_resolv_conf(resolv_conf_path) + if resolv_conf.nil? || resolv_conf[:nameserver].nil? || resolv_conf[:nameserver].empty? + raise LocalResolversUnavailableError, "Could not find nameservers in #{resolv_conf_path}" + end + + new(resolv_conf[:nameserver]) + end + end + + end + +end diff --git a/app/lib/message_dequeuer.rb b/app/lib/message_dequeuer.rb new file mode 100644 index 0000000..1ab1f6e --- /dev/null +++ b/app/lib/message_dequeuer.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module MessageDequeuer + + class << self + + def process(message, logger:) + processor = InitialProcessor.new(message, logger: logger) + processor.process + end + + end + +end diff --git a/app/lib/message_dequeuer/base.rb b/app/lib/message_dequeuer/base.rb new file mode 100644 index 0000000..c2e84bc --- /dev/null +++ b/app/lib/message_dequeuer/base.rb @@ -0,0 +1,108 @@ +# frozen_string_literal: true + +module MessageDequeuer + class Base + + class StopProcessing < StandardError + end + + attr_reader :queued_message + attr_reader :logger + attr_reader :state + + def initialize(queued_message, logger:, state: nil) + @queued_message = queued_message + @logger = logger + @state = state || State.new + end + + def process + raise NotImplemented + end + + class << self + + def process(message, **kwargs) + new(message, **kwargs).process + end + + end + + private + + def stop_processing + raise StopProcessing + end + + def catch_stops + yield if block_given? + true + rescue StopProcessing + false + end + + def remove_from_queue + @queued_message.destroy + end + + def create_delivery(type, **kwargs) + @queued_message.message.create_delivery(type, **kwargs) + end + + def log(text, **tags) + logger.info text, **tags + end + + def increment_live_stats + queued_message.message.database.live_stats.increment(queued_message.message.scope) + end + + def hold_if_server_development_mode + return if queued_message.manual? + return unless queued_message.server.mode == "Development" + + log "server is in development mode, holding" + create_delivery "Held", details: "Server is in development mode." + remove_from_queue + stop_processing + end + + def log_sender_result + log_details = @result.details + + if @additional_delivery_details + log_details += "." unless log_details =~ /\.\z/ + log_details += " " + log_details += @additional_delivery_details + end + + create_delivery @result.type, details: log_details, + output: @result.output&.strip, + sent_with_ssl: @result.secure, + log_id: @result.log_id, + time: @result.time + end + + def handle_exception(exception) + log "internal error: #{exception.class}: #{exception.message}" + exception.backtrace.each { |line| log(line) } + + queued_message.retry_later unless queued_message.destroyed? + log "message requeued for trying later, at #{queued_message.retry_after}" + + if defined?(Sentry) + Sentry.capture_exception(exception, extra: { + server_id: queued_message.server_id, + queued_message_id: queued_message.message_id + }) + end + + queued_message.message&.create_delivery("Error", + details: "An internal error occurred while sending " \ + "this message. This message will be retried " \ + "automatically.", + output: "#{exception.class}: #{exception.message}") + end + + end +end diff --git a/app/lib/message_dequeuer/incoming_message_processor.rb b/app/lib/message_dequeuer/incoming_message_processor.rb new file mode 100644 index 0000000..d0aca58 --- /dev/null +++ b/app/lib/message_dequeuer/incoming_message_processor.rb @@ -0,0 +1,215 @@ +# frozen_string_literal: true + +module MessageDequeuer + class IncomingMessageProcessor < Base + + attr_reader :route + + def process + log "message is incoming" + + catch_stops do + handle_bounces + increment_live_stats + inspect_message + fail_if_spam + hold_if_server_development_mode + find_route + hold_or_reject_spam + accept_mail_without_endpoints + hold_messages + bounce_messages + send_message_to_sender + send_bounce_on_hard_fail + log_sender_result + finish_processing + end + rescue StandardError => e + handle_exception(e) + end + + private + + def handle_bounces + return unless queued_message.message.bounce + + log "message is a bounce" + original_messages = queued_message.message.original_messages + unless original_messages.empty? + queued_message.message.original_messages.each do |orig_msg| + queued_message.message.update(bounce_for_id: orig_msg.id, domain_id: orig_msg.domain_id) + create_delivery "Processed", details: "This has been detected as a bounce message for ." + orig_msg.bounce!(queued_message.message) + log "bounce linked with message #{orig_msg.id}" + end + remove_from_queue + stop_processing + end + + # This message was sent to the return path but hasn't been matched + # to an original message. If we have a route for this, route it + # otherwise we'll drop at this point. + return unless queued_message.message.route_id.nil? + + log "no source messages found, hard failing" + create_delivery "HardFail", details: "This message was a bounce but we couldn't link it with any outgoing message and there was no route for it." + remove_from_queue + stop_processing + end + + def inspect_message + return if queued_message.message.inspected + + log "inspecting message" + queued_message.message.inspect_message + return unless queued_message.message.inspected + + is_spam = queued_message.message.spam_score > queued_message.server.spam_threshold + if is_spam + queued_message.message.update(spam: true) + log "message is spam (scored #{queued_message.message.spam_score}, threshold is #{queued_message.server.spam_threshold})" + end + + queued_message.message.append_headers( + "X-Postal-Spam: #{queued_message.message.spam ? 'yes' : 'no'}", + "X-Postal-Spam-Threshold: #{queued_message.server.spam_threshold}", + "X-Postal-Spam-Score: #{queued_message.message.spam_score}", + "X-Postal-Threat: #{queued_message.message.threat ? 'yes' : 'no'}" + ) + log "message inspected, headers added", spam: queued_message.message.spam?, spam_score: queued_message.message.spam_score, threat: queued_message.message.threat? + end + + def fail_if_spam + return if queued_message.message.spam_score < queued_message.server.spam_failure_threshold + + log "message has a spam score higher than the server's maxmimum, hard failing", server_threshold: queued_message.server.spam_failure_threshold + create_delivery "HardFail", + details: "Message's spam score is higher than the failure threshold for this server. " \ + "Threshold is currently #{queued_message.server.spam_failure_threshold}." + remove_from_queue + stop_processing + end + + def find_route + @route = queued_message.message.route + return if @route + + log "no route and/or endpoint available for processing, hard failing" + create_delivery "HardFail", details: "Message does not have a route and/or endpoint available for delivery." + remove_from_queue + stop_processing + end + + def hold_or_reject_spam + return unless queued_message.message.spam + return if queued_message.manual? + + case @route.spam_mode + when "Quarantine" + log "message is spam and route says to quarantine spam message, holding" + create_delivery "Held", details: "Message placed into quarantine." + when "Fail" + log "message is spam and route says to fail spam message, hard failing" + create_delivery "HardFail", details: "Message is spam and the route specifies it should be failed." + else + return + end + + remove_from_queue + stop_processing + end + + def accept_mail_without_endpoints + return unless @route.mode == "Accept" + + log "route says to accept without endpoint, marking as processed" + create_delivery "Processed", details: "Message has been accepted but not sent to any endpoints." + remove_from_queue + stop_processing + end + + def hold_messages + return unless @route.mode == "Hold" + + if queued_message.manual? + log "route says to hold and message was queued manually, marking as processed" + create_delivery "Processed", details: "Message has been processed." + else + log "route says to hold, marking as held" + create_delivery "Held", details: "Message has been accepted but not sent to any endpoints." + end + + remove_from_queue + stop_processing + end + + def bounce_messages + return unless route.mode == "Bounce" || route.mode == "Reject" + + log "route says to bounce, hard failing and sending bounce" + + if id = queued_message.send_bounce + log "bounce sent with id #{id}" + create_delivery "HardFail", details: "Message has been bounced because the route asks for this. See message " + end + + remove_from_queue + stop_processing + end + + def send_message_to_sender + @result = @state.send_result + return if @result + + case queued_message.message.endpoint + when SMTPEndpoint + sender = @state.sender_for(SMTPSender, queued_message.message.recipient_domain, nil, servers: [queued_message.message.endpoint]) + when HTTPEndpoint + sender = @state.sender_for(HTTPSender, queued_message.message.endpoint) + when AddressEndpoint + sender = @state.sender_for(SMTPSender, queued_message.message.endpoint.domain, nil, rcpt_to: queued_message.message.endpoint.address) + else + log "invalid endpoint for route (#{queued_message.message.endpoint_type})" + create_delivery "HardFail", details: "Invalid endpoint for route." + remove_from_queue + stop_processing + end + + @result = sender.send_message(queued_message.message) + return unless @result.connect_error + + @state.send_result = @result + end + + def send_bounce_on_hard_fail + return unless @result.type == "HardFail" + + if @result.suppress_bounce + log "suppressing bounce message after hard fail" + return + end + + return unless queued_message.message.send_bounces? + + log "sending a bounce because message hard failed" + return unless bounce_id = queued_message.send_bounce + + @additional_delivery_details = "Sent bounce message to sender (see message )" + end + + def finish_processing + if @result.retry + queued_message.retry_later(@result.retry.is_a?(Integer) ? @result.retry : nil) + log "message requeued for trying later, at #{queued_message.retry_after}" + queued_message.allocate_ip_address + queued_message.update_column(:ip_address_id, queued_message.ip_address&.id) + stop_processing + end + + log "message processing completed" + queued_message.message.endpoint.mark_as_used + remove_from_queue + end + + end +end diff --git a/app/lib/message_dequeuer/initial_processor.rb b/app/lib/message_dequeuer/initial_processor.rb new file mode 100644 index 0000000..46ebda1 --- /dev/null +++ b/app/lib/message_dequeuer/initial_processor.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module MessageDequeuer + class InitialProcessor < Base + + attr_accessor :send_result + + def process + logger.tagged(original_queued_message: @queued_message.id) do + logger.info "starting message unqueue" + begin + catch_stops do + check_message_exists + check_message_is_ready + find_other_messages_for_batch + + # Process the original message and then all of those + # found for batching. + process_message(@queued_message) + @other_messages.each { |message| process_message(message) } + end + ensure + @state.finished + end + logger.info "finished message unqueue" + end + end + + private + + def check_message_exists + @queued_message.message + rescue Postal::MessageDB::Message::NotFound + log "unqueue because backend message has been removed." + remove_from_queue + stop_processing + end + + def check_message_is_ready + return if @queued_message.ready? + + log "skipping because message isn't ready for processing" + @queued_message.unlock + stop_processing + end + + def find_other_messages_for_batch + @other_messages = @queued_message.batchable_messages(100) + log "found #{@other_messages.size} associated messages to process at the same time", batch_key: @queued_message.batch_key + rescue StandardError + @queued_message.unlock + raise + end + + def process_message(queued_message) + logger.tagged(queued_message: queued_message.id) do + SingleMessageProcessor.process(queued_message, logger: @logger, state: @state) + end + end + + end +end diff --git a/app/lib/message_dequeuer/outgoing_message_processor.rb b/app/lib/message_dequeuer/outgoing_message_processor.rb new file mode 100644 index 0000000..e930b33 --- /dev/null +++ b/app/lib/message_dequeuer/outgoing_message_processor.rb @@ -0,0 +1,190 @@ +# frozen_string_literal: true + +module MessageDequeuer + class OutgoingMessageProcessor < Base + + def process + catch_stops do + check_domain + check_rcpt_to + add_tag + hold_if_credential_is_set_to_hold + hold_if_recipient_on_suppression_list + parse_content + inspect_message + fail_if_spam + add_outgoing_headers + check_send_limits + increment_live_stats + hold_if_server_development_mode + send_message_to_sender + add_recipient_to_suppression_list_on_too_many_hard_fails + remove_recipient_from_suppression_list_on_success + log_sender_result + finish_processing + end + rescue StandardError => e + handle_exception(e) + end + + private + + def check_domain + return if queued_message.message.domain + + log "message has no domain, hard failing" + create_delivery "HardFail", details: "Message's domain no longer exist" + remove_from_queue + stop_processing + end + + def check_rcpt_to + return unless queued_message.message.rcpt_to.blank? + + log "message has no 'to' address, hard failing" + create_delivery "HardFail", details: "Message doesn't have an RCPT to" + remove_from_queue + stop_processing + end + + def add_tag + return if queued_message.message.tag + return unless tag = queued_message.message.headers["x-postal-tag"] + + log "added tag: #{tag.last}" + queued_message.message.update(tag: tag.last) + end + + def hold_if_credential_is_set_to_hold + return if queued_message.manual? + return if queued_message.message.credential.nil? + return unless queued_message.message.credential.hold? + + log "credential wants us to hold messages, holding" + create_delivery "Held", details: "Credential is configured to hold all messages authenticated by it." + remove_from_queue + stop_processing + end + + def hold_if_recipient_on_suppression_list + return if queued_message.manual? + return unless sl = queued_message.server.message_db.suppression_list.get(:recipient, queued_message.message.rcpt_to) + + log "recipient is on the suppression list, holding" + create_delivery "Held", details: "Recipient (#{queued_message.message.rcpt_to}) is on the suppression list (reason: #{sl['reason']})" + remove_from_queue + stop_processing + end + + def parse_content + return unless queued_message.message.should_parse? + + log "parsing message content as it hasn't been parsed before" + queued_message.message.parse_content + end + + def inspect_message + return if queued_message.message.inspected + return unless queued_message.server.outbound_spam_threshold + + log "inspecting message" + queued_message.message.inspect_message + return unless queued_message.message.inspected + + if queued_message.message.spam_score >= queued_message.server.outbound_spam_threshold + queued_message.message.update(spam: true) + end + + log "message inspected successfully", spam: queued_message.message.spam?, spam_score: queued_message.message.spam_score + end + + def fail_if_spam + return unless queued_message.message.spam + + log "message is spam (#{queued_message.message.spam_score}), hard failing", server_threshold: queued_message.server.outbound_spam_threshold + create_delivery "HardFail", + details: "Message is likely spam. Threshold is #{queued_message.server.outbound_spam_threshold} and " \ + "the message scored #{queued_message.message.spam_score}." + remove_from_queue + stop_processing + end + + def add_outgoing_headers + return if queued_message.message.has_outgoing_headers? + + queued_message.message.add_outgoing_headers + end + + def check_send_limits + if queued_message.server.send_limit_exceeded? + # If we're over the limit, we're going to be holding this message + log "server send limit has been exceeded, holding", send_limit: queued_message.server.send_limit + queued_message.server.update_columns(send_limit_exceeded_at: Time.now, send_limit_approaching_at: nil) + create_delivery "Held", details: "Message held because send limit (#{queued_message.server.send_limit}) has been reached." + remove_from_queue + stop_processing + elsif queued_message.server.send_limit_approaching? + # If we're approaching the limit, just say we are but continue to process the message + queued_message.server.update_columns(send_limit_approaching_at: Time.now, send_limit_exceeded_at: nil) + else + queued_message.server.update_columns(send_limit_approaching_at: nil, send_limit_exceeded_at: nil) + end + end + + def send_message_to_sender + @result = @state.send_result + return if @result + + sender = @state.sender_for(SMTPSender, + queued_message.message.recipient_domain, + queued_message.ip_address) + + @result = sender.send_message(queued_message.message) + return unless @result.connect_error + + @state.send_result = @result + end + + def add_recipient_to_suppression_list_on_too_many_hard_fails + return unless @result.type == "HardFail" + + recent_hard_fails = queued_message.server.message_db.select(:messages, + where: { + rcpt_to: queued_message.message.rcpt_to, + status: "HardFail", + timestamp: { greater_than: 24.hours.ago.to_f } + }, + count: true) + return if recent_hard_fails < 1 + + added = queued_message.server.message_db.suppression_list.add(:recipient, queued_message.message.rcpt_to, + reason: "too many hard fails") + return unless added + + log "Added #{queued_message.message.rcpt_to} to suppression list because #{recent_hard_fails} hard fails in 24 hours" + @additional_delivery_details = "Recipient added to suppression list (too many hard fails)" + end + + def remove_recipient_from_suppression_list_on_success + return unless @result.type == "Sent" + + removed = queued_message.server.message_db.suppression_list.remove(:recipient, queued_message.message.rcpt_to) + return unless removed + + log "removed #{queued_message.message.rcpt_to} from suppression list" + @additional_delivery_details = "Recipient removed from suppression list" + end + + def finish_processing + if @result.retry + queued_message.retry_later(@result.retry.is_a?(Integer) ? @result.retry : nil) + log "message requeued for trying later", retry_after: queued_message.retry_after + stop_processing + end + + log "message processing complete" + remove_from_queue + end + + end +end diff --git a/app/lib/message_dequeuer/single_message_processor.rb b/app/lib/message_dequeuer/single_message_processor.rb new file mode 100644 index 0000000..68e1569 --- /dev/null +++ b/app/lib/message_dequeuer/single_message_processor.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module MessageDequeuer + class SingleMessageProcessor < Base + + def process + catch_stops do + check_message_exists + check_server_suspension + check_delivery_attempts + check_raw_message_exists + + processor = nil + case queued_message.message.scope + when "incoming" + processor = IncomingMessageProcessor + when "outgoing" + processor = OutgoingMessageProcessor + else + create_delivery "HardFail", details: "Scope #{queued_message.message.scope} is not valid" + remove_from_queue + stop_processing + end + + processor.process(queued_message, logger: @logger, state: @state) + end + rescue StandardError => e + handle_exception(e) + end + + private + + def check_message_exists + queued_message.message + rescue Postal::MessageDB::Message::NotFound + log "unqueueing because backend message has been removed" + remove_from_queue + stop_processing + end + + def check_server_suspension + return unless queued_message.server.suspended? + + log "server is suspended, holding message" + create_delivery "Held", details: "Mail server has been suspended. No e-mails can be processed at present. Contact support for assistance." + remove_from_queue + stop_processing + end + + def check_delivery_attempts + return if queued_message.attempts < Postal::Config.postal.default_maximum_delivery_attempts + + details = "Maximum number of delivery attempts (#{queued_message.attempts}) has been reached." + if queued_message.message.scope == "incoming" + # Send bounces to incoming e-mails when they are hard failed + if bounce_id = queued_message.send_bounce + details += " Bounce sent to sender (see message )" + end + elsif queued_message.message.scope == "outgoing" + # Add the recipient to the suppression list + if queued_message.server.message_db.suppression_list.add(:recipient, queued_message.message.rcpt_to, reason: "too many soft fails") + log "added #{queued_message.message.rcpt_to} to suppression list because maximum attempts has been reached" + details += " Added #{queued_message.message.rcpt_to} to suppression list because delivery has failed #{queued_message.attempts} times." + end + end + + log "message has reached maximum number of attempts, hard failing" + create_delivery "HardFail", details: details + remove_from_queue + stop_processing + end + + def check_raw_message_exists + return if queued_message.message.raw_message? + + log "raw message has been removed, not sending" + create_delivery "HardFail", details: "Raw message has been removed. Cannot send message." + remove_from_queue + stop_processing + end + + end +end diff --git a/app/lib/message_dequeuer/state.rb b/app/lib/message_dequeuer/state.rb new file mode 100644 index 0000000..f0b4ab6 --- /dev/null +++ b/app/lib/message_dequeuer/state.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module MessageDequeuer + class State + + attr_accessor :send_result + + def sender_for(klass, *args, **kwargs) + @cached_senders ||= {} + @cached_senders[[klass, args, kwargs]] ||= begin + klass_instance = klass.new(*args, **kwargs) + klass_instance.start + klass_instance + end + end + + def finished + @cached_senders&.each_value do |sender| + sender.finish + rescue StandardError + false + end + end + + end +end diff --git a/app/lib/query_string.rb b/app/lib/query_string.rb new file mode 100644 index 0000000..b2c9de3 --- /dev/null +++ b/app/lib/query_string.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +class QueryString + + def initialize(string) + @string = string.strip + " " + end + + def [](value) + hash[value.to_s] + end + + delegate :empty?, to: :hash + + def hash + @hash ||= @string.scan(/([a-z]+):\s*(?:(\d{2,4}-\d{2}-\d{2}\s\d{2}:\d{2})|"(.*?)"|(.*?))(\s|\z)/).each_with_object({}) do |(key, date, string_with_spaces, value), hash| + if date + actual_value = date + elsif string_with_spaces + actual_value = string_with_spaces + elsif value == "[blank]" + actual_value = nil + else + actual_value = value + end + + if hash.keys.include?(key.to_s) + hash[key.to_s] = [hash[key.to_s]].flatten + hash[key.to_s] << actual_value + else + hash[key.to_s] = actual_value + end + end + end + +end diff --git a/app/lib/received_header.rb b/app/lib/received_header.rb new file mode 100644 index 0000000..17eb1c7 --- /dev/null +++ b/app/lib/received_header.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class ReceivedHeader + + OUR_HOSTNAMES = { + smtp: Postal::Config.postal.smtp_hostname, + http: Postal::Config.postal.web_hostname + }.freeze + + class << self + + def generate(server, helo, ip_address, method) + our_hostname = OUR_HOSTNAMES[method] + if our_hostname.nil? + raise Error, "`method` is invalid (must be one of #{OUR_HOSTNAMES.join(', ')})" + end + + header = "by #{our_hostname} with #{method.to_s.upcase}; #{Time.now.utc.rfc2822}" + + if server.nil? || server.privacy_mode == false + hostname = DNSResolver.local.ip_to_hostname(ip_address) + header = "from #{helo} (#{hostname} [#{ip_address}]) #{header}" + end + + header + end + + end + +end diff --git a/app/lib/reply_separator.rb b/app/lib/reply_separator.rb new file mode 100644 index 0000000..b5167ed --- /dev/null +++ b/app/lib/reply_separator.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +class ReplySeparator + + RULES = [ + /^-{2,10} $.*/m, + /^>*\s*----- ?Original Message ?-----.*/m, + /^>*\s*From:[^\r\n]*[\r\n]+Sent:.*/m, + /^>*\s*From:[^\r\n]*[\r\n]+Date:.*/m, + /^>*\s*-----Urspr.ngliche Nachricht----- .*/m, + /^>*\s*Le[^\r\n]{10,200}a .crit ?:\s*$.*/, + /^>*\s*__________________.*/m, + /^>*\s*On.{10,200}wrote:\s*$.*/m, + /^>*\s*Sent from my.*/m, + /^>*\s*=== Please reply above this line ===.*/m, + /(^>.*\n?){10,}/ + ].freeze + + def self.separate(text) + return "" unless text.is_a?(String) + + text = text.gsub("\r", "") + stripped = String.new + RULES.each do |rule| + text.gsub!(rule) do + stripped = ::Regexp.last_match(0).to_s + "\n" + stripped + "" + end + end + stripped = stripped.strip + [text.strip, stripped.presence] + end + +end diff --git a/app/lib/smtp_client/endpoint.rb b/app/lib/smtp_client/endpoint.rb new file mode 100644 index 0000000..5449e69 --- /dev/null +++ b/app/lib/smtp_client/endpoint.rb @@ -0,0 +1,169 @@ +# frozen_string_literal: true + +module SMTPClient + class Endpoint + + class SMTPSessionNotStartedError < StandardError + end + + attr_reader :server + attr_reader :ip_address + attr_accessor :smtp_client + + # @param server [Server] the server that this IP address is for + # @param ip_address [String] the IP address + def initialize(server, ip_address) + @server = server + @ip_address = ip_address + end + + # Return a description of this server with its IP address + # + # @return [String] + def description + "#{@ip_address}:#{@server.port} (#{@server.hostname})" + end + + # Return a string representation of this server + # + # @return [String] + def to_s + description + end + + # Return true if this is an IPv6 address + # + # @return [Boolean] + def ipv6? + @ip_address.include?(":") + end + + # Return true if this is an IPv4 address + # + # @return [Boolean] + def ipv4? + !ipv6? + end + + # Start a new SMTP session and store the client with this server for future use as needed + # + # @param source_ip_address [IPAddress] the IP address to use as the source address for the connection + # @param allow_ssl [Boolean] whether to allow SSL for this connection, if false SSL mode is ignored + # + # @return [Net::SMTP] + def start_smtp_session(source_ip_address: nil, allow_ssl: true) + @smtp_client = Net::SMTP.new(@ip_address, @server.port) + @smtp_client.open_timeout = Postal::Config.smtp_client.open_timeout + @smtp_client.read_timeout = Postal::Config.smtp_client.read_timeout + @smtp_client.tls_hostname = @server.hostname + + if source_ip_address + @source_ip_address = source_ip_address + end + + if @source_ip_address + @smtp_client.source_address = ipv6? ? @source_ip_address.ipv6 : @source_ip_address.ipv4 + end + + if allow_ssl + case @server.ssl_mode + when SSLModes::AUTO + @smtp_client.enable_starttls_auto(self.class.ssl_context_without_verify) + when SSLModes::STARTTLS + @smtp_client.enable_starttls(self.class.ssl_context_with_verify) + when SSLModes::TLS + @smtp_client.enable_tls(self.class.ssl_context_with_verify) + else + @smtp_client.disable_starttls + @smtp_client.disable_tls + end + else + @smtp_client.disable_starttls + @smtp_client.disable_tls + end + + @smtp_client.start(@source_ip_address ? @source_ip_address.hostname : self.class.default_helo_hostname) + + @smtp_client + end + + # Send a message to the current SMTP session (or create one if there isn't one for this endpoint). + # If sending messsage encouters some connection errors, retry again after re-establishing the SMTP + # session. + # + # @param raw_message [String] the raw message to send + # @param mail_from [String] the MAIL FROM address + # @param rcpt_to [String] the RCPT TO address + # @param retry_on_connection_error [Boolean] whether to retry the connection if there is a connection error + # + # @return [void] + def send_message(raw_message, mail_from, rcpt_to, retry_on_connection_error: true) + raise SMTPSessionNotStartedError if @smtp_client.nil? || (@smtp_client && !@smtp_client.started?) + + @smtp_client.rset_errors + @smtp_client.send_message(raw_message, mail_from, [rcpt_to]) + rescue Errno::ECONNRESET, Errno::EPIPE, OpenSSL::SSL::SSLError + if retry_on_connection_error + finish_smtp_session + start_smtp_session + return send_message(raw_message, mail_from, rcpt_to, retry_on_connection_error: false) + end + + raise + end + + # Reset the current SMTP session for this server if possible otherwise + # finish the session + # + # @return [void] + def reset_smtp_session + @smtp_client&.rset + rescue StandardError + finish_smtp_session + end + + # Finish the current SMTP session for this server if possible. + # + # @return [void] + def finish_smtp_session + @smtp_client&.finish + rescue StandardError + nil + ensure + @smtp_client = nil + end + + class << self + + # Return the default HELO hostname to present to SMTP servers that + # we connect to + # + # @return [String] + def default_helo_hostname + Postal::Config.dns.helo_hostname || + Postal::Config.postal.smtp_hostname || + "localhost" + end + + def ssl_context_with_verify + @ssl_context_with_verify ||= begin + c = OpenSSL::SSL::SSLContext.new + c.verify_mode = OpenSSL::SSL::VERIFY_PEER + c.cert_store = OpenSSL::X509::Store.new + c.cert_store.set_default_paths + c + end + end + + def ssl_context_without_verify + @ssl_context_without_verify ||= begin + c = OpenSSL::SSL::SSLContext.new + c.verify_mode = OpenSSL::SSL::VERIFY_NONE + c + end + end + + end + + end +end diff --git a/app/lib/smtp_client/server.rb b/app/lib/smtp_client/server.rb new file mode 100644 index 0000000..8630fff --- /dev/null +++ b/app/lib/smtp_client/server.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module SMTPClient + class Server + + attr_reader :hostname + attr_reader :port + attr_accessor :ssl_mode + + def initialize(hostname, port: 25, ssl_mode: SSLModes::AUTO) + @hostname = hostname + @port = port + @ssl_mode = ssl_mode + end + + # Return all IP addresses for this server by resolving its hostname. + # IPv6 addresses will be returned first. + # + # @return [Array] + def endpoints + ips = [] + + DNSResolver.local.aaaa(@hostname).each do |ip| + ips << Endpoint.new(self, ip) + end + + DNSResolver.local.a(@hostname).each do |ip| + ips << Endpoint.new(self, ip) + end + + ips + end + + end +end diff --git a/app/lib/smtp_client/ssl_modes.rb b/app/lib/smtp_client/ssl_modes.rb new file mode 100644 index 0000000..0d58763 --- /dev/null +++ b/app/lib/smtp_client/ssl_modes.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module SMTPClient + module SSLModes + + AUTO = "Auto" + STARTTLS = "STARTLS" + TLS = "TLS" + NONE = "None" + + end +end diff --git a/app/lib/smtp_server/client.rb b/app/lib/smtp_server/client.rb new file mode 100644 index 0000000..53cf678 --- /dev/null +++ b/app/lib/smtp_server/client.rb @@ -0,0 +1,596 @@ +# frozen_string_literal: true + +require "nifty/utils/random_string" + +module SMTPServer + class Client + + extend HasPrometheusMetrics + include HasPrometheusMetrics + + CRAM_MD5_DIGEST = OpenSSL::Digest.new("md5") + LOG_REDACTION_STRING = "[redacted]" + + attr_reader :logging_enabled + attr_reader :credential + attr_reader :ip_address + attr_reader :recipients + attr_reader :headers + attr_reader :state + attr_reader :helo_name + + def initialize(ip_address) + @logging_enabled = true + @ip_address = ip_address + + @cr_present = false + @previous_cr_present = nil + + if @ip_address + check_ip_address + @state = :welcome + else + @state = :preauth + end + transaction_reset + end + + def check_ip_address + return unless @ip_address && + Postal::Config.smtp_server.log_ip_address_exclusion_matcher && + @ip_address =~ Regexp.new(Postal::Config.smtp_server.log_ip_address_exclusion_matcher) + + @logging_enabled = false + end + + def transaction_reset + @recipients = [] + @mail_from = nil + @data = nil + @headers = nil + end + + def id + @id ||= Nifty::Utils::RandomString.generate(length: 6).upcase + end + + def handle(data) + if data[-1] == "\r" + @cr_present = true + data = data.chop # remove last character (\r) + else + Postal.logger.debug("\e[33m WARN: Detected line with invalid line ending (missing )\e[0m", id: id) + @cr_present = false + end + + Postal.logger.tagged(id: id) do + if @state == :preauth + return proxy(data) + end + + log "\e[32m<= #{sanitize_input_for_log(data.strip)}\e[0m" + if @proc + @proc.call(data) + else + handle_command(data) + end + end + ensure + @previous_cr_present = @cr_present + end + + def finished? + @finished || false + end + + def start_tls? + @start_tls || false + end + + attr_writer :start_tls + + def handle_command(data) + case data + when /^QUIT/i then quit + when /^STARTTLS/i then starttls + when /^EHLO/i then ehlo(data) + when /^HELO/i then helo(data) + when /^RSET/i then rset + when /^NOOP/i then noop + when /^AUTH PLAIN/i then auth_plain(data) + when /^AUTH LOGIN/i then auth_login(data) + when /^AUTH CRAM-MD5/i then auth_cram_md5(data) + when /^MAIL FROM/i then mail_from(data) + when /^RCPT TO/i then rcpt_to(data) + when /^DATA/i then data(data) + else + increment_error_count("invalid-command") + "502 Invalid/unsupported command" + end + end + + def log(text) + return false unless @logging_enabled + + Postal.logger.debug(text, id: id) + end + + private + + def proxy(data) + if m = data.match(/\APROXY (.+) (.+) (.+) (.+) (.+)\z/) + @ip_address = m[2] + check_ip_address + @state = :welcome + log "\e[35m Client identified as #{@ip_address}\e[0m" + increment_command_count("PROXY") + "220 #{Postal::Config.postal.smtp_hostname} ESMTP Postal/#{id}" + else + @finished = true + increment_error_count("proxy-error") + "502 Proxy Error" + end + end + + def quit + @finished = true + "221 Closing Connection" + end + + def starttls + if Postal::Config.smtp_server.tls_enabled? + @start_tls = true + @tls = true + increment_command_count("STARTLS") + "220 Ready to start TLS" + else + increment_error_count("tls-unavailable") + "502 TLS not available" + end + end + + def ehlo(data) + @helo_name = data.strip.split(" ", 2)[1] + transaction_reset + @state = :welcomed + increment_command_count("EHLO") + [ + "250-My capabilities are", + Postal::Config.smtp_server.tls_enabled? && !@tls ? "250-STARTTLS" : nil, + "250 AUTH CRAM-MD5 PLAIN LOGIN" + ].compact + end + + def helo(data) + @helo_name = data.strip.split(" ", 2)[1] + transaction_reset + @state = :welcomed + increment_command_count("HELO") + "250 #{Postal::Config.postal.smtp_hostname}" + end + + def rset + transaction_reset + @state = :welcomed + increment_command_count("RSET") + "250 OK" + end + + def noop + "250 OK" + end + + def auth_plain(data) + increment_command_count("AUTH PLAIN") + + handler = proc do |idata| + @proc = nil + idata = Base64.decode64(idata) + parts = idata.split("\0") + username = parts[-2] + password = parts[-1] + unless username && password + increment_error_count("missing-credentials") + next "535 Authenticated failed - protocol error" + end + + authenticate(password) + end + + data = data.gsub(/AUTH PLAIN ?/i, "") + if data.strip == "" + @proc = handler + @password_expected_next = true + "334" + else + handler.call(data) + end + end + + def auth_login(data) + increment_command_count("AUTH LOGIN") + + password_handler = proc do |idata| + @proc = nil + password = Base64.decode64(idata) + authenticate(password) + end + + username_handler = proc do + @proc = password_handler + @password_expected_next = true + "334 UGFzc3dvcmQ6" # "Password:" + end + + data = data.gsub(/AUTH LOGIN ?/i, "") + if data.strip == "" + @proc = username_handler + "334 VXNlcm5hbWU6" # "Username:" + else + username_handler.call(nil) + end + end + + def authenticate(password) + if @credential = Credential.where(type: "SMTP", key: password).first + @credential.use + "235 Granted for #{@credential.server.organization.permalink}/#{@credential.server.permalink}" + else + log "\e[33m WARN: AUTH failure for #{@ip_address}\e[0m" + increment_error_count("invalid-credentials") + "535 Invalid credential" + end + end + + def auth_cram_md5(data) + increment_command_count("AUTH CRAM-MD5") + + challenge = Digest::SHA1.hexdigest(Time.now.to_i.to_s + rand(100_000).to_s) + challenge = "<#{challenge[0, 20]}@#{Postal::Config.postal.smtp_hostname}>" + + handler = proc do |idata| + @proc = nil + username, password = Base64.decode64(idata).split(" ", 2).map { |a| a.chomp } + org_permlink, server_permalink = username.split(/[\/_]/, 2) + server = ::Server.includes(:organization).where(organizations: { permalink: org_permlink }, permalink: server_permalink).first + if server.nil? + log "\e[33m WARN: AUTH failure for #{@ip_address}\e[0m" + increment_error_count("invalid-credentials") + next "535 Denied" + end + + grant = nil + server.credentials.where(type: "SMTP").each do |credential| + correct_response = OpenSSL::HMAC.hexdigest(CRAM_MD5_DIGEST, credential.key, challenge) + next unless password == correct_response + + @credential = credential + @credential.use + grant = "235 Granted for #{credential.server.organization.permalink}/#{credential.server.permalink}" + break + end + + if grant.nil? + log "\e[33m WARN: AUTH failure for #{@ip_address}\e[0m" + increment_error_count("invalid-credentials") + next "535 Denied" + end + + grant + end + + @proc = handler + "334 " + Base64.encode64(challenge).gsub(/[\r\n]/, "") + end + + def mail_from(data) + unless in_state(:welcomed, :mail_from_received) + increment_error_count("mail-from-out-of-order") + return "503 EHLO/HELO first please" + end + + @state = :mail_from_received + transaction_reset + if data =~ /AUTH=/ + # Discard AUTH= parameter and anything that follows. + # We don't need this parameter as we don't trust any client to set it + mail_from_line = data.sub(/ *AUTH=.*/, "") + else + mail_from_line = data + end + @mail_from = mail_from_line.gsub(/MAIL FROM\s*:\s*/i, "").gsub(/.*.*/, "").strip + "250 OK" + end + + def rcpt_to(data) + unless in_state(:mail_from_received, :rcpt_to_received) + increment_error_count("rcpt-to-out-of-order") + return "503 EHLO/HELO and MAIL FROM first please" + end + + rcpt_to = data.gsub(/RCPT TO\s*:\s*/i, "").gsub(/.*.*/, "").strip + + if rcpt_to.blank? + increment_error_count("empty-rcpt-to") + return "501 RCPT TO should not be empty" + end + + uname, domain = rcpt_to.split("@", 2) + + if domain.blank? + increment_error_count("invalid-rcpt-to") + return "501 Invalid RCPT TO" + end + + uname, tag = uname.split("+", 2) + + if domain == Postal::Config.dns.return_path_domain || domain =~ /\A#{Regexp.escape(Postal::Config.dns.custom_return_path_prefix)}\./ + # This is a return path + @state = :rcpt_to_received + if server = ::Server.where(token: uname).first + if server.suspended? + increment_error_count("server-suspended") + "535 Mail server has been suspended" + else + log "Added bounce on server #{server.id}" + @recipients << [:bounce, rcpt_to, server] + "250 OK" + end + else + increment_error_count("invalid-server-token") + "550 Invalid server token" + end + + elsif domain == Postal::Config.dns.route_domain + # This is an email direct to a route. This isn't actually supported yet. + @state = :rcpt_to_received + if route = Route.where(token: uname).first + if route.server.suspended? + increment_error_count("server-suspended") + "535 Mail server has been suspended" + elsif route.mode == "Reject" + increment_error_count("route-rejected") + "550 Route does not accept incoming messages" + else + log "Added route #{route.id} to recipients (tag: #{tag.inspect})" + actual_rcpt_to = "#{route.name}#{tag ? "+#{tag}" : ''}@#{route.domain.name}" + @recipients << [:route, actual_rcpt_to, route.server, { route: route }] + "250 OK" + end + else + "550 Invalid route token" + end + + elsif @credential + # This is outgoing mail for an authenticated user + @state = :rcpt_to_received + if @credential.server.suspended? + increment_error_count("server-suspended") + "535 Mail server has been suspended" + else + log "Added external address '#{rcpt_to}'" + @recipients << [:credential, rcpt_to, @credential.server] + "250 OK" + end + + elsif uname && domain && route = Route.find_by_name_and_domain(uname, domain) + # This is incoming mail for a route + @state = :rcpt_to_received + if route.server.suspended? + increment_error_count("server-suspended") + "535 Mail server has been suspended" + elsif route.mode == "Reject" + increment_error_count("route-rejection") + "550 Route does not accept incoming messages" + else + log "Added route #{route.id} to recipients (tag: #{tag.inspect})" + @recipients << [:route, rcpt_to, route.server, { route: route }] + "250 OK" + end + + else + # User is trying to relay but is not authenticated. Try to authenticate by IP address + @credential = Credential.where(type: "SMTP-IP").all.sort_by { |c| c.ipaddr&.prefix || 0 }.reverse.find do |credential| + credential.ipaddr.include?(@ip_address) || (credential.ipaddr.ipv4? && credential.ipaddr.ipv4_mapped.include?(@ip_address)) + end + + if @credential + # Retry with credential + @credential.use + rcpt_to(data) + else + increment_error_count("authentication-required") + "530 Authentication required" + end + end + end + + def data(_data) + unless in_state(:rcpt_to_received) + increment_error_count("data-out-of-order") + return "503 HELO/EHLO, MAIL FROM and RCPT TO before sending data" + end + + @data = String.new.force_encoding("BINARY") + @headers = {} + @receiving_headers = true + + received_header = ReceivedHeader.generate(@credential&.server, @helo_name, @ip_address, :smtp) + .force_encoding("BINARY") + + @data << "Received: #{received_header}\r\n" + @headers["received"] = [received_header] + + handler = proc do |idata| + if idata == "." && @cr_present && @previous_cr_present + @logging_enabled = true + @proc = nil + finished + else + idata = idata.to_s.sub(/\A\.\./, ".") + + if @credential&.server&.log_smtp_data? + # We want to log if enabled + else + log "Not logging further message data." + @logging_enabled = false + end + + if @receiving_headers + if idata&.length&.zero? + @receiving_headers = false + elsif idata.to_s =~ /^\s/ + # This is a continuation of a header + if @header_key && @headers[@header_key.downcase] && @headers[@header_key.downcase].last + @headers[@header_key.downcase].last << idata.to_s + end + else + @header_key, value = idata.split(/:\s*/, 2) + @headers[@header_key.downcase] ||= [] + @headers[@header_key.downcase] << value + end + end + @data << idata + @data << "\r\n" + nil + end + end + + @proc = handler + "354 Go ahead" + end + + def finished + if @data.bytesize > Postal::Config.smtp_server.max_message_size.megabytes.to_i + transaction_reset + @state = :welcomed + increment_error_count("message-too-large") + return format("552 Message too large (maximum size %dMB)", Postal::Config.smtp_server.max_message_size) + end + + if @headers["received"].grep(/by #{Postal::Config.postal.smtp_hostname}/).count > 4 + transaction_reset + @state = :welcomed + increment_error_count("loop-detected") + return "550 Loop detected" + end + + authenticated_domain = nil + if @credential + authenticated_domain = @credential.server.find_authenticated_domain_from_headers(@headers) + if authenticated_domain.nil? + transaction_reset + @state = :welcomed + increment_error_count("from-name-invalid") + return "530 From/Sender name is not valid" + end + end + + @recipients.each do |recipient| + type, rcpt_to, server, options = recipient + + case type + when :credential + increment_message_count("outgoing") + + # Outgoing messages are just inserted + message = server.message_db.new_message + message.rcpt_to = rcpt_to + message.mail_from = @mail_from + message.raw_message = @data + message.received_with_ssl = @tls + message.scope = "outgoing" + message.domain_id = authenticated_domain&.id + message.credential_id = @credential.id + message.save + + when :bounce + increment_message_count("bounce") + if rp_route = server.routes.where(name: "__returnpath__").first + # If there's a return path route, we can use this to create the message + rp_route.create_messages do |msg| + msg.rcpt_to = rcpt_to + msg.mail_from = @mail_from + msg.raw_message = @data + msg.received_with_ssl = @tls + msg.bounce = 1 + end + else + # There's no return path route, we just need to insert the mesage + # without going through the route. + message = server.message_db.new_message + message.rcpt_to = rcpt_to + message.mail_from = @mail_from + message.raw_message = @data + message.received_with_ssl = @tls + message.scope = "incoming" + message.bounce = 1 + message.save + end + when :route + increment_message_count("incoming") + options[:route].create_messages do |msg| + msg.rcpt_to = rcpt_to + msg.mail_from = @mail_from + msg.raw_message = @data + msg.received_with_ssl = @tls + end + end + end + transaction_reset + @state = :welcomed + "250 OK" + end + + def in_state(*states) + states.include?(@state) + end + + def sanitize_input_for_log(data) + if @password_expected_next + @password_expected_next = false + if data =~ /\A[a-z0-9]{3,}=*\z/i + return LOG_REDACTION_STRING + end + end + + data = data.dup + data.gsub!(/(.*AUTH \w+) (.*)\z/i) { "#{::Regexp.last_match(1)} #{LOG_REDACTION_STRING}" } + data + end + + def increment_error_count(error) + increment_prometheus_counter :postal_smtp_server_client_errors, labels: { error: error } + end + + def increment_command_count(command) + increment_prometheus_counter :postal_smtp_server_commands_total, labels: { command: command } + end + + def increment_message_count(type) + increment_prometheus_counter :postal_smtp_server_messages_total, labels: { + type: type, + tls: @tls ? "yes" : "no" + } + end + + class << self + + def register_prometheus_metrics + register_prometheus_counter :postal_smtp_server_commands_total, + docstring: "The number of key commands received by the server", + labels: [:command] + + register_prometheus_counter :postal_smtp_server_client_errors, + docstring: "The number of errors sent to a client", + labels: [:error] + + register_prometheus_counter :postal_smtp_server_messages_total, + docstring: "The number of messages accepted by the SMTP server", + labels: [:type, :tls] + end + + end + + end +end diff --git a/app/lib/smtp_server/server.rb b/app/lib/smtp_server/server.rb new file mode 100644 index 0000000..dadb48d --- /dev/null +++ b/app/lib/smtp_server/server.rb @@ -0,0 +1,317 @@ +# frozen_string_literal: true + +require "ipaddr" +require "nio" + +module SMTPServer + class Server + + include HasPrometheusMetrics + + class << self + + def tls_private_key + @tls_private_key ||= OpenSSL::PKey.read(File.read(Postal::Config.smtp_server.tls_private_key_path)) + end + + def tls_certificates + @tls_certificates ||= begin + data = File.read(Postal::Config.smtp_server.tls_certificate_path) + certs = data.scan(/-----BEGIN CERTIFICATE-----.+?-----END CERTIFICATE-----/m) + certs.map do |c| + OpenSSL::X509::Certificate.new(c) + end.freeze + end + end + + end + + def initialize(options = {}) + @options = options + @options[:debug] ||= false + register_prometheus_metrics + prepare_environment + end + + def run + logger.tagged(component: "smtp-server") do + listen + run_event_loop + end + end + + private + + def prepare_environment + $\ = "\r\n" + BasicSocket.do_not_reverse_lookup = true + + trap("TERM") do + $stdout.puts "Received TERM signal, shutting down." + unlisten + end + + trap("INT") do + $stdout.puts "Received INT signal, shutting down." + unlisten + end + end + + def ssl_context + @ssl_context ||= begin + ssl_context = OpenSSL::SSL::SSLContext.new + ssl_context.cert = Postal.smtp_certificates[0] + ssl_context.extra_chain_cert = self.class.tls_certificates[1..] + ssl_context.key = self.class.tls_private_key + ssl_context.ssl_version = Postal::Config.smtp_server.ssl_version if Postal::Config.smtp_server.ssl_version + ssl_context.ciphers = Postal::Config.smtp_server.tls_ciphers if Postal::Config.smtp_server.tls_ciphers + ssl_context + end + end + + def listen + bind_address = ENV.fetch("BIND_ADDRESS", Postal::Config.smtp_server.default_bind_address) + port = ENV.fetch("PORT", Postal::Config.smtp_server.default_port) + + @server = TCPServer.open(bind_address, port) + @server.autoclose = false + @server.close_on_exec = false + if defined?(Socket::SOL_SOCKET) && defined?(Socket::SO_KEEPALIVE) + @server.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true) + end + if defined?(Socket::SOL_TCP) && defined?(Socket::TCP_KEEPIDLE) && defined?(Socket::TCP_KEEPINTVL) && defined?(Socket::TCP_KEEPCNT) + @server.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPIDLE, 50) + @server.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPINTVL, 10) + @server.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPCNT, 5) + end + + logger.info "Listening on #{bind_address}:#{port}" + end + + def unlisten + # Instruct the nio loop to unlisten and wake it + @unlisten = true + @io_selector.wakeup + end + + def run_event_loop + # Set up an instance of nio4r to monitor for connections and data + @io_selector = NIO::Selector.new + # Register the SMTP listener + @io_selector.register(@server, :r) + # Create a hash to contain a buffer for each client. + buffers = Hash.new { |h, k| h[k] = String.new.force_encoding("BINARY") } + loop do + # Wait for an event to occur + @io_selector.select do |monitor| + # Get the IO from the nio monitor + io = monitor.io + # Is this event an incoming connection? + if io.is_a?(TCPServer) + begin + # Accept the connection + new_io = io.accept + increment_prometheus_counter :postal_smtp_server_connections_total + if Postal::Config.smtp_server.proxy_protocol? + # If we are using the haproxy proxy protocol, we will be sent the + # client's IP later. Delay the welcome process. + client = Client.new(nil) + if Postal::Config.smtp_server.log_connections? + logger.debug "[#{client.id}] \e[35m Connection opened from #{new_io.remote_address.ip_address}\e[0m" + end + else + # We're not using the proxy protocol so we already know the client's IP + client = Client.new(new_io.remote_address.ip_address) + if Postal::Config.smtp_server.log_connections? + logger.debug "[#{client.id}] \e[35m Connection opened from #{new_io.remote_address.ip_address}\e[0m" + end + # We know who the client is, welcome them. + client.log "\e[35m Client identified as #{new_io.remote_address.ip_address}\e[0m" + new_io.print("220 #{Postal::Config.postal.smtp_hostname} ESMTP Postal/#{client.id}") + end + # Register the client and its socket with nio4r + monitor = @io_selector.register(new_io, :r) + monitor.value = client + rescue StandardError => e + # If something goes wrong, log as appropriate and disconnect the client + if defined?(Sentry) + Sentry.capture_exception(e, extra: { log_id: begin + client.id + rescue StandardError + nil + end }) + end + logger.error "An error occurred while accepting a new client." + logger.error "#{e.class}: #{e.message}" + e.backtrace.each do |line| + logger.error line + end + increment_prometheus_counter :postal_smtp_server_exceptions_total, + error: e.class.to_s, + type: "client-accept" + begin + new_io.close + rescue StandardError + nil + end + end + else + # This event is not an incoming connection so it must be data from a client + begin + # Get the client from the nio monitor + client = monitor.value + # For now we assume the connection isn't closed + eof = false + # Is the client negotiating a TLS handshake? + if client.start_tls? + begin + # Can we accept the TLS connection at this time? + io.accept_nonblock + # Increment prometheus + increment_prometheus_counter :postal_smtp_server_tls_connections_total + # We were able to accept the connection, the client is no longer handshaking + client.start_tls = false + rescue IO::WaitReadable, IO::WaitWritable => e + # Could not accept without blocking + # We will try again later + next + rescue OpenSSL::SSL::SSLError => e + client.log "SSL Negotiation Failed: #{e.message}" + eof = true + end + else + # The client is not negotiating a TLS handshake at this time + begin + # Read 10kiB of data at a time from the socket. + buffers[io] << io.readpartial(10_240) + + # There is an extra step for SSL sockets + if io.is_a?(OpenSSL::SSL::SSLSocket) + buffers[io] << io.readpartial(10_240) while io.pending.positive? + end + rescue EOFError, Errno::ECONNRESET, Errno::ETIMEDOUT + # Client went away + eof = true + end + + # We line buffer, so look to see if we have received a newline + # and keep doing so until all buffered lines have been processed. + while buffers[io].index("\n") + # Extract the line + line, buffers[io] = buffers[io].split("\n", 2) + # Send the received line to the client object for processing + result = client.handle(line) + # If the client object returned some data, write it back to the client + next if result.nil? + + result = [result] unless result.is_a?(Array) + result.compact.each do |iline| + client.log "\e[34m=> #{iline.strip}\e[0m" + begin + io.write(iline.to_s + "\r\n") + io.flush + rescue Errno::ECONNRESET + # Client disconnected before we could write response + eof = true + end + end + end + + # Did the client request STARTTLS? + if !eof && client.start_tls? + # Deregister the unencrypted IO + @io_selector.deregister(io) + buffers.delete(io) + io = OpenSSL::SSL::SSLSocket.new(io, ssl_context) + # Close the underlying IO when the TLS socket is closed + io.sync_close = true + # Register the new TLS socket with nio + monitor = @io_selector.register(io, :r) + monitor.value = client + end + end + + # Has the client requested we close the connection? + if client.finished? || eof + client.log "\e[35m Connection closed\e[0m" + # Deregister the socket and close it + @io_selector.deregister(io) + buffers.delete(io) + io.close + # If we have no more clients or listeners left, exit the process + if @io_selector.empty? + Process.exit(0) + end + end + rescue StandardError => e + # Something went wrong, log as appropriate + client_id = client ? client.id : "------" + if defined?(Sentry) + Sentry.capture_exception(e, extra: { log_id: begin + client.id + rescue StandardError + nil + end }) + end + logger.error "[#{client_id}] An error occurred while processing data from a client." + logger.error "[#{client_id}] #{e.class}: #{e.message}" + e.backtrace.each do |iline| + logger.error "[#{client_id}] #{iline}" + end + + increment_prometheus_counter :postal_smtp_server_exceptions_total, + error: e.class.to_s, + type: "data" + + # Close all IO and forget this client + begin + @io_selector.deregister(io) + rescue StandardError + nil + end + buffers.delete(io) + begin + io.close + rescue StandardError + nil + end + if @io_selector.empty? + Process.exit(0) + end + end + end + end + # If unlisten has been called, stop listening + next unless @unlisten + + @io_selector.deregister(@server) + @server.close + # If there's nothing left to do, shut down the process + if @io_selector.empty? + Process.exit(0) + end + # Clear the request + @unlisten = false + end + end + + def logger + Postal.logger + end + + def register_prometheus_metrics + register_prometheus_counter :postal_smtp_server_connections_total, + docstring: "The number of connections made to the Postal SMTP server." + + register_prometheus_counter :postal_smtp_server_exceptions_total, + docstring: "The number of server exceptions encountered by the SMTP server", + labels: [:type, :error] + + register_prometheus_counter :postal_smtp_server_tls_connections_total, + docstring: "The number of successfuly TLS connections established" + + Client.register_prometheus_metrics + end + + end +end diff --git a/app/lib/worker/jobs/base_job.rb b/app/lib/worker/jobs/base_job.rb new file mode 100644 index 0000000..f324b51 --- /dev/null +++ b/app/lib/worker/jobs/base_job.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Worker + module Jobs + class BaseJob + + def initialize(logger:) + @logger = logger + end + + def call + # Override me. + end + + def work_completed? + @work_completed == true + end + + private + + def work_completed! + @work_completed = true + end + + attr_reader :logger + + end + end +end diff --git a/app/lib/worker/jobs/process_queued_messages_job.rb b/app/lib/worker/jobs/process_queued_messages_job.rb new file mode 100644 index 0000000..18489b5 --- /dev/null +++ b/app/lib/worker/jobs/process_queued_messages_job.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Worker + module Jobs + class ProcessQueuedMessagesJob < BaseJob + + def call + @lock_time = Time.current + @locker = Postal.locker_name_with_suffix(SecureRandom.hex(8)) + + find_ip_addresses + lock_message_for_processing + obtain_locked_messages + process_messages + @messages_to_process + end + + private + + # Returns an array of IP address IDs that are present on the host that is + # running this job. + # + # @return [Array] + def find_ip_addresses + ip_addresses = { 4 => [], 6 => [] } + Socket.ip_address_list.each do |address| + next if local_ip?(address.ip_address) + + ip_addresses[address.ipv4? ? 4 : 6] << address.ip_address + end + @ip_addresses = IPAddress.where(ipv4: ip_addresses[4]).or(IPAddress.where(ipv6: ip_addresses[6])).pluck(:id) + end + + # Is the given IP address a local address? + # + # @param [String] ip + # @return [Boolean] + def local_ip?(ip) + !!(ip =~ /\A(127\.|fe80:|::)/) + end + + # Obtain a queued message from the database for processing + # + # @return [void] + def lock_message_for_processing + QueuedMessage.where(ip_address_id: [nil, @ip_addresses]) + .where(locked_by: nil, locked_at: nil) + .ready_with_delayed_retry + .limit(1) + .update_all(locked_by: @locker, locked_at: @lock_time) + end + + # Get a full list of all messages which we can process (i.e. those which have just + # been locked by us for processing) + # + # @return [void] + def obtain_locked_messages + @messages_to_process = QueuedMessage.where(locked_by: @locker, locked_at: @lock_time) + end + + # Process the messages we obtained from the database + # + # @return [void] + def process_messages + @messages_to_process.each do |message| + work_completed! + MessageDequeuer.process(message, logger: logger) + end + end + + end + end +end diff --git a/app/lib/worker/jobs/process_webhook_requests_job.rb b/app/lib/worker/jobs/process_webhook_requests_job.rb new file mode 100644 index 0000000..53751a1 --- /dev/null +++ b/app/lib/worker/jobs/process_webhook_requests_job.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Worker + module Jobs + class ProcessWebhookRequestsJob < BaseJob + + def call + @lock_time = Time.current + @locker = Postal.locker_name_with_suffix(SecureRandom.hex(8)) + + lock_request_for_processing + obtain_locked_requests + process_requests + end + + private + + # Obtain a webhook request from the database for processing + # + # @return [void] + def lock_request_for_processing + WebhookRequest.unlocked + .ready + .limit(1) + .update_all(locked_by: @locker, locked_at: @lock_time) + end + + # Get a full list of all webhooks which we can process (i.e. those which have just + # been locked by us for processing) + # + # @return [void] + def obtain_locked_requests + @requests_to_process = WebhookRequest.where(locked_by: @locker, locked_at: @lock_time) + end + + # Process the webhook requests we obtained from the database + # + # @return [void] + def process_requests + @requests_to_process.each do |request| + work_completed! + + WebhookDeliveryService.new(webhook_request: request).call + end + end + + end + end +end diff --git a/app/lib/worker/process.rb b/app/lib/worker/process.rb new file mode 100644 index 0000000..11167e5 --- /dev/null +++ b/app/lib/worker/process.rb @@ -0,0 +1,293 @@ +# frozen_string_literal: true + +module Worker + # The Postal Worker process is responsible for handling all background tasks. This includes processing of all + # messages, webhooks and other administrative tasks. There are two main types of background work which is completed, + # jobs and scheduled tasks. + # + # The 'Jobs' here allow for the continuous monitoring of a database table (or queue) and processing of any new items + # which may appear in that. The polling takes place every 5 seconds by default and the work is able to run multiple + # threads to look for and process this work. + # + # Scheduled Tasks allow for code to be executed on a ROUGH schedule. This is used for administrative tasks. A single + # thread will run within each worker process and attempt to acquire the 'tasks' role. If successful it will run all + # tasks which are due to be run. The tasks are then scheduled to run again at a future time. Workers which are not + # successful in acquiring the role will not run any tasks but will still attempt to acquire a lock in case the current + # acquiree disappears. + # + # The worker process will run until it receives a TERM or INT signal. It will then attempt to gracefully shut down + # after it has completed any outstanding jobs which are already inflight. + class Process + + include HasPrometheusMetrics + + # An array of job classes that should be processed each time the worker ticks. + # + # @return [Array] + JOBS = [ + Jobs::ProcessQueuedMessagesJob, + Jobs::ProcessWebhookRequestsJob + ].freeze + + # An array of tasks that should be processed + # + # @return [Array] + TASKS = [ + ActionDeletionsScheduledTask, + CheckAllDNSScheduledTask, + CleanupAuthieSessionsScheduledTask, + ExpireHeldMessagesScheduledTask, + ProcessMessageRetentionScheduledTask, + PruneSuppressionListsScheduledTask, + PruneWebhookRequestsScheduledTask, + SendNotificationsScheduledTask + ].freeze + + # @param [Integer] thread_count The number of worker threads to run in this process + def initialize(thread_count: 2, work_sleep_time: 5, task_sleep_time: 60) + @thread_count = thread_count + @exit_pipe_read, @exit_pipe_write = IO.pipe + @work_sleep_time = work_sleep_time + @task_sleep_time = task_sleep_time + @threads = [] + + setup_prometheus + end + + def run + logger.tagged(component: "worker") do + setup_traps + start_work_threads + start_tasks_thread + wait_for_threads + end + end + + private + + # Install signal traps to allow for graceful shutdown + # + # @return [void] + def setup_traps + trap("INT") { receive_signal("INT") } + trap("TERM") { receive_signal("TERM") } + end + + # Receive a signal and set the shutdown flag + # + # @param [String] signal The signal that was received z + # @return [void] + def receive_signal(signal) + puts "Received #{signal} signal. Stopping when able." + @shutdown = true + @exit_pipe_write.close + end + + # Wait for the period of time and return true or false if shutdown has been requested. If the shutdown is + # requested during the wait, it will return immediately otherwise it will return false when it has finished + # waiting for the period of time. + # + # @param [Integer] wait_time The time to wait for + # @return [Boolean] + def shutdown_after_wait?(wait_time) + @exit_pipe_read.wait_readable(wait_time) ? true : false + end + + # Wait for all threads to complete + # + # @return [void] + def wait_for_threads + @threads.each(&:join) + end + + # Start the worker threads + # + # @return [void] + def start_work_threads + logger.info "starting #{@thread_count} work threads" + @thread_count.times do |index| + start_work_thread(index) + end + end + + # Start a worker thread + # + # @return [void] + def start_work_thread(index) + @threads << Thread.new do + logger.tagged(component: "worker", thread: "work#{index}") do + logger.info "started work thread #{index}" + loop do + work_completed = work(index) + + if shutdown_after_wait?(work_completed ? 0 : @work_sleep_time) + break + end + end + + logger.info "stopping work thread #{index}" + end + end + end + + # Actually perform the work for this tick. This will call each job which has been registered. + # + # @return [Boolean] Whether any work was completed in this job or not + def work(thread) + completed_work = 0 + ActiveRecord::Base.connection_pool.with_connection do + JOBS.each do |job_class| + capture_errors do + job = job_class.new(logger: logger) + + time = Benchmark.realtime { job.call } + + observe_prometheus_histogram :postal_worker_job_runtime, + time, + labels: { + thread: thread, + job: job_class.to_s.split("::").last + } + + if job.work_completed? + completed_work += 1 + increment_prometheus_counter :postal_worker_job_executions, + labels: { + thread: thread, + job: job_class.to_s.split("::").last + } + end + end + end + end + completed_work.positive? + end + + # Start the tasks thread + # + # @return [void] + def start_tasks_thread + logger.info "starting tasks thread" + @threads << Thread.new do + logger.tagged(component: "worker", thread: "tasks") do + loop do + run_tasks + + if shutdown_after_wait?(@task_sleep_time) + break + end + end + + logger.info "stopping tasks thread" + ActiveRecord::Base.connection_pool.with_connection do + if WorkerRole.release(:tasks) + logger.info "releasesd tasks role" + end + end + end + end + end + + # Run the tasks. This will attempt to acquire the tasks role and if successful it will all the registered + # tasks if they are due to be run. + # + # @return [void] + def run_tasks + role_acquisition_status = ActiveRecord::Base.connection_pool.with_connection do + WorkerRole.acquire(:tasks) + end + + case role_acquisition_status + when :stolen + logger.info "acquired task role by stealing it from a lazy worker" + when :created + logger.info "acquired task role by creating it" + when :renewed + logger.debug "acquired task role by renewing it" + else + logger.debug "could not acquire task role, not doing anything" + return false + end + + ActiveRecord::Base.connection_pool.with_connection do + TASKS.each { |task| run_task(task) } + end + end + + # Run a single task + # + # @param [Class] task The task to run + # @return [void] + def run_task(task) + logger.tagged task: task do + scheduled_task = ScheduledTask.find_by(name: task.to_s) + if scheduled_task.nil? + logger.info "no existing task object, creating it now" + scheduled_task = ScheduledTask.create!(name: task.to_s, next_run_after: task.next_run_after) + end + + next unless scheduled_task.next_run_after < Time.current + + logger.info "running task" + + time = 0 + capture_errors do + time = Benchmark.realtime do + task.new(logger: logger).call + end + + observe_prometheus_histogram :postal_worker_task_runtime, + time, + labels: { + task: task.to_s.split("::").last + } + end + + next_run_after = task.next_run_after + logger.info "scheduling task to next run at #{next_run_after}" + scheduled_task.update!(next_run_after: next_run_after) + end + end + + # Return the logger + # + # @return [Klogger::Logger] + def logger + Postal.logger + end + + # Capture exceptions and handle this as appropriate. + # + # @yield The block of code to run + # @return [void] + def capture_errors + yield + rescue StandardError => e + logger.error "#{e.class} (#{e.message})" + e.backtrace.each { |line| logger.error line } + Sentry.capture_exception(e) if defined?(Sentry) + + increment_prometheus_counter :postal_worker_errors, + labels: { error: e.class.to_s } + end + + def setup_prometheus + register_prometheus_counter :postal_worker_job_executions, + docstring: "The number of jobs worked by a worker where work was completed", + labels: [:thread, :job] + + register_prometheus_histogram :postal_worker_job_runtime, + docstring: "The time taken to process jobs", + labels: [:thread, :job] + + register_prometheus_counter :postal_worker_errors, + docstring: "The number of errors encountered while processing jobs", + labels: [:error] + + register_prometheus_histogram :postal_worker_task_runtime, + docstring: "The time taken to process tasks", + labels: [:task] + end + + end +end diff --git a/app/mailers/application_mailer.rb b/app/mailers/application_mailer.rb index 95b7f19..11289d8 100644 --- a/app/mailers/application_mailer.rb +++ b/app/mailers/application_mailer.rb @@ -2,7 +2,7 @@ class ApplicationMailer < ActionMailer::Base - default from: "#{Postal.smtp_from_name} <#{Postal.smtp_from_address}>" + default from: "#{Postal::Config.smtp.from_name} <#{Postal::Config.smtp.from_address}>" layout false end diff --git a/app/models/bounce_message.rb b/app/models/bounce_message.rb new file mode 100644 index 0000000..0320072 --- /dev/null +++ b/app/models/bounce_message.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +class BounceMessage + + def initialize(server, message) + @server = server + @message = message + end + + def raw_message + mail = Mail.new + mail.to = @message.mail_from + mail.from = "Mail Delivery Service <#{@message.route.description}>" + mail.subject = "Mail Delivery Failed (#{@message.subject})" + mail.text_part = body + mail.attachments["Original Message.eml"] = { mime_type: "message/rfc822", encoding: "quoted-printable", content: @message.raw_message } + mail.message_id = "<#{SecureRandom.uuid}@#{Postal::Config.dns.return_path_domain}>" + mail.to_s + end + + def queue + message = @server.message_db.new_message + message.scope = "outgoing" + message.rcpt_to = @message.mail_from + message.mail_from = @message.route.description + message.domain_id = @message.domain&.id + message.raw_message = raw_message + message.bounce = true + message.bounce_for_id = @message.id + message.save + message.id + end + + def postmaster_address + @server.postmaster_address || "postmaster@#{@message.domain&.name || Postal::Config.postal.web_hostname}" + end + + private + + def body + <<~BODY + This is the mail delivery service responsible for delivering mail to #{@message.route.description}. + + The message you've sent cannot be delivered. Your original message is attached to this message. + + For further assistance please contact #{postmaster_address}. Please include the details below to help us identify the issue. + + Message Token: #{@message.token}@#{@server.token} + Orginal Message ID: #{@message.message_id} + Mail from: #{@message.mail_from} + Rcpt To: #{@message.rcpt_to} + BODY + end + +end diff --git a/app/models/concerns/has_authentication.rb b/app/models/concerns/has_authentication.rb index 80a52ce..4491a77 100644 --- a/app/models/concerns/has_authentication.rb +++ b/app/models/concerns/has_authentication.rb @@ -8,13 +8,7 @@ module HasAuthentication has_secure_password validates :password, length: { minimum: 8, allow_blank: true } - - when_attribute :password_digest, changes_to: :anything do - before_save do - self.password_reset_token = nil - self.password_reset_token_valid_until = nil - end - end + before_save :clear_password_reset_token_on_password_change end class_methods do @@ -42,6 +36,15 @@ module HasAuthentication AppMailer.password_reset(self, return_to).deliver end + private + + def clear_password_reset_token_on_password_change + return unless password_digest_changed? + + self.password_reset_token = nil + self.password_reset_token_valid_until = nil + end + end # -*- SkipSchemaAnnotations diff --git a/app/models/concerns/has_dns_checks.rb b/app/models/concerns/has_dns_checks.rb index a4d6ec5..789aeff 100644 --- a/app/models/concerns/has_dns_checks.rb +++ b/app/models/concerns/has_dns_checks.rb @@ -43,16 +43,16 @@ module HasDNSChecks # def check_spf_record - result = resolver.getresources(name, Resolv::DNS::Resource::IN::TXT) - spf_records = result.map(&:data).grep(/\Av=spf1/) + result = resolver.txt(name) + spf_records = result.grep(/\Av=spf1/) if spf_records.empty? self.spf_status = "Missing" self.spf_error = "No SPF record exists for this domain" else - suitable_spf_records = spf_records.grep(/include:\s*#{Regexp.escape(Postal.config.dns.spf_include)}/) + suitable_spf_records = spf_records.grep(/include:\s*#{Regexp.escape(Postal::Config.dns.spf_include)}/) if suitable_spf_records.empty? self.spf_status = "Invalid" - self.spf_error = "An SPF record exists but it doesn't include #{Postal.config.dns.spf_include}" + self.spf_error = "An SPF record exists but it doesn't include #{Postal::Config.dns.spf_include}" false else self.spf_status = "OK" @@ -73,8 +73,7 @@ module HasDNSChecks def check_dkim_record domain = "#{dkim_record_name}.#{name}" - result = resolver.getresources(domain, Resolv::DNS::Resource::IN::TXT) - records = result.map(&:data) + records = resolver.txt(domain) if records.empty? self.dkim_status = "Missing" self.dkim_error = "No TXT records were returned for #{domain}" @@ -104,17 +103,16 @@ module HasDNSChecks # def check_mx_records - result = resolver.getresources(name, Resolv::DNS::Resource::IN::MX) - records = result.map(&:exchange) + records = resolver.mx(name).map(&:last) if records.empty? self.mx_status = "Missing" self.mx_error = "There are no MX records for #{name}" else - missing_records = Postal.config.dns.mx_records.dup - records.map { |r| r.to_s.downcase } + missing_records = Postal::Config.dns.mx_records.dup - records.map { |r| r.to_s.downcase } if missing_records.empty? self.mx_status = "OK" self.mx_error = nil - elsif missing_records.size == Postal.config.dns.mx_records.size + elsif missing_records.size == Postal::Config.dns.mx_records.size self.mx_status = "Missing" self.mx_error = "You have MX records but none of them point to us." else @@ -134,17 +132,16 @@ module HasDNSChecks # def check_return_path_record - result = resolver.getresources(return_path_domain, Resolv::DNS::Resource::IN::CNAME) - records = result.map { |r| r.name.to_s.downcase } + records = resolver.cname(return_path_domain) if records.empty? self.return_path_status = "Missing" self.return_path_error = "There is no return path record at #{return_path_domain}" - elsif records.size == 1 && records.first == Postal.config.dns.return_path + elsif records.size == 1 && records.first == Postal::Config.dns.return_path_domain self.return_path_status = "OK" self.return_path_error = nil else self.return_path_status = "Invalid" - self.return_path_error = "There is a CNAME record at #{return_path_domain} but it points to #{records.first} which is incorrect. It should point to #{Postal.config.dns.return_path}." + self.return_path_error = "There is a CNAME record at #{return_path_domain} but it points to #{records.first} which is incorrect. It should point to #{Postal::Config.dns.return_path_domain}." end end diff --git a/app/models/concerns/has_locking.rb b/app/models/concerns/has_locking.rb new file mode 100644 index 0000000..6223416 --- /dev/null +++ b/app/models/concerns/has_locking.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +# This concern provides functionality for locking items along with additional functionality to handle +# the concept of retrying items after a certain period of time. The following database columns are +# required on the model +# +# * locked_by - A string column to store the name of the process that has locked the item +# * locked_at - A datetime column to store the time the item was locked +# * retry_after - A datetime column to store the time after which the item should be retried +# * attempts - An integer column to store the number of attempts that have been made to process the item +# +# 'ready' means that it's ready to be processed. +module HasLocking + + extend ActiveSupport::Concern + + included do + scope :unlocked, -> { where(locked_at: nil) } + scope :ready, -> { where("retry_after IS NULL OR retry_after < ?", Time.now) } + end + + def ready? + retry_after.nil? || retry_after < Time.now + end + + def unlock + self.locked_by = nil + self.locked_at = nil + update_columns(locked_by: nil, locked_at: nil) + end + + def locked? + locked_at.present? + end + + def retry_later(time = nil) + retry_time = time || calculate_retry_time(attempts, 5.minutes) + self.locked_by = nil + self.locked_at = nil + update_columns(locked_by: nil, locked_at: nil, retry_after: Time.now + retry_time, attempts: attempts + 1) + end + + def calculate_retry_time(attempts, initial_period) + (1.3**attempts) * initial_period + end + +end diff --git a/app/models/concerns/has_soft_destroy.rb b/app/models/concerns/has_soft_destroy.rb index 55cfcaf..cb07790 100644 --- a/app/models/concerns/has_soft_destroy.rb +++ b/app/models/concerns/has_soft_destroy.rb @@ -14,7 +14,6 @@ module HasSoftDestroy run_callbacks :soft_destroy do self.deleted_at = Time.now save! - ActionDeletionJob.queue(:main, type: self.class.name, id: id) end end diff --git a/app/models/credential.rb b/app/models/credential.rb index 18e3266..8062eb5 100644 --- a/app/models/credential.rb +++ b/app/models/credential.rb @@ -39,7 +39,7 @@ class Credential < ApplicationRecord return if type == "SMTP-IP" return if persisted? - self.key = SecureRandomString.new(24) + self.key = SecureRandom.alphanumeric(24) end def to_param diff --git a/app/models/domain.rb b/app/models/domain.rb index 4a845f8..19e184d 100644 --- a/app/models/domain.rb +++ b/app/models/domain.rb @@ -61,23 +61,15 @@ class Domain < ApplicationRecord scope :verified, -> { where.not(verified_at: nil) } - when_attribute :verification_method, changes_to: :anything do - before_save do - if verification_method == "DNS" - self.verification_token = Nifty::Utils::RandomString.generate(length: 32) - elsif verification_method == "Email" - self.verification_token = rand(999_999).to_s.ljust(6, "0") - else - self.verification_token = nil - end - end - end + before_save :update_verification_token_on_method_change def verified? verified_at.present? end - def verify + def mark_as_verified + return false if verified? + self.verified_at = Time.now save! end @@ -94,6 +86,8 @@ class Domain < ApplicationRecord end def dkim_key + return nil unless dkim_private_key + @dkim_key ||= OpenSSL::PKey::RSA.new(dkim_private_key) end @@ -110,67 +104,72 @@ class Domain < ApplicationRecord end def spf_record - "v=spf1 a mx include:#{Postal.config.dns.spf_include} ~all" + "v=spf1 a mx include:#{Postal::Config.dns.spf_include} ~all" end def dkim_record + return if dkim_key.nil? + public_key = dkim_key.public_key.to_s.gsub(/-+[A-Z ]+-+\n/, "").gsub(/\n/, "") "v=DKIM1; t=s; h=sha256; p=#{public_key};" end def dkim_identifier - Postal.config.dns.dkim_identifier + "-#{dkim_identifier_string}" + return nil unless dkim_identifier_string + + Postal::Config.dns.dkim_identifier + "-#{dkim_identifier_string}" end def dkim_record_name - "#{dkim_identifier}._domainkey" + identifier = dkim_identifier + return if identifier.nil? + + "#{identifier}._domainkey" end def return_path_domain - "#{Postal.config.dns.custom_return_path_prefix}.#{name}" - end - - def nameservers - @nameservers ||= get_nameservers + "#{Postal::Config.dns.custom_return_path_prefix}.#{name}" end + # Returns a DNSResolver instance that can be used to perform DNS lookups needed for + # the verification and DNS checking for this domain. + # + # @return [DNSResolver] def resolver - @resolver ||= Postal.config.general.use_local_ns_for_domains? ? Resolv::DNS.new : Resolv::DNS.new(nameserver: nameservers) + return DNSResolver.local if Postal::Config.postal.use_local_ns_for_domain_verification? + + @resolver ||= DNSResolver.for_domain(name) end def dns_verification_string - "#{Postal.config.dns.domain_verify_prefix} #{verification_token}" + "#{Postal::Config.dns.domain_verify_prefix} #{verification_token}" end def verify_with_dns return false unless verification_method == "DNS" - result = resolver.getresources(name, Resolv::DNS::Resource::IN::TXT) - if result.map { |d| d.data.to_s.strip }.include?(dns_verification_string) + result = resolver.txt(name) + + if result.include?(dns_verification_string) self.verified_at = Time.now - save - else - false + return save end + + false end private - def get_nameservers - local_resolver = Resolv::DNS.new - ns_records = [] - parts = name.split(".") - (parts.size - 1).times do |n| - d = parts[n, parts.size - n + 1].join(".") - ns_records = local_resolver.getresources(d, Resolv::DNS::Resource::IN::NS).map { |s| s.name.to_s } - break if ns_records.present? + def update_verification_token_on_method_change + return unless verification_method_changed? + + if verification_method == "DNS" + self.verification_token = Nifty::Utils::RandomString.generate(length: 32) + elsif verification_method == "Email" + self.verification_token = rand(999_999).to_s.ljust(6, "0") + else + self.verification_token = nil end - return [] if ns_records.blank? - - ns_records = ns_records.map { |r| local_resolver.getresources(r, Resolv::DNS::Resource::IN::A).map { |s| s.address.to_s } }.flatten - return [] if ns_records.blank? - - ns_records end end diff --git a/app/models/incoming_message_prototype.rb b/app/models/incoming_message_prototype.rb index 334df62..9380300 100644 --- a/app/models/incoming_message_prototype.rb +++ b/app/models/incoming_message_prototype.rb @@ -88,14 +88,14 @@ class IncomingMessagePrototype mail.from = @from mail.subject = @subject mail.text_part = @plain_body - mail.message_id = "<#{SecureRandom.uuid}@#{Postal.config.dns.return_path}>" + mail.message_id = "<#{SecureRandom.uuid}@#{Postal::Config.dns.return_path_domain}>" attachments.each do |attachment| mail.attachments[attachment[:name]] = { mime_type: attachment[:content_type], content: attachment[:data] } end - mail.header["Received"] = Postal::ReceivedHeader.generate(@server, @source_type, @ip, :http) + mail.header["Received"] = ReceivedHeader.generate(@server, @source_type, @ip, :http) mail.to_s end end diff --git a/app/models/outgoing_message_prototype.rb b/app/models/outgoing_message_prototype.rb index 7362f30..276c180 100644 --- a/app/models/outgoing_message_prototype.rb +++ b/app/models/outgoing_message_prototype.rb @@ -25,7 +25,7 @@ class OutgoingMessagePrototype @source_type = source_type @custom_headers = {} @attachments = [] - @message_id = "#{SecureRandom.uuid}@#{Postal.config.dns.return_path}" + @message_id = "#{SecureRandom.uuid}@#{Postal::Config.dns.return_path_domain}" attributes.each do |key, value| instance_variable_set("@#{key}", value) end @@ -177,7 +177,7 @@ class OutgoingMessagePrototype content: attachment[:data] } end - mail.header["Received"] = Postal::ReceivedHeader.generate(@server, @source_type, @ip, :http) + mail.header["Received"] = ReceivedHeader.generate(@server, @source_type, @ip, :http) mail.message_id = "<#{@message_id}>" mail.to_s end diff --git a/app/models/queued_message.rb b/app/models/queued_message.rb index 5fc413b..69988de 100644 --- a/app/models/queued_message.rb +++ b/app/models/queued_message.rb @@ -29,39 +29,24 @@ class QueuedMessage < ApplicationRecord include HasMessage + include HasLocking belongs_to :server belongs_to :ip_address, optional: true belongs_to :user, optional: true before_create :allocate_ip_address - after_commit :queue, on: :create - scope :unlocked, -> { where(locked_at: nil) } - scope :retriable, -> { where("retry_after IS NULL OR retry_after < ?", Time.now) } - scope :requeueable, -> { where("retry_after IS NULL OR retry_after < ?", 30.seconds.ago) } + scope :ready_with_delayed_retry, -> { where("retry_after IS NULL OR retry_after < ?", 30.seconds.ago) } - def retriable? - retry_after.nil? || retry_after < Time.now - end - - def queue - UnqueueMessageJob.queue(queue_name, id: id) - end - - def queue! - update_column(:retry_after, nil) - queue - end - - def queue_name - ip_address ? :"outgoing-#{ip_address.id}" : :main + def retry_now + update(retry_after: nil) end def send_bounce return unless message.send_bounces? - Postal::BounceMessage.new(server, message).queue + BounceMessage.new(server, message).queue end def allocate_ip_address @@ -70,40 +55,6 @@ class QueuedMessage < ApplicationRecord self.ip_address = pool.ip_addresses.select_by_priority end - def acquire_lock - time = Time.now - locker = Postal.locker_name - rows = self.class.where(id: id, locked_by: nil, locked_at: nil).update_all(locked_by: locker, locked_at: time) - if rows == 1 - self.locked_by = locker - self.locked_at = time - true - else - false - end - end - - def retry_later(time = nil) - retry_time = time || self.class.calculate_retry_time(attempts, 5.minutes) - self.locked_by = nil - self.locked_at = nil - update_columns(locked_by: nil, locked_at: nil, retry_after: Time.now + retry_time, attempts: attempts + 1) - end - - def unlock - self.locked_by = nil - self.locked_at = nil - update_columns(locked_by: nil, locked_at: nil) - end - - def self.calculate_retry_time(attempts, initial_period) - (1.3**attempts) * initial_period - end - - def locked? - locked_at.present? - end - def batchable_messages(limit = 10) unless locked? raise Postal::Error, "Must lock current message before locking any friends" @@ -114,13 +65,9 @@ class QueuedMessage < ApplicationRecord else time = Time.now locker = Postal.locker_name - self.class.retriable.where(batch_key: batch_key, ip_address_id: ip_address_id, locked_by: nil, locked_at: nil).limit(limit).update_all(locked_by: locker, locked_at: time) + self.class.ready.where(batch_key: batch_key, ip_address_id: ip_address_id, locked_by: nil, locked_at: nil).limit(limit).update_all(locked_by: locker, locked_at: time) QueuedMessage.where(batch_key: batch_key, ip_address_id: ip_address_id, locked_by: locker, locked_at: time).where.not(id: id) end end - def self.requeue_all - unlocked.requeueable.each(&:queue) - end - end diff --git a/app/models/route.rb b/app/models/route.rb index 3ef415c..d1167cc 100644 --- a/app/models/route.rb +++ b/app/models/route.rb @@ -89,7 +89,7 @@ class Route < ApplicationRecord end def forward_address - @forward_address ||= "#{token}@#{Postal.config.dns.route_domain}" + @forward_address ||= "#{token}@#{Postal::Config.dns.route_domain}" end def wildcard? @@ -128,7 +128,7 @@ class Route < ApplicationRecord # # This message will create a suitable number of message objects for messages that - #  are destined for this route. It receives a block which can set the message content + # are destined for this route. It receives a block which can set the message content # but most information is specified already. # # Returns an array of created messages. diff --git a/app/models/scheduled_task.rb b/app/models/scheduled_task.rb new file mode 100644 index 0000000..0989815 --- /dev/null +++ b/app/models/scheduled_task.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: scheduled_tasks +# +# id :bigint not null, primary key +# name :string(255) +# next_run_after :datetime +# +# Indexes +# +# index_scheduled_tasks_on_name (name) UNIQUE +# +class ScheduledTask < ApplicationRecord +end diff --git a/app/models/server.rb b/app/models/server.rb index fd23c4e..096133b 100644 --- a/app/models/server.rb +++ b/app/models/server.rb @@ -71,8 +71,8 @@ class Server < ApplicationRecord default_value :raw_message_retention_days, -> { 30 } default_value :raw_message_retention_size, -> { 2048 } default_value :message_retention_days, -> { 60 } - default_value :spam_threshold, -> { Postal.config.general.default_spam_threshold } - default_value :spam_failure_threshold, -> { Postal.config.general.default_spam_failure_threshold } + default_value :spam_threshold, -> { Postal::Config.postal.default_spam_threshold } + default_value :spam_failure_threshold, -> { Postal::Config.postal.default_spam_failure_threshold } validates :name, presence: true, uniqueness: { scope: :organization_id, case_sensitive: false } validates :mode, inclusion: { in: MODES } @@ -192,37 +192,33 @@ class Server < ApplicationRecord end def send_limit_approaching? - send_limit && (send_volume >= send_limit * 0.90) + return false unless send_limit + + (send_volume >= send_limit * 0.90) end def send_limit_exceeded? - send_limit && send_volume >= send_limit + return false unless send_limit + + send_volume >= send_limit end def send_limit_warning(type) - AppMailer.send("server_send_limit_#{type}", self).deliver + if organization.notification_addresses.present? + AppMailer.send("server_send_limit_#{type}", self).deliver + end + update_column("send_limit_#{type}_notified_at", Time.now) WebhookRequest.trigger(self, "SendLimit#{type.to_s.capitalize}", server: webhook_hash, volume: send_volume, limit: send_limit) end def queue_size - @queue_size ||= queued_messages.retriable.count - end - - def stats - { - queue: queue_size, - held: held_messages, - bounce_rate: bounce_rate, - message_rate: message_rate, - throughput: throughput_stats, - size: message_db.total_size - } + @queue_size ||= queued_messages.ready.count end # Return the domain which can be used to authenticate emails sent from the given e-mail address. # - #  @param address [String] an e-mail address + # @param address [String] an e-mail address # @return [Domain, nil] the domain to use for authentication def authenticated_domain_for_address(address) return nil if address.blank? @@ -274,7 +270,10 @@ class Server < ApplicationRecord self.suspended_at = Time.now self.suspension_reason = reason save! - AppMailer.server_suspended(self).deliver + if organization.notification_addresses.present? + AppMailer.server_suspended(self).deliver + end + true end def unsuspend @@ -283,12 +282,6 @@ class Server < ApplicationRecord save! end - def validate_ip_pool_belongs_to_organization - return unless ip_pool && ip_pool_id_changed? && !organization.ip_pools.include?(ip_pool) - - errors.add :ip_pool_id, "must belong to the organization" - end - def ip_pool_for_message(message) return unless message.scope == "outgoing" @@ -300,46 +293,48 @@ class Server < ApplicationRecord end end end + ip_pool end - def self.triggered_send_limit(type) - servers = where("send_limit_#{type}_at IS NOT NULL AND send_limit_#{type}_at > ?", 3.minutes.ago) - servers.where("send_limit_#{type}_notified_at IS NULL OR send_limit_#{type}_notified_at < ?", 1.hour.ago) + private + + def validate_ip_pool_belongs_to_organization + return unless ip_pool && ip_pool_id_changed? && !organization.ip_pools.include?(ip_pool) + + errors.add :ip_pool_id, "must belong to the organization" end - def self.send_send_limit_notifications - [:approaching, :exceeded].each_with_object({}) do |type, hash| - hash[type] = 0 - servers = triggered_send_limit(type) - next if servers.empty? + class << self - servers.each do |server| - hash[type] += 1 - server.send_limit_warning(type) - end - end - end - - def self.[](id, extra = nil) - server = nil - if id.is_a?(String) - if id =~ /\A(\w+)\/(\w+)\z/ - server = includes(:organization).where(organizations: { permalink: ::Regexp.last_match(1) }, permalink: ::Regexp.last_match(2)).first - end - else - server = where(id: id).first + def triggered_send_limit(type) + servers = where("send_limit_#{type}_at IS NOT NULL AND send_limit_#{type}_at > ?", 3.minutes.ago) + servers.where("send_limit_#{type}_notified_at IS NULL OR send_limit_#{type}_notified_at < ?", 1.hour.ago) end - if extra - if extra.is_a?(String) - server.domains.where(name: extra.to_s).first + def send_send_limit_notifications + [:approaching, :exceeded].each_with_object({}) do |type, hash| + hash[type] = 0 + servers = triggered_send_limit(type) + next if servers.empty? + + servers.each do |server| + hash[type] += 1 + server.send_limit_warning(type) + end + end + end + + def [](id, extra = nil) + if id.is_a?(String) && id =~ /\A(\w+)\/(\w+)\z/ + joins(:organization).where( + organizations: { permalink: ::Regexp.last_match(1) }, permalink: ::Regexp.last_match(2) + ).first else - server.message(extra.to_i) + find_by(id: id.to_i) end - else - server end + end end diff --git a/app/models/track_domain.rb b/app/models/track_domain.rb index 810b6ea..776c1dc 100644 --- a/app/models/track_domain.rb +++ b/app/models/track_domain.rb @@ -54,17 +54,16 @@ class TrackDomain < ApplicationRecord end def check_dns - result = domain.resolver.getresources(full_name, Resolv::DNS::Resource::IN::CNAME) - records = result.map { |r| r.name.to_s.downcase } + records = domain.resolver.cname(full_name) if records.empty? self.dns_status = "Missing" self.dns_error = "There is no record at #{full_name}" - elsif records.size == 1 && records.first == Postal.config.dns.track_domain + elsif records.size == 1 && records.first == Postal::Config.dns.track_domain self.dns_status = "OK" self.dns_error = nil else self.dns_status = "Invalid" - self.dns_error = "There is a CNAME record at #{full_name} but it points to #{records.first} which is incorrect. It should point to #{Postal.config.dns.track_domain}." + self.dns_error = "There is a CNAME record at #{full_name} but it points to #{records.first} which is incorrect. It should point to #{Postal::Config.dns.track_domain}." end self.dns_checked_at = Time.now save! diff --git a/app/models/webhook.rb b/app/models/webhook.rb index a334632..cfa9891 100644 --- a/app/models/webhook.rb +++ b/app/models/webhook.rb @@ -35,12 +35,7 @@ class Webhook < ApplicationRecord scope :enabled, -> { where(enabled: true) } after_save :save_events - - when_attribute :all_events, changes_to: true do - after_save do - webhook_events.destroy_all - end - end + after_save :destroy_events_when_all_events_enabled def events @events ||= webhook_events.map(&:event) @@ -50,13 +45,22 @@ class Webhook < ApplicationRecord @events = value.map(&:to_s).select(&:present?) end + private + def save_events return unless @events @events.each do |event| webhook_events.where(event: event).first_or_create! end + webhook_events.where.not(event: @events).destroy_all end + def destroy_events_when_all_events_enabled + return unless all_events + + webhook_events.destroy_all + end + end diff --git a/app/models/webhook_request.rb b/app/models/webhook_request.rb index 97d4b6f..7609b5a 100644 --- a/app/models/webhook_request.rb +++ b/app/models/webhook_request.rb @@ -5,23 +5,28 @@ # Table name: webhook_requests # # id :integer not null, primary key +# attempts :integer default(0) +# error :text(65535) +# event :string(255) +# locked_at :datetime +# locked_by :string(255) +# payload :text(65535) +# retry_after :datetime +# url :string(255) +# uuid :string(255) +# created_at :datetime # server_id :integer # webhook_id :integer -# url :string(255) -# event :string(255) -# uuid :string(255) -# payload :text(65535) -# attempts :integer default(0) -# retry_after :datetime -# error :text(65535) -# created_at :datetime +# +# Indexes +# +# index_webhook_requests_on_locked_by (locked_by) # class WebhookRequest < ApplicationRecord include HasUUID - - RETRIES = { 1 => 2.minutes, 2 => 3.minutes, 3 => 6.minutes, 4 => 10.minutes, 5 => 15.minutes }.freeze + include HasLocking belongs_to :server belongs_to :webhook, optional: true @@ -31,64 +36,19 @@ class WebhookRequest < ApplicationRecord serialize :payload, Hash - after_commit :queue, on: :create + class << self - def self.trigger(server, event, payload = {}) - unless server.is_a?(Server) - server = Server.find(server.to_i) - end - - webhooks = server.webhooks.enabled.includes(:webhook_events).references(:webhook_events).where("webhooks.all_events = ? OR webhook_events.event = ?", true, event) - webhooks.each do |webhook| - server.webhook_requests.create!(event: event, payload: payload, webhook: webhook, url: webhook.url) - end - end - - def self.requeue_all - where("retry_after < ?", Time.now).find_each(&:queue) - end - - def queue - WebhookDeliveryJob.queue(:main, id: id) - end - - def deliver - logger = Postal.logger_for(:webhooks) - payload = { event: event, timestamp: created_at.to_f, payload: self.payload, uuid: uuid }.to_json - logger.info "[#{id}] Sending webhook request to `#{url}`" - result = Postal::HTTP.post(url, sign: true, json: payload, timeout: 5) - self.attempts += 1 - self.retry_after = RETRIES[self.attempts]&.from_now - server.message_db.webhooks.record( - event: event, - url: url, - webhook_id: webhook_id, - attempt: self.attempts, - timestamp: Time.now.to_f, - payload: self.payload.to_json, - uuid: uuid, - status_code: result[:code], - body: result[:body], - will_retry: (retry_after ? 0 : 1) - ) - - if result[:code] >= 200 && result[:code] < 300 - logger.info "[#{id}] -> Received #{result[:code]} status code. That's OK." - destroy - webhook&.update_column(:last_used_at, Time.now) - true - else - logger.error "[#{id}] -> Received #{result[:code]} status code. That's not OK." - self.error = "Couldn't send to URL. Code received was #{result[:code]}" - if retry_after - logger.info "[#{id}] -> Will retry #{retry_after} (this was attempt #{self.attempts})" - save - else - logger.info "[#{id}] -> Have tried #{self.attempts} times. Giving up." - destroy + def trigger(server, event, payload = {}) + unless server.is_a?(Server) + server = Server.find(server.to_i) + end + + webhooks = server.webhooks.enabled.includes(:webhook_events).references(:webhook_events).where("webhooks.all_events = ? OR webhook_events.event = ?", true, event) + webhooks.each do |webhook| + server.webhook_requests.create!(event: event, payload: payload, webhook: webhook, url: webhook.url) end - false end + end end diff --git a/app/models/worker_role.rb b/app/models/worker_role.rb new file mode 100644 index 0000000..22e83ea --- /dev/null +++ b/app/models/worker_role.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: worker_roles +# +# id :bigint not null, primary key +# acquired_at :datetime +# role :string(255) +# worker :string(255) +# +# Indexes +# +# index_worker_roles_on_role (role) UNIQUE +# +class WorkerRole < ApplicationRecord + + class << self + + # Acquire or renew a lock for the given role. + # + # @param role [String] The name of the role to acquire + # @return [Symbol, false] True if the lock was acquired or renewed, false otherwise + def acquire(role) + # update our existing lock if we already have one + updates = where(role: role, worker: Postal.locker_name).update_all(acquired_at: Time.current) + return :renewed if updates.positive? + + # attempt to steal a role from another worker + updates = where(role: role).where("acquired_at is null OR acquired_at < ?", 5.minutes.ago) + .update_all(acquired_at: Time.current, worker: Postal.locker_name) + return :stolen if updates.positive? + + # attempt to create a new role for this worker + begin + create!(role: role, worker: Postal.locker_name, acquired_at: Time.current) + :created + rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid + false + end + end + + # Release a lock for the given role for the current process. + # + # @param role [String] The name of the role to release + # @return [Boolean] True if the lock was released, false otherwise + def release(role) + updates = where(role: role, worker: Postal.locker_name).delete_all + updates.positive? + end + + end + +end diff --git a/app/scheduled_tasks/action_deletions_scheduled_task.rb b/app/scheduled_tasks/action_deletions_scheduled_task.rb new file mode 100644 index 0000000..e5c7d5a --- /dev/null +++ b/app/scheduled_tasks/action_deletions_scheduled_task.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class ActionDeletionsScheduledTask < ApplicationScheduledTask + + def call + Organization.deleted.each do |org| + logger.info "permanently removing organization #{org.id} (#{org.permalink})" + org.destroy + end + + Server.deleted.each do |server| + logger.info "permanently removing server #{server.id} (#{server.full_permalink})" + server.destroy + end + end + +end diff --git a/app/scheduled_tasks/application_scheduled_task.rb b/app/scheduled_tasks/application_scheduled_task.rb new file mode 100644 index 0000000..0aebe09 --- /dev/null +++ b/app/scheduled_tasks/application_scheduled_task.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +class ApplicationScheduledTask + + def initialize(logger:) + @logger = logger + end + + def call + # override me + end + + attr_reader :logger + + class << self + + def next_run_after + quarter_past_each_hour + end + + private + + def quarter_past_each_hour + time = Time.current + time = time.change(min: 15, sec: 0) + time += 1.hour if time < Time.current + time + end + + def quarter_to_each_hour + time = Time.current + time = time.change(min: 45, sec: 0) + time += 1.hour if time < Time.current + time + end + + def three_am + time = Time.current + time = time.change(hour: 3, min: 0, sec: 0) + time += 1.day if time < Time.current + time + end + + end + +end diff --git a/app/jobs/check_all_dns_job.rb b/app/scheduled_tasks/check_all_dns_scheduled_task.rb similarity index 62% rename from app/jobs/check_all_dns_job.rb rename to app/scheduled_tasks/check_all_dns_scheduled_task.rb index 0393afa..140d4a8 100644 --- a/app/jobs/check_all_dns_job.rb +++ b/app/scheduled_tasks/check_all_dns_scheduled_task.rb @@ -1,15 +1,15 @@ # frozen_string_literal: true -class CheckAllDNSJob < Postal::Job +class CheckAllDNSScheduledTask < ApplicationScheduledTask - def perform + def call Domain.where.not(dns_checked_at: nil).where("dns_checked_at <= ?", 1.hour.ago).each do |domain| - log "Checking DNS for domain: #{domain.name}" + logger.info "checking DNS for domain: #{domain.name}" domain.check_dns(:auto) end TrackDomain.where("dns_checked_at IS NULL OR dns_checked_at <= ?", 1.hour.ago).includes(:domain).each do |domain| - log "Checking DNS for track domain: #{domain.full_name}" + logger.info "checking DNS for track domain: #{domain.full_name}" domain.check_dns end end diff --git a/app/jobs/cleanup_authie_sessions_job.rb b/app/scheduled_tasks/cleanup_authie_sessions_scheduled_task.rb similarity index 55% rename from app/jobs/cleanup_authie_sessions_job.rb rename to app/scheduled_tasks/cleanup_authie_sessions_scheduled_task.rb index 36a93d0..333fec5 100644 --- a/app/jobs/cleanup_authie_sessions_job.rb +++ b/app/scheduled_tasks/cleanup_authie_sessions_scheduled_task.rb @@ -2,9 +2,9 @@ require "authie/session" -class CleanupAuthieSessionsJob < Postal::Job +class CleanupAuthieSessionsScheduledTask < ApplicationScheduledTask - def perform + def call Authie::Session.cleanup end diff --git a/app/jobs/expire_held_messages_job.rb b/app/scheduled_tasks/expire_held_messages_scheduled_task.rb similarity index 77% rename from app/jobs/expire_held_messages_job.rb rename to app/scheduled_tasks/expire_held_messages_scheduled_task.rb index e54d052..7abfa5b 100644 --- a/app/jobs/expire_held_messages_job.rb +++ b/app/scheduled_tasks/expire_held_messages_scheduled_task.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -class ExpireHeldMessagesJob < Postal::Job +class ExpireHeldMessagesScheduledTask < ApplicationScheduledTask - def perform + def call Server.all.each do |server| messages = server.message_db.messages(where: { status: "Held", diff --git a/app/jobs/process_message_retention_job.rb b/app/scheduled_tasks/process_message_retention_scheduled_task.rb similarity index 55% rename from app/jobs/process_message_retention_job.rb rename to app/scheduled_tasks/process_message_retention_scheduled_task.rb index 35c3bf3..671ec40 100644 --- a/app/jobs/process_message_retention_job.rb +++ b/app/scheduled_tasks/process_message_retention_scheduled_task.rb @@ -1,25 +1,29 @@ # frozen_string_literal: true -class ProcessMessageRetentionJob < Postal::Job +class ProcessMessageRetentionScheduledTask < ApplicationScheduledTask def perform Server.all.each do |server| if server.raw_message_retention_days # If the server has a maximum number of retained raw messages, remove any that are older than this - log "Tidying raw messages (by days) for #{server.permalink} (ID: #{server.id}). Keeping #{server.raw_message_retention_days} days." + logger.info "Tidying raw messages (by days) for #{server.permalink} (ID: #{server.id}). Keeping #{server.raw_message_retention_days} days." server.message_db.provisioner.remove_raw_tables_older_than(server.raw_message_retention_days) end if server.raw_message_retention_size - log "Tidying raw messages (by size) for #{server.permalink} (ID: #{server.id}). Keeping #{server.raw_message_retention_size} MB of data." + logger.info "Tidying raw messages (by size) for #{server.permalink} (ID: #{server.id}). Keeping #{server.raw_message_retention_size} MB of data." server.message_db.provisioner.remove_raw_tables_until_less_than_size(server.raw_message_retention_size * 1024 * 1024) end if server.message_retention_days - log "Tidying messages for #{server.permalink} (ID: #{server.id}). Keeping #{server.message_retention_days} days." + logger.info "Tidying messages for #{server.permalink} (ID: #{server.id}). Keeping #{server.message_retention_days} days." server.message_db.provisioner.remove_messages(server.message_retention_days) end end end + def self.next_run_after + three_am + end + end diff --git a/app/scheduled_tasks/prune_suppression_lists_scheduled_task.rb b/app/scheduled_tasks/prune_suppression_lists_scheduled_task.rb new file mode 100644 index 0000000..2f73904 --- /dev/null +++ b/app/scheduled_tasks/prune_suppression_lists_scheduled_task.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class PruneSuppressionListsScheduledTask < ApplicationScheduledTask + + def call + Server.all.each do |s| + logger.info "Pruning suppression lists for server #{s.id}" + s.message_db.suppression_list.prune + end + end + + def self.next_run_after + three_am + end + +end diff --git a/app/scheduled_tasks/prune_webhook_requests_scheduled_task.rb b/app/scheduled_tasks/prune_webhook_requests_scheduled_task.rb new file mode 100644 index 0000000..95147d7 --- /dev/null +++ b/app/scheduled_tasks/prune_webhook_requests_scheduled_task.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class PruneWebhookRequestsScheduledTask < ApplicationScheduledTask + + def call + Server.all.each do |s| + logger.info "Pruning webhook requests for server #{s.id}" + s.message_db.webhooks.prune + end + end + + def self.next_run_after + quarter_to_each_hour + end + +end diff --git a/app/scheduled_tasks/send_notifications_scheduled_task.rb b/app/scheduled_tasks/send_notifications_scheduled_task.rb new file mode 100644 index 0000000..53f3f6d --- /dev/null +++ b/app/scheduled_tasks/send_notifications_scheduled_task.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class SendNotificationsScheduledTask < ApplicationScheduledTask + + def call + Server.send_send_limit_notifications + end + + def self.next_run_after + 1.minute.from_now + end + +end diff --git a/app/senders/base_sender.rb b/app/senders/base_sender.rb new file mode 100644 index 0000000..a009e9f --- /dev/null +++ b/app/senders/base_sender.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class BaseSender + + def start + end + + def send_message(message) + end + + def finish + end + +end diff --git a/app/senders/http_sender.rb b/app/senders/http_sender.rb new file mode 100644 index 0000000..4da8c7f --- /dev/null +++ b/app/senders/http_sender.rb @@ -0,0 +1,134 @@ +# frozen_string_literal: true + +class HTTPSender < BaseSender + + def initialize(endpoint, options = {}) + super() + @endpoint = endpoint + @options = options + @log_id = Nifty::Utils::RandomString.generate(length: 8).upcase + end + + def send_message(message) + start_time = Time.now + result = SendResult.new + result.log_id = @log_id + + request_options = {} + request_options[:sign] = true + request_options[:timeout] = @endpoint.timeout || 5 + case @endpoint.encoding + when "BodyAsJSON" + request_options[:json] = parameters(message, flat: false).to_json + when "FormData" + request_options[:params] = parameters(message, flat: true) + end + + log "Sending request to #{@endpoint.url}" + response = Postal::HTTP.post(@endpoint.url, request_options) + result.secure = !!response[:secure] # rubocop:disable Style/DoubleNegation + result.details = "Received a #{response[:code]} from #{@endpoint.url}" + log " -> Received: #{response[:code]}" + if response[:body] + log " -> Body: #{response[:body][0, 255]}" + result.output = response[:body].to_s[0, 500].strip + end + if response[:code] >= 200 && response[:code] < 300 + # This is considered a success + result.type = "Sent" + elsif response[:code] >= 500 && response[:code] < 600 + # This is temporary. They might fix their server so it should soft fail. + result.type = "SoftFail" + result.retry = true + elsif response[:code].negative? + # Connection/SSL etc... errors + result.type = "SoftFail" + result.retry = true + result.connect_error = true + elsif response[:code] == 429 + # Rate limit exceeded, treat as a hard fail and don't send bounces + result.type = "HardFail" + result.suppress_bounce = true + else + # This is permanent. Any other error isn't cool with us. + result.type = "HardFail" + end + result.time = (Time.now - start_time).to_f.round(2) + result + end + + private + + def log(text) + Postal.logger.info text, id: @log_id, component: "http-sender" + end + + def parameters(message, options = {}) + case @endpoint.format + when "Hash" + hash = { + id: message.id, + rcpt_to: message.rcpt_to, + mail_from: message.mail_from, + token: message.token, + subject: message.subject, + message_id: message.message_id, + timestamp: message.timestamp.to_f, + size: message.size, + spam_status: message.spam_status, + bounce: message.bounce, + received_with_ssl: message.received_with_ssl, + to: message.headers["to"]&.last, + cc: message.headers["cc"]&.last, + from: message.headers["from"]&.last, + date: message.headers["date"]&.last, + in_reply_to: message.headers["in-reply-to"]&.last, + references: message.headers["references"]&.last, + html_body: message.html_body, + attachment_quantity: message.attachments.size, + auto_submitted: message.headers["auto-submitted"]&.last, + reply_to: message.headers["reply-to"] + } + + if @endpoint.strip_replies + hash[:plain_body], hash[:replies_from_plain_body] = ReplySeparator.separate(message.plain_body) + else + hash[:plain_body] = message.plain_body + end + + if @endpoint.include_attachments? + if options[:flat] + message.attachments.each_with_index do |a, i| + hash["attachments[#{i}][filename]"] = a.filename + hash["attachments[#{i}][content_type]"] = a.content_type + hash["attachments[#{i}][size]"] = a.body.to_s.bytesize.to_s + hash["attachments[#{i}][data]"] = Base64.encode64(a.body.to_s) + end + else + hash[:attachments] = message.attachments.map do |a| + { + filename: a.filename, + content_type: a.mime_type, + size: a.body.to_s.bytesize, + data: Base64.encode64(a.body.to_s) + } + end + end + end + + hash + when "RawMessage" + { + id: message.id, + rcpt_to: message.rcpt_to, + mail_from: message.mail_from, + message: Base64.encode64(message.raw_message), + base64: true, + size: message.size.to_i + } + else + {} + end + end + +end diff --git a/app/senders/send_result.rb b/app/senders/send_result.rb new file mode 100644 index 0000000..c8a6353 --- /dev/null +++ b/app/senders/send_result.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class SendResult + + attr_accessor :type + attr_accessor :details + attr_accessor :retry + attr_accessor :output + attr_accessor :secure + attr_accessor :connect_error + attr_accessor :log_id + attr_accessor :time + attr_accessor :suppress_bounce + + def initialize + @details = "" + yield self if block_given? + end + +end diff --git a/app/senders/smtp_sender.rb b/app/senders/smtp_sender.rb new file mode 100644 index 0000000..440fa7c --- /dev/null +++ b/app/senders/smtp_sender.rb @@ -0,0 +1,258 @@ +# frozen_string_literal: true + +class SMTPSender < BaseSender + + attr_reader :endpoints + + # @param domain [String] the domain to send mesages to + # @param source_ip_address [IPAddress] the IP address to send messages from + # @param log_id [String] an ID to use when logging requests + def initialize(domain, source_ip_address = nil, servers: nil, log_id: nil, rcpt_to: nil) + super() + @domain = domain + @source_ip_address = source_ip_address + @rcpt_to = rcpt_to + + # An array of servers to forcefully send the message to + @servers = servers + # Stores all connection errors which we have seen during this send sesssion. + @connection_errors = [] + # Stores all endpoints that we have attempted to deliver mail to + @endpoints = [] + # Generate a log ID which can be used if none has been provided to trace + # this SMTP session. + @log_id = log_id || SecureRandom.alphanumeric(8).upcase + end + + def start + servers = @servers || self.class.smtp_relays || resolve_mx_records_for_domain || [] + + servers.each do |server| + server.endpoints.each do |endpoint| + result = connect_to_endpoint(endpoint) + return endpoint if result + end + end + + false + end + + def send_message(message) + # If we don't have a current endpoint than we should raise an error. + if @current_endpoint.nil? + return create_result("SoftFail") do |r| + r.retry = true + r.details = "No SMTP servers were available for #{@domain}." + if @endpoints.empty? + r.details += " No hosts to try." + else + hostnames = @endpoints.map { |e| e.server.hostname }.uniq + r.details += " Tried #{hostnames.to_sentence}." + end + r.output = @connection_errors.join(", ") + r.connect_error = true + end + end + + mail_from = determine_mail_from_for_message(message) + raw_message = message.raw_message + + # Append the Resent-Sender header to the mesage to include the + # MAIL FROM if the installation is configured to use that? + if Postal::Config.postal.use_resent_sender_header? + raw_message = "Resent-Sender: #{mail_from}\r\n" + raw_message + end + + rcpt_to = determine_rcpt_to_for_message(message) + logger.info "Sending message #{message.server.id}::#{message.id} to #{rcpt_to}" + send_message_to_smtp_client(raw_message, mail_from, rcpt_to) + end + + def finish + @endpoints.each(&:finish_smtp_session) + end + + private + + # Take a message and attempt to send it to the SMTP server that we are + # currently connected to. If there is a connection error, we will just + # reset the client and retry again once. + # + # @param raw_message [String] the raw message to send + # @param mail_from [String] the MAIL FROM address to use + # @param rcpt_to [String] the RCPT TO address to use + # @param retry_on_connection_error [Boolean] if true, we will retry the connection if there is an error + # + # @return [SendResult] + def send_message_to_smtp_client(raw_message, mail_from, rcpt_to, retry_on_connection_error: true) + start_time = Time.now + smtp_result = @current_endpoint.send_message(raw_message, mail_from, [rcpt_to]) + logger.info "Accepted by #{@current_endpoint} for #{rcpt_to}" + create_result("Sent", start_time) do |r| + r.details = "Message for #{rcpt_to} accepted by #{@current_endpoint}" + r.details += " (from #{@current_endpoint.smtp_client.source_address})" if @current_endpoint.smtp_client.source_address + r.output = smtp_result.string + end + rescue Net::SMTPServerBusy, Net::SMTPAuthenticationError, Net::SMTPSyntaxError, Net::SMTPUnknownError, Net::ReadTimeout => e + logger.error "#{e.class}: #{e.message}" + @current_endpoint.reset_smtp_session + + create_result("SoftFail", start_time) do |r| + r.details = "Temporary SMTP delivery error when sending to #{@current_endpoint}" + r.output = e.message + if e.message =~ /(\d+) seconds/ + r.retry = ::Regexp.last_match(1).to_i + 10 + elsif e.message =~ /(\d+) minutes/ + r.retry = (::Regexp.last_match(1).to_i * 60) + 10 + else + r.retry = true + end + end + rescue Net::SMTPFatalError => e + logger.error "#{e.class}: #{e.message}" + @current_endpoint.reset_smtp_session + + create_result("HardFail", start_time) do |r| + r.details = "Permanent SMTP delivery error when sending to #{@current_endpoint}" + r.output = e.message + end + rescue StandardError => e + logger.error "#{e.class}: #{e.message}" + @current_endpoint.reset_smtp_session + + if defined?(Sentry) + # Sentry.capture_exception(e, extra: { log_id: @log_id, server_id: message.server.id, message_id: message.id }) + end + + create_result("SoftFail", start_time) do |r| + r.type = "SoftFail" + r.retry = true + r.details = "An error occurred while sending the message to #{@current_endpoint}" + r.output = e.message + end + end + + # Return the MAIL FROM which should be used for the given message + # + # @param message [MessageDB::Message] + # @return [String] + def determine_mail_from_for_message(message) + return "" if message.bounce + + # If the domain has a valid custom return path configured, return + # that. + if message.domain.return_path_status == "OK" + return "#{message.server.token}@#{message.domain.return_path_domain}" + end + + "#{message.server.token}@#{Postal::Config.dns.return_path_domain}" + end + + # Return the RCPT TO to use for the given message in this sending session + # + # @param message [MessageDB::Message] + # @return [String] + def determine_rcpt_to_for_message(message) + return @rcpt_to if @rcpt_to + + message.rcpt_to + end + + # Return an array of server hostnames which should receive this message + # + # @return [Array] + def resolve_mx_records_for_domain + hostnames = DNSResolver.local.mx(@domain, raise_timeout_errors: true).map(&:last) + return [SMTPClient::Server.new(@domain)] if hostnames.empty? + + hostnames.map { |hostname| SMTPClient::Server.new(hostname) } + end + + # Attempt to begin an SMTP sesssion for the given endpoint. If successful, this endpoint + # becomes the current endpoints for the SMTP sender. + # + # Returns true if the session was established. + # Returns false if the session could not be established. + # + # @param endpoint [SMTPClient::Endpoint] + # @return [Boolean] + def connect_to_endpoint(endpoint, allow_ssl: true) + if @source_ip_address && @source_ip_address.ipv6.blank? && endpoint.ipv6? + # Don't try to use IPv6 if the IP address we're sending from doesn't support it. + return false + end + + # Add this endpoint to the list of endpoints that we have attempted to connect to + @endpoints << endpoint unless @endpoints.include?(endpoint) + + endpoint.start_smtp_session(allow_ssl: allow_ssl, source_ip_address: @source_ip_address) + logger.info "Connected to #{endpoint}" + @current_endpoint = endpoint + + true + rescue StandardError => e + # Disconnect the SMTP client if we get any errors to avoid leaving + # a connection around. + endpoint.finish_smtp_session + + # If we get an SSL error, we can retry a connection without + # ssl. + if e.is_a?(OpenSSL::SSL::SSLError) && endpoint.server.ssl_mode == "Auto" + logger.error "SSL error (#{e.message}), retrying without SSL" + return connect_to_endpoint(endpoint, allow_ssl: false) + end + + # Otherwise, just log the connection error and return false + logger.error "Cannot connect to #{endpoint} (#{e.class}: #{e.message})" + @connection_errors << e.message unless @connection_errors.include?(e.message) + + false + end + + # Create a new result object + # + # @param type [String] the type of result + # @param start_time [Time] the time the operation started + # @yieldparam [SendResult] the result object + # @yieldreturn [void] + # + # @return [SendResult] + def create_result(type, start_time = nil) + result = SendResult.new + result.type = type + result.log_id = @log_id + result.secure = @current_endpoint&.smtp_client&.secure_socket? ? true : false + yield result if block_given? + if start_time + result.time = (Time.now - start_time).to_f.round(2) + end + result + end + + def logger + @logger ||= Postal.logger.create_tagged_logger(log_id: @log_id) + end + + class << self + + # Return an array of SMTP relays as configured. Returns nil + # if no SMTP relays are configured. + # + def smtp_relays + return @smtp_relays if instance_variable_defined?("@smtp_relays") + + relays = Postal::Config.postal.smtp_relays + return nil if relays.nil? + + relays.map do |relay| + next unless relay.host.present? + + SMTPClient::Server.new(relay.host, relay.port, ssl_mode: relay.ssl_mode) + end.compact + + @smtp_relays = hosts.empty? ? nil : hosts + end + + end + +end diff --git a/app/services/webhook_delivery_service.rb b/app/services/webhook_delivery_service.rb new file mode 100644 index 0000000..4aeb50c --- /dev/null +++ b/app/services/webhook_delivery_service.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +class WebhookDeliveryService + + RETRIES = { 1 => 2.minutes, 2 => 3.minutes, 3 => 6.minutes, 4 => 10.minutes, 5 => 15.minutes }.freeze + + def initialize(webhook_request:) + @webhook_request = webhook_request + end + + def call + logger.tagged(webhook: @webhook_request.webhook_id, webhook_request: @webhook_request.id) do + generate_payload + send_request + record_attempt + appreciate_http_result + update_webhook_request + end + end + + def success? + @success == true + end + + private + + def generate_payload + @payload = { + event: @webhook_request.event, + timestamp: @webhook_request.created_at.to_f, + payload: @webhook_request.payload, + uuid: @webhook_request.uuid + }.to_json + end + + def send_request + @http_result = Postal::HTTP.post(@webhook_request.url, + sign: true, + json: @payload, + timeout: 5) + + @success = (@http_result[:code] >= 200 && @http_result[:code] < 300) + end + + def record_attempt + @webhook_request.attempts += 1 + + if success? + @webhook_request.retry_after = nil + else + @webhook_request.retry_after = RETRIES[@webhook_request.attempts]&.from_now + end + + @attempt = @webhook_request.server.message_db.webhooks.record( + event: @webhook_request.event, + url: @webhook_request.url, + webhook_id: @webhook_request.webhook_id, + attempt: @webhook_request.attempts, + timestamp: Time.now.to_f, + payload: @webhook_request.payload.to_json, + uuid: @webhook_request.uuid, + status_code: @http_result[:code], + body: @http_result[:body], + will_retry: @webhook_request.retry_after.present? + ) + end + + def appreciate_http_result + if success? + logger.info "Received #{@http_result[:code]} status code. That's OK." + @webhook_request.destroy! + @webhook_request.webhook&.update_column(:last_used_at, Time.current) + return + end + + logger.error "Received #{@http_result[:code]} status code. That's not OK." + @webhook_request.error = "Couldn't send to URL. Code received was #{@http_result[:code]}" + end + + def update_webhook_request + if @webhook_request.retry_after + logger.info "Will retry #{@webhook_request.retry_after} (this was attempt #{@webhook_request.attempts})" + @webhook_request.locked_by = nil + @webhook_request.locked_at = nil + @webhook_request.save! + return + end + + logger.info "Have tried #{@webhook_request.attempts} times. Giving up." + @webhook_request.destroy! + end + + def logger + Postal.logger + end + +end diff --git a/app/util/has_prometheus_metrics.rb b/app/util/has_prometheus_metrics.rb new file mode 100644 index 0000000..bd2224e --- /dev/null +++ b/app/util/has_prometheus_metrics.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module HasPrometheusMetrics + + def register_prometheus_counter(name, **kwargs) + counter = Prometheus::Client::Counter.new(name, **kwargs) + registry.register(counter) + end + + def register_prometheus_histogram(name, **kwargs) + histogram = Prometheus::Client::Histogram.new(name, **kwargs) + registry.register(histogram) + end + + def increment_prometheus_counter(name, labels: {}) + counter = registry.get(name) + return if counter.nil? + + counter.increment(labels: labels) + end + + def observe_prometheus_histogram(name, time, labels: {}) + histogram = registry.get(name) + return if histogram.nil? + + histogram.observe(time, labels: labels) + end + + private + + def registry + Prometheus::Client.registry + end + +end diff --git a/app/util/health_server.rb b/app/util/health_server.rb new file mode 100644 index 0000000..9e8cef2 --- /dev/null +++ b/app/util/health_server.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +require "socket" +require "rack/handler/webrick" +require "prometheus/client/formats/text" + +class HealthServer + + def initialize(name: "unnamed-process") + @name = name + end + + def call(env) + case env["PATH_INFO"] + when "/health" + ok + when "/metrics" + metrics + when "/" + root + else + not_found + end + end + + private + + def root + [200, { "Content-Type" => "text/plain" }, ["#{@name} (pid: #{Process.pid}, host: #{hostname})"]] + end + + def ok + [200, { "Content-Type" => "text/plain" }, ["OK"]] + end + + def not_found + [404, { "Content-Type" => "text/plain" }, ["Not Found"]] + end + + def metrics + registry = Prometheus::Client.registry + body = Prometheus::Client::Formats::Text.marshal(registry) + [200, { "Content-Type" => "text/plain" }, [body]] + end + + def hostname + Socket.gethostname + rescue StandardError + "unknown-hostname" + end + + class << self + + def run(default_port:, default_bind_address:, **options) + port = ENV.fetch("HEALTH_SERVER_PORT", default_port) + bind_address = ENV.fetch("HEALTH_SERVER_BIND_ADDRESS", default_bind_address) + + Rack::Handler::WEBrick.run(new(**options), + Port: port, + BindAddress: bind_address, + AccessLog: [], + Logger: LoggerProxy.new) + rescue Errno::EADDRINUSE + Postal.logger.info "health server port (#{bind_address}:#{port}) is already " \ + "in use, not starting health server" + end + + def start(**options) + thread = Thread.new { run(**options) } + thread.abort_on_exception = false + thread + end + + end + + class LoggerProxy + + [:info, :debug, :warn, :error, :fatal].each do |severity| + define_method(severity) do |message| + add(severity, message) + end + + define_method("#{severity}?") do + severity != :debug + end + end + + def add(severity, message) + return if severity == :debug + + case message + when /\AWEBrick::HTTPServer#start:.*port=(\d+)/ + Postal.logger.info "started health server on port #{::Regexp.last_match(1)}", component: "health-server" + when /\AWEBrick::HTTPServer#start done/ + Postal.logger.info "stopped health server", component: "health-server" + when /\AWEBrick [\d.]+/, + /\Aruby ([\d.]+)/, + /\ARack::Handler::WEBrick is mounted/, + /\Aclose TCPSocket/, + /\Agoing to shutdown/ + # Don't actually print routine messages to avoid too much + # clutter when processes start it + else + Postal.logger.debug message, component: "health-server" + end + end + + end + +end diff --git a/lib/postal/user_creator.rb b/app/util/user_creator.rb similarity index 93% rename from lib/postal/user_creator.rb rename to app/util/user_creator.rb index 27bb973..49b5d70 100644 --- a/lib/postal/user_creator.rb +++ b/app/util/user_creator.rb @@ -2,10 +2,11 @@ require "highline" -module Postal - module UserCreator +module UserCreator - def self.start(&block) + class << self + + def start(&block) cli = HighLine.new puts "\e[32mPostal User Creator\e[0m" puts "Enter the information required to create a new Postal user." @@ -31,4 +32,5 @@ module Postal end end + end diff --git a/app/views/app_mailer/password_reset.text.erb b/app/views/app_mailer/password_reset.text.erb index 59bdc22..0c72e55 100644 --- a/app/views/app_mailer/password_reset.text.erb +++ b/app/views/app_mailer/password_reset.text.erb @@ -8,5 +8,5 @@ If you didn't request this, you can ignore this e-mail. Thanks, -<%= Postal.smtp_from_name %> -<%= Postal.smtp_from_address %> +<%= Postal::Config.smtp.from_name %> +<%= Postal::Config.smtp.from_address %> diff --git a/app/views/app_mailer/server_send_limit_approaching.text.erb b/app/views/app_mailer/server_send_limit_approaching.text.erb index 3f6e1e9..c77dce5 100644 --- a/app/views/app_mailer/server_send_limit_approaching.text.erb +++ b/app/views/app_mailer/server_send_limit_approaching.text.erb @@ -13,5 +13,5 @@ You can view more information about this server at: Thanks, -<%= Postal.smtp_from_name %> -<%= Postal.smtp_from_address %> +<%= Postal::Config.smtp.from_name %> +<%= Postal::Config.smtp.from_address %> diff --git a/app/views/app_mailer/server_send_limit_exceeded.text.erb b/app/views/app_mailer/server_send_limit_exceeded.text.erb index 8b11982..6c755b6 100644 --- a/app/views/app_mailer/server_send_limit_exceeded.text.erb +++ b/app/views/app_mailer/server_send_limit_exceeded.text.erb @@ -13,5 +13,5 @@ You can view more information about this server at: Thanks, -<%= Postal.smtp_from_name %> -<%= Postal.smtp_from_address %> +<%= Postal::Config.smtp.from_name %> +<%= Postal::Config.smtp.from_address %> diff --git a/app/views/app_mailer/server_suspended.text.erb b/app/views/app_mailer/server_suspended.text.erb index 28a2f1d..1667478 100644 --- a/app/views/app_mailer/server_suspended.text.erb +++ b/app/views/app_mailer/server_suspended.text.erb @@ -8,5 +8,5 @@ Reason: <%= @server.actual_suspension_reason %> Thanks, -<%= Postal.smtp_from_name %> -<%= Postal.smtp_from_address %> +<%= Postal::Config.smtp.from_name %> +<%= Postal::Config.smtp.from_address %> diff --git a/app/views/app_mailer/verify_domain.text.erb b/app/views/app_mailer/verify_domain.text.erb index 43e22df..1d1f889 100644 --- a/app/views/app_mailer/verify_domain.text.erb +++ b/app/views/app_mailer/verify_domain.text.erb @@ -10,5 +10,5 @@ If you don't agree, just ignore this e-mail. Thanks, -<%= Postal.smtp_from_name %> -<%= Postal.smtp_from_address %> +<%= Postal::Config.smtp.from_name %> +<%= Postal::Config.smtp.from_address %> diff --git a/app/views/domains/index.html.haml b/app/views/domains/index.html.haml index eafd3a3..2238e01 100644 --- a/app/views/domains/index.html.haml +++ b/app/views/domains/index.html.haml @@ -63,7 +63,7 @@ %ul.domainList__properties - if domain.verified? - %li.domainList__verificationTime Verified on #{domain.verified_at.to_s(:long)} + %li.domainList__verificationTime Verified on #{domain.verified_at.to_fs(:long)} - else %li= link_to "Verify this domain", [:verify, organization, @server, domain], :class => "domainList__verificationLink" %li.domainList__links diff --git a/app/views/domains/setup.html.haml b/app/views/domains/setup.html.haml index cb2f47f..34d9e5a 100644 --- a/app/views/domains/setup.html.haml +++ b/app/views/domains/setup.html.haml @@ -43,7 +43,7 @@ %p.pageContent__text You need to add a TXT record at the apex/root of your domain (@) with the following content. If you already send mail from another service, you may just need to add - include:#{Postal.config.dns.spf_include} to your existing record. + include:#{Postal::Config.dns.spf_include} to your existing record. %pre.codeBlock.u-margin= @domain.spf_record %h3.pageContent__subTitle DKIM Record @@ -78,7 +78,7 @@ %p.pageContent__text This is optional but we recommend adding this to improve deliverability. You should add a CNAME record at #{@domain.return_path_domain} to point to the hostname below. - %pre.codeBlock.u-margin= Postal.config.dns.return_path + %pre.codeBlock.u-margin= Postal::Config.dns.return_path_domain %h3.pageContent__subTitle MX Records @@ -99,6 +99,4 @@ If you wish to receive incoming e-mail for this domain, you need to add the following MX records to the domain. You don't have to do this and we'll only tell you if they're set up or not. Both records should be priority 10. - %pre.codeBlock.u-margin= Postal.config.dns.mx_records.join("\n") - - + %pre.codeBlock.u-margin= Postal::Config.dns.mx_records.join("\n") diff --git a/app/views/help/incoming.html.haml b/app/views/help/incoming.html.haml index 351ef4d..19da1bf 100644 --- a/app/views/help/incoming.html.haml +++ b/app/views/help/incoming.html.haml @@ -36,6 +36,5 @@ %dl.pageContent__definitions %dt MX Records %dd - - for mx in Postal.config.dns.mx_records + - for mx in Postal::Config.dns.mx_records %p.pageContent__definitionCode= mx - diff --git a/app/views/help/outgoing.html.haml b/app/views/help/outgoing.html.haml index 1bd420c..7203b8e 100644 --- a/app/views/help/outgoing.html.haml +++ b/app/views/help/outgoing.html.haml @@ -18,7 +18,7 @@ Mail servers can be enabled to send mail from any domain by the administrator. %li If a message cannot be delivered, the system will not send you a bounce message but dispatch a webhook (if you set one up). - If a message delivery fails but can be retried, the system will try #{Postal.config.general.maximum_delivery_attempts} times to deliver it before giving up. + If a message delivery fails but can be retried, the system will try #{Postal::Config.postal.default_maximum_delivery_attempts} times to deliver it before giving up. .u-margin %h2.pageContent__subTitle Sending using SMTP %p.pageContent__text @@ -27,10 +27,10 @@ %dl.pageContent__definitions %dt SMTP Server Address %dd - %p.pageContent__definitionCode= Postal.config.dns.smtp_server_hostname + %p.pageContent__definitionCode= Postal::Config.postal.smtp_hostname %dt Port %dd - %p.pageContent__definitionCode= Postal.config.smtp_server.port + %p.pageContent__definitionCode= Postal::Config.smtp_server.default_port %p.pageContent__definitionText The SMTP service supports STARTTLS if you wish to send messages securely. Be aware that security cannot guaranteed all the way to their final destination. diff --git a/app/views/ip_addresses/_form.html.haml b/app/views/ip_addresses/_form.html.haml index ea8053f..e6f534e 100644 --- a/app/views/ip_addresses/_form.html.haml +++ b/app/views/ip_addresses/_form.html.haml @@ -17,16 +17,15 @@ %p.fieldSet__text This priority will determine the likelihood of this IP address being selected for use when sending a message. The higher the number the more likely the IP - is to be chosen. By defalt, the priority is set to the maximum value of 100. + is to be chosen. By default, the priority is set to the maximum value of 100. This can be used to warm up new IP addresses by adding them with a low priority. To give an indication of how this works, if you have three IPs with 1, 50 and 100 as their priorities, and you send 100,000 emails, the priority 1 address will receive - a tiny percentage, the priority 50 will receive roughly 25% and the priority 100 will - receive roughly 75%. + a tiny percentage, the priority 50 will receive roughly one third of e-mails and the + priority 100 will receive roughly two thirds. .fieldSetSubmit.buttonSet = f.submit :class => 'button button--positive js-form-submit' .fieldSetSubmit__delete - if @ip_address.persisted? = link_to "Delete IP address", [@ip_pool, @ip_address], :class => 'button button--danger', :method => :delete, :remote => true, :data => {:confirm => "Are you sure you wish to remove this IP from the pool?"} - diff --git a/app/views/messages/_deliveries.html.haml b/app/views/messages/_deliveries.html.haml index 707948b..dc9f4f9 100644 --- a/app/views/messages/_deliveries.html.haml +++ b/app/views/messages/_deliveries.html.haml @@ -11,7 +11,7 @@ %p This message has been held. By releasing the message, we will allow it to continue on its way to its destination. - if @message.hold_expiry - It will be held until #{@message.hold_expiry.to_s(:long)}. + It will be held until #{@message.hold_expiry.to_fs(:long)}. %p.buttonSet = link_to "Release message", retry_organization_server_message_path(organization, @server, message.id), :class => "button button--small", :remote => true, :method => :post = link_to "Cancel hold", cancel_hold_organization_server_message_path(organization, @server, message.id), :class => "button button--small button--danger", :remote => true, :method => :post @@ -33,7 +33,7 @@ %li.deliveryList__item .deliveryList__top .deliveryList__time - = delivery.timestamp.to_s(:long) + = delivery.timestamp.to_fs(:long) .deliveryList__status - if delivery.sent_with_ssl = image_tag 'icons/lock.svg', :class => 'deliveryList__secure' diff --git a/app/views/messages/_list.html.haml b/app/views/messages/_list.html.haml index 79e5fdb..4d1a4ad 100644 --- a/app/views/messages/_list.html.haml +++ b/app/views/messages/_list.html.haml @@ -17,7 +17,7 @@ %dd= message.mail_from || "none" .messageList__meta - %p.messageList__timestamp= message.timestamp.in_time_zone.to_s(:long) + %p.messageList__timestamp= message.timestamp.in_time_zone.to_fs(:long) %p.messageList__status - if message.read? %span.label.label--purple Opened diff --git a/app/views/messages/_message_header.html.haml b/app/views/messages/_message_header.html.haml index 41c537c..0ec98b4 100644 --- a/app/views/messages/_message_header.html.haml +++ b/app/views/messages/_message_header.html.haml @@ -23,7 +23,7 @@ = link_to @message.rcpt_to || "[blank]", send("#{@message.scope}_organization_server_messages_path", organization, @server, :query => "to: #{@message.rcpt_to}"), :class => 'u-link' %dl %dt Received - %dd= @message.timestamp.in_time_zone.to_s(:long) + %dd= @message.timestamp.in_time_zone.to_fs(:long) .navBar.navBar--tertiary %ul diff --git a/app/views/messages/activity.html.haml b/app/views/messages/activity.html.haml index fba7341..6e28cca 100644 --- a/app/views/messages/activity.html.haml +++ b/app/views/messages/activity.html.haml @@ -11,7 +11,7 @@ - for entry in @entries.reverse - if entry.is_a?(Postal::MessageDB::Delivery) %li.messageActivity__event - %p.messageActivity__timestamp= entry.timestamp.to_s(:long) + %p.messageActivity__timestamp= entry.timestamp.to_fs(:long) .messageActivity__details.messageActivity--detailsDelivery %p.messageActivity__subject =# entry.status.underscore.humanize @@ -21,20 +21,20 @@ - elsif entry.is_a?(Postal::MessageDB::Click) %li.messageActivity__event - %p.messageActivity__timestamp= entry.timestamp.to_s(:long) + %p.messageActivity__timestamp= entry.timestamp.to_fs(:long) .messageActivity__details.messageActivity--detailsClick %p.messageActivity__subject Click for #{entry.url} %p.messageActivity__extra Clicked from #{entry.ip_address} (#{entry.user_agent}) - elsif entry.is_a?(Postal::MessageDB::Load) %li.messageActivity__event - %p.messageActivity__timestamp= entry.timestamp.to_s(:long) + %p.messageActivity__timestamp= entry.timestamp.to_fs(:long) .messageActivity__details.messageActivity--detailsLoad %p.messageActivity__subject Message Viewed %p.messageActivity__extra Opened from #{entry.ip_address} (#{entry.user_agent}) %li.messageActivity__event - %p.messageActivity__timestamp= @message.timestamp.to_s(:long) + %p.messageActivity__timestamp= @message.timestamp.to_fs(:long) .messageActivity__details %p.messageActivity__subject Message received by Postal diff --git a/app/views/messages/suppressions.html.haml b/app/views/messages/suppressions.html.haml index 7fbfee8..725d96f 100644 --- a/app/views/messages/suppressions.html.haml +++ b/app/views/messages/suppressions.html.haml @@ -14,7 +14,7 @@ - else %p.pageContent__intro.u-margin When messages cannot be delivered, addresses are added to the suppression list which stops - future messages to the same recipient being sent through. Recipients are removed from the list after #{Postal.config.general.suppression_list_removal_delay} days. + future messages to the same recipient being sent through. Recipients are removed from the list after #{Postal::Config.postal.default_suppression_list_automatic_removal_days} days. %ul.suppressionList - for suppression in @suppressions[:records] %li.suppressionList__item @@ -22,9 +22,9 @@ %p.suppressionList__address= link_to suppression['address'], outgoing_organization_server_messages_path(organization, @server, :query => "to: #{suppression['address']}") %p.suppressionList__reason= suppression['reason'].capitalize .suppressionList__right - %p.suppressionList__timestamp Added #{Time.zone.at(suppression['timestamp']).to_s(:long)} + %p.suppressionList__timestamp Added #{Time.zone.at(suppression['timestamp']).to_fs(:long)} %p.suppressionList__timestamp - Expires #{Time.zone.at(suppression['keep_until']).to_s(:long)} + Expires #{Time.zone.at(suppression['keep_until']).to_fs(:long)} - if suppression['keep_until'] < Time.now.to_f %span.u-red expired = render 'shared/message_db_pagination', :data => @suppressions, :name => "suppression" diff --git a/app/views/servers/show.html.haml b/app/views/servers/show.html.haml index 3f6558c..3ae1b42 100644 --- a/app/views/servers/show.html.haml +++ b/app/views/servers/show.html.haml @@ -35,7 +35,7 @@ %li #{@first_date.strftime("%A at %l%P")} → %li Today at #{Time.now.strftime("%l%P")} - else - %li #{@first_date.to_date.to_s(:long)} → + %li #{@first_date.to_date.to_fs(:long)} → %li Today .titleWithLinks.u-margin @@ -44,4 +44,3 @@ %li= link_to "View message queue", [:queue, organization, @server], :class => 'titleWithLinks__link' %li= link_to "View full e-mail history", [:outgoing, organization, @server, :messages], :class => 'titleWithLinks__link' = render 'messages/list', :messages => @messages - diff --git a/app/views/track_domains/_form.html.haml b/app/views/track_domains/_form.html.haml index e6a6355..0fb3cff 100644 --- a/app/views/track_domains/_form.html.haml +++ b/app/views/track_domains/_form.html.haml @@ -10,7 +10,7 @@ = f.select :domain_id, domain_options_for_select(@server, @track_domain.domain), {}, :class => 'input input--select routeNameInput__domain', :disabled => @track_domain.persisted? %p.fieldSet__text This is the domain that requests for tracked links will be directed through when you use click tracking. We recommend using something like - click.yourdomain.com. You will need to a CNAME record to point to #{Postal.config.dns.track_domain} for this once you've added it. + click.yourdomain.com. You will need to a CNAME record to point to #{Postal::Config.dns.track_domain} for this once you've added it. .fieldSet__field = f.label :ssl_enabled, :class => 'fieldSet__label' @@ -47,4 +47,3 @@ .fieldSetSubmit.buttonSet = f.submit @track_domain.new_record? ? "Create Track Domain" : "Save Track Domain", :class => 'button button--positive js-form-submit' - diff --git a/bin/postal b/bin/postal index b113e2d..34a6d22 100755 --- a/bin/postal +++ b/bin/postal @@ -16,21 +16,13 @@ case "$1" in ;; smtp-server) - run "bundle exec rake postal:smtp_server" + run "bundle exec ruby script/smtp_server.rb" ;; worker) run "bundle exec ruby script/worker.rb" ;; - cron) - run "bundle exec rake postal:cron" - ;; - - requeuer) - run "bundle exec rake postal:requeuer" - ;; - initialize) echo 'Initializing database' run "bundle exec rake db:create db:schema:load db:seed" @@ -69,8 +61,6 @@ case "$1" in echo -e " * \033[35mweb-server\033[0m - run the web server" echo -e " * \033[35msmtp-server\033[0m - run the SMTP server" echo -e " * \033[35mworker\033[0m - run a worker" - echo -e " * \033[35mcron\033[0m - run the cron process" - echo -e " * \033[35mrequeuer\033[0m - run the message requeuer" echo echo "Setup/upgrade tools:" echo diff --git a/config.ru b/config.ru index 9f3cd86..6fd1995 100644 --- a/config.ru +++ b/config.ru @@ -3,5 +3,4 @@ # This file is used by Rack-based servers to start the application. require_relative "config/environment" -$0 = "[postal] #{ENV.fetch('PROC_NAME', nil)}" run Rails.application diff --git a/config/application.rb b/config/application.rb index d0fb313..a52bee3 100644 --- a/config/application.rb +++ b/config/application.rb @@ -17,7 +17,7 @@ Bundler.require(*Rails.groups) module Postal class Application < Rails::Application - config.load_defaults 6.0 + config.load_defaults 7.0 # Disable most generators config.generators do |g| @@ -35,12 +35,14 @@ module Postal config.action_view.field_error_proc = proc { |t, _| t } # Load the tracking server middleware - require "postal/tracking_middleware" - config.middleware.insert_before ActionDispatch::HostAuthorization, Postal::TrackingMiddleware + require "tracking_middleware" + config.middleware.insert_before ActionDispatch::HostAuthorization, TrackingMiddleware - config.logger = Postal.logger_for(:rails) + config.hosts << Postal::Config.postal.web_hostname - config.hosts << Postal.config.web.host + unless Postal::Config.logging.rails_log_enabled? + config.logger = Logger.new("/dev/null") + end end end diff --git a/config/boot.rb b/config/boot.rb index 73db71c..d934bb4 100644 --- a/config/boot.rb +++ b/config/boot.rb @@ -2,12 +2,8 @@ ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__) -$stdout.sync = true -$stderr.sync = true - require "bundler/setup" # Set up gems listed in the Gemfile. require_relative "../lib/postal/config" -Postal.check_config! -ENV["RAILS_ENV"] = Postal.config.rails&.environment || "development" +ENV["RAILS_ENV"] = Postal::Config.rails.environment || "development" diff --git a/config/cron.rb b/config/cron.rb deleted file mode 100644 index ab18723..0000000 --- a/config/cron.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Clockwork - - configure do |config| - config[:tz] = "UTC" - config[:logger] = Postal.logger_for(:cron) - end - - every 1.minute, "every-1-minutes" do - RequeueWebhooksJob.queue(:main) - SendNotificationsJob.queue(:main) - end - - every 1.hour, "every-hour", at: ["**:15"] do - CheckAllDNSJob.queue(:main) - ExpireHeldMessagesJob.queue(:main) - CleanupAuthieSessionsJob.queue(:main) - end - - every 1.hour, "every-hour", at: ["**:45"] do - PruneWebhookRequestsJob.queue(:main) - end - - every 1.day, "every-day", at: ["03:00"] do - ProcessMessageRetentionJob.queue(:main) - PruneSuppressionListsJob.queue(:main) - end - -end diff --git a/config/database.yml b/config/database.yml index 9c7d94b..fa90aff 100644 --- a/config/database.yml +++ b/config/database.yml @@ -1,13 +1,13 @@ default: &default adapter: mysql2 reconnect: true - encoding: <%= Postal.config.main_db.encoding %> - pool: <%= Postal.config.main_db.pool_size %> - username: <%= Postal.config.main_db.username %> - password: <%= Postal.config.main_db.password %> - host: <%= Postal.config.main_db.host %> - port: <%= Postal.config.main_db.port %> - database: <%= Postal.config.main_db.database %> + encoding: "<%= Postal::Config.main_db.encoding %>" + pool: <%= Postal::Config.main_db.pool_size %> + username: "<%= Postal::Config.main_db.username %>" + password: "<%= Postal::Config.main_db.password %>" + host: "<%= Postal::Config.main_db.host %>" + port: <%= Postal::Config.main_db.port %> + database: "<%= Postal::Config.main_db.database %>" development: <<: *default diff --git a/config/environments/production.rb b/config/environments/production.rb index cca5f81..386e938 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -72,12 +72,6 @@ Rails.application.configure do # require 'syslog/logger' # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name') - if ENV["RAILS_LOG_TO_STDOUT"].present? - logger = ActiveSupport::Logger.new($stdout) - logger.formatter = config.log_formatter - config.logger = ActiveSupport::TaggedLogging.new(logger) - end - # Do not dump schema after migrations. config.active_record.dump_schema_after_migration = false end diff --git a/config/examples/development.yml b/config/examples/development.yml new file mode 100644 index 0000000..09d5dba --- /dev/null +++ b/config/examples/development.yml @@ -0,0 +1,30 @@ +# This is an example Postal configuration file for use in +# development environments. For a production example, see +# the https://github.com/postalserver/install repository. + +version: 2 + +postal: + web_hostname: postal.example.com + web_protocol: https + smtp_hostname: postal.example.com + +main_db: + host: 127.0.0.1 + username: root + password: + database: postal + +message_db: + host: 127.0.0.1 + username: root + password: + prefix: postal + +logging: + rails_log_enabled: true + highlighting_enabled: true + +rails: + environment: development + secret_key: 7f27856d26e864bafd49d0df37ad3d1339086e86ef0447e0f1814dde5277452fea97dab9e3aad6dfa11bfe359c82ce302d97bf1e58f6103c4408e4fbad4eeccf diff --git a/config/examples/test.yml b/config/examples/test.yml new file mode 100644 index 0000000..e47f3ab --- /dev/null +++ b/config/examples/test.yml @@ -0,0 +1,25 @@ +# This is an example Postal configuration file for use in +# test environments. For a production example, see +# the https://github.com/postalserver/install repository. + +version: 2 + +main_db: + host: 127.0.0.1 + username: root + password: + database: postal-test + +message_db: + host: 127.0.0.1 + username: root + password: + prefix: postal-test + +logging: + enabled: false + rails_log_enabled: false + +rails: + environment: test + secret_key: 7f27856d26e864bafd49d0df37ad3d1339086e86ef0447e0f1814dde5277452fea97dab9e3aad6dfa11bfe359c82ce302d97bf1e58f6103c4408e4fbad4eeccf diff --git a/config/initializers/_wait_for_migrations.rb b/config/initializers/_wait_for_migrations.rb new file mode 100644 index 0000000..12640f1 --- /dev/null +++ b/config/initializers/_wait_for_migrations.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +require "migration_waiter" + +MigrationWaiter.wait_if_appropriate diff --git a/config/initializers/content_security_policy.rb b/config/initializers/content_security_policy.rb new file mode 100644 index 0000000..691cfa1 --- /dev/null +++ b/config/initializers/content_security_policy.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true +# Be sure to restart your server when you modify this file. + +# Define an application-wide content security policy. +# See the Securing Rails Applications Guide for more information: +# https://guides.rubyonrails.org/security.html#content-security-policy-header + +# Rails.application.configure do +# config.content_security_policy do |policy| +# policy.default_src :self, :https +# policy.font_src :self, :https, :data +# policy.img_src :self, :https, :data +# policy.object_src :none +# policy.script_src :self, :https +# policy.style_src :self, :https +# # Specify URI for violation reports +# # policy.report_uri "/csp-violation-report-endpoint" +# end +# +# # Generate session nonces for permitted importmap and inline scripts +# config.content_security_policy_nonce_generator = ->(request) { request.session.id.to_s } +# config.content_security_policy_nonce_directives = %w(script-src) +# +# # Report violations without enforcing the policy. +# # config.content_security_policy_report_only = true +# end diff --git a/config/initializers/filter_parameter_logging.rb b/config/initializers/filter_parameter_logging.rb index 7a4f47b..ca55f95 100644 --- a/config/initializers/filter_parameter_logging.rb +++ b/config/initializers/filter_parameter_logging.rb @@ -2,5 +2,9 @@ # Be sure to restart your server when you modify this file. -# Configure sensitive parameters which will be filtered from the log file. -Rails.application.config.filter_parameters += [:password] +# Configure parameters to be filtered from the log file. Use this to limit dissemination of +# sensitive information. See the ActiveSupport::ParameterFilter documentation for supported +# notations and behaviors. +Rails.application.config.filter_parameters += [ + :passw, :secret, :token, :_key, :crypt, :salt, :certificate, :otp, :ssn +] diff --git a/config/initializers/inflections.rb b/config/initializers/inflections.rb index da17573..3a68b27 100644 --- a/config/initializers/inflections.rb +++ b/config/initializers/inflections.rb @@ -21,6 +21,7 @@ ActiveSupport::Inflector.inflections(:en) do |inflect| inflect.acronym "API" inflect.acronym "DNS" + inflect.acronym "SSL" inflect.acronym "MySQL" inflect.acronym "DB" diff --git a/config/initializers/logging.rb b/config/initializers/logging.rb new file mode 100644 index 0000000..22e16ab --- /dev/null +++ b/config/initializers/logging.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +begin + def add_exception_to_payload(payload, event) + return unless exception = event.payload[:exception_object] + + payload[:exception_class] = exception.class.name + payload[:exception_message] = exception.message + payload[:exception_backtrace] = exception.backtrace[0, 4].join("\n") + end + + ActiveSupport::Notifications.subscribe "process_action.action_controller" do |*args| + event = ActiveSupport::Notifications::Event.new(*args) + + payload = { + event: "request", + transaction: event.transaction_id, + controller: event.payload[:controller], + action: event.payload[:action], + format: event.payload[:format], + method: event.payload[:method], + path: event.payload[:path], + request_id: event.payload[:request].request_id, + ip_address: event.payload[:request].ip, + status: event.payload[:status], + view_runtime: event.payload[:view_runtime], + db_runtime: event.payload[:db_runtime] + } + + add_exception_to_payload(payload, event) + + string = "#{payload[:method]} #{payload[:path]} (#{payload[:status]})" + + if payload[:exception_class] + Postal.logger.error(string, **payload) + else + Postal.logger.info(string, **payload) + end + end + + ActiveSupport::Notifications.subscribe "deliver.action_mailer" do |*args| + event = ActiveSupport::Notifications::Event.new(*args) + + Postal.logger.info({ + event: "send_email", + transaction: event.transaction_id, + message_id: event.payload[:message_id], + subject: event.payload[:subject], + from: event.payload[:from], + to: event.payload[:to].is_a?(Array) ? event.payload[:to].join(", ") : event.payload[:to].to_s + }) + end +end diff --git a/config/initializers/new_framework_defaults.rb b/config/initializers/new_framework_defaults.rb deleted file mode 100644 index 420535c..0000000 --- a/config/initializers/new_framework_defaults.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -# Be sure to restart your server when you modify this file. -# -# This file contains migration options to ease your Rails 5.0 upgrade. -# -# Read the Rails 5.0 release notes for more info on each option. - -# Enable per-form CSRF tokens. Previous versions had false. -Rails.application.config.action_controller.per_form_csrf_tokens = true - -# Enable origin-checking CSRF mitigation. Previous versions had false. -Rails.application.config.action_controller.forgery_protection_origin_check = true - -# Make Ruby 2.4 preserve the timezone of the receiver when calling `to_time`. -# Previous versions had false. -ActiveSupport.to_time_preserves_timezone = true - -# Require `belongs_to` associations by default. Previous versions had false. -Rails.application.config.active_record.belongs_to_required_by_default = true - -# Configure SSL options to enable HSTS with subdomains. Previous versions had false. -Rails.application.config.ssl_options = false diff --git a/config/initializers/new_framework_defaults_7_0.rb b/config/initializers/new_framework_defaults_7_0.rb new file mode 100644 index 0000000..a13554e --- /dev/null +++ b/config/initializers/new_framework_defaults_7_0.rb @@ -0,0 +1,142 @@ +# frozen_string_literal: true +# Be sure to restart your server when you modify this file. +# +# This file eases your Rails 7.0 framework defaults upgrade. +# +# Uncomment each configuration one by one to switch to the new default. +# Once your application is ready to run with all new defaults, you can remove +# this file and set the `config.load_defaults` to `7.0`. +# +# Read the Guide for Upgrading Ruby on Rails for more info on each option. +# https://guides.rubyonrails.org/upgrading_ruby_on_rails.html + +# `button_to` view helper will render `