feat: more consistent logging

lib/postal/app_logger.rb

@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-require "logger"
-
-module Postal
-
-  class AppLogger < Logger
-
-    def initialize(log_name, *args)
-      @log_name = log_name
-      super(*args)
-      self.formatter = LogFormatter.new
-    end
-
-    def add(severity, message = nil, progname = nil)
-      super
-      if severity >= @level && n = self.class.graylog_notifier
-        begin
-          if message.nil?
-            message = block_given? ? yield : progname
-          end
-          message = message.to_s.force_encoding("UTF-8").scrub
-          message_without_ansi = begin
-            message.gsub(/\e\[([\d;]+)?m/, "")
-          rescue StandardError
-            message
-          end
-          n.notify!(short_message: message_without_ansi, log_name: @log_name, facility: "postal", application_name: "postal", process_name: ENV.fetch("PROC_NAME", nil), pid: Process.pid)
-        rescue StandardError
-          # Can't log this to GELF. Soz.
-        end
-      end
-      true
-    end
-
-    def self.graylog?
-      !!Postal.config.logging.graylog&.host
-    end
-
-    def self.graylog_notifier
-      @graylog_notifier ||= graylog? ? GELF::Notifier.new(Postal.config.logging.graylog.host, Postal.config.logging.graylog.port) : nil
-    end
-
-  end
-
-  class LogFormatter
-
-    TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%3N"
-    COLORS = [32, 34, 35, 31, 32, 33].freeze
-
-    def call(severity, datetime, progname, msg)
-      time = datetime.strftime(TIME_FORMAT)
-      if number = ENV["PROC_NAME"]
-        id = number.split(".").last.to_i
-        proc_text = "\e[#{COLORS[id % COLORS.size]}m[#{ENV['PROC_NAME']}:#{Process.pid}]\e[0m"
-      else
-        proc_text = "[#{Process.pid}]"
-      end
-      "#{proc_text} [#{time}] #{severity} -- : #{msg}\n"
-    end
-
-  end
-
-end

lib/postal/config.rb

@@ -49,14 +49,6 @@ module Postal
     end
   end
 
-  def self.log_root
-    if config.logging.root
-      @log_root ||= Pathname.new(config.logging.root)
-    else
-      @log_root ||= app_root.join("log")
-    end
-  end
-
   def self.config_file_path
     if env == "default"
       @config_file_path ||= File.join(config_root, "postal.yml")
@@ -93,16 +85,11 @@ module Postal
     end
   end
 
-  def self.logger_for(name)
-    @loggers ||= {}
-    @loggers[name.to_sym] ||= begin
-      require "postal/app_logger"
-      if config.logging.stdout || ENV["LOG_TO_STDOUT"]
-        Postal::AppLogger.new(name, $stdout)
-      else
-        FileUtils.mkdir_p(log_root)
-        Postal::AppLogger.new(name, log_root.join("#{name}.log"), config.logging.max_log_files, config.logging.max_log_file_size.megabytes)
-      end
+  def self.logger
+    @logger ||= begin
+      k = Klogger.new(nil, destination: Rails.env.test? ? "/dev/null" : $stdout, highlight: Rails.env.development?)
+      k.add_destination(graylog_logging_destination) if config.logging&.graylog&.host.present?
+      k
     end
   end
 
@@ -187,4 +174,19 @@ module Postal
     config.general.use_ip_pools?
   end
 
+  def self.graylog_logging_destination
+    @graylog_destination ||= begin
+      notifier = GELF::Notifier.new(config.logging.graylog.host, config.logging.graylog.port, "WAN")
+      proc do |_logger, payload, group_ids|
+        short_message = payload.delete(:message) || "[message missing]"
+        notifier.notify!(short_message: short_message, **{
+          facility: config.logging.graylog.facility,
+          _environment: Rails.env.to_s,
+          _version: Postal::VERSION.to_s,
+          _group_ids: group_ids.join(" ")
+        }.merge(payload.transform_keys { |k| "_#{k}".to_sym }.transform_values(&:to_s)))
+      end
+    end
+  end
+
 end

lib/postal/http_sender.rb

@@ -61,7 +61,7 @@ module Postal
     private
 
     def log(text)
-      Postal.logger_for(:http_sender).info("[#{@log_id}] #{text}")
+      Postal.logger.info text, id: @log_id, component: "http-sender"
     end
 
     def parameters(message, options = {})

lib/postal/job.rb

@@ -30,7 +30,7 @@ module Postal
     end
 
     def log(text)
-      Worker.logger.info "[#{@id}] #{text}"
+      Worker.logger.info(text)
     end
 
     def self.queue(queue, params = {})

lib/postal/message_db/database.rb

@@ -325,12 +325,12 @@ module Postal
         result = connection.query(query, cast_booleans: true)
         time = Time.now.to_f - start_time
         logger.debug "  \e[4;34mMessageDB Query (#{time.round(2)}s) \e[0m  \e[33m#{query}\e[0m"
-        if time > 0.5 && query =~ /\A(SELECT|UPDATE|DELETE) /
+        if time.positive? && query =~ /\A(SELECT|UPDATE|DELETE) /
           id = Nifty::Utils::RandomString.generate(length: 6).upcase
           explain_result = ResultForExplainPrinter.new(connection.query("EXPLAIN #{query}"))
-          slow_query_logger.info "[#{id}] EXPLAIN #{query}"
+          logger.info "  [#{id}] EXPLAIN #{query}"
           ActiveRecord::ConnectionAdapters::MySQL::ExplainPrettyPrinter.new.pp(explain_result, time).split("\n").each do |line|
-            slow_query_logger.info "[#{id}] " + line
+            logger.info "  [#{id}] " + line
           end
         end
         result
@@ -340,10 +340,6 @@ module Postal
         defined?(Rails) ? Rails.logger : Logger.new($stdout)
       end
 
-      def slow_query_logger
-        Postal.logger_for(:slow_message_db_queries)
-      end
-
       def with_mysql(&block)
         self.class.connection_pool.use(&block)
       end

lib/postal/message_inspector.rb

@@ -15,7 +15,7 @@ module Postal
     private
 
     def logger
-      Postal.logger_for(:message_inspection)
+      Postal.logger
     end
 
     class << self

lib/postal/message_requeuer.rb

@@ -20,7 +20,7 @@ module Postal
     private
 
     def log(text)
-      Postal.logger_for(:message_requeuer).info text
+      Postal.logger.info text, component: "message-requeuer"
     end
 
     def check_exit

lib/postal/smtp_sender.rb

@@ -234,7 +234,7 @@ module Postal
     end
 
     def log(text)
-      Postal.logger_for(:smtp_sender).info "[#{@log_id}] #{text}"
+      Postal.logger.info text, id: @log_id, component: "smtp-sender"
     end
 
     def destination_host_description

lib/postal/smtp_server/client.rb

@@ -48,16 +48,18 @@ module Postal
       end
 
       def handle(data)
-        if @state == :preauth
-          return proxy(data)
-        end
+        Postal.logger.tagged(id: id) do
+          if @state == :preauth
+            return proxy(data)
+          end
 
-        log "\e[32m<= #{sanitize_input_for_log(data.strip)}\e[0m"
-        if @proc
-          @proc.call(data)
+          log "\e[32m<= #{sanitize_input_for_log(data.strip)}\e[0m"
+          if @proc
+            @proc.call(data)
 
-        else
-          handle_command(data)
+          else
+            handle_command(data)
+          end
         end
       end
 
@@ -93,7 +95,7 @@ module Postal
       def log(text)
         return false unless @logging_enabled
 
-        Postal.logger_for(:smtp_server).debug "[#{id}] #{text}"
+        Postal.logger.debug(text, id: id)
       end
 
       private

lib/postal/smtp_server/server.rb

@@ -14,8 +14,10 @@ module Postal
       end
 
       def run
-        listen
-        run_event_loop
+        logger.tagged(component: "smtp-server") do
+          listen
+          run_event_loop
+        end
       end
 
       private
@@ -264,7 +266,7 @@ module Postal
       end
 
       def logger
-        Postal.logger_for(:smtp_server)
+        Postal.logger
       end
 
     end

lib/postal/worker.rb

@@ -49,33 +49,25 @@ module Postal
 
     private
 
-    def receive_job(delivery_info, properties, body)
-      message = begin
-        JSON.parse(body)
-      rescue StandardError
-        nil
-      end
+    def receive_job(delivery_info, properties, message)
       if message && message["class_name"]
         @running_jobs << message["id"]
         set_process_name
         start_time = Time.now
         Thread.current[:job_id] = message["id"]
-        logger.info "[#{message['id']}] Started processing \e[34m#{message['class_name']}\e[0m job"
+        logger.info "Processing job"
         begin
           klass = message["class_name"].constantize.new(message["id"], message["params"])
           klass.perform
           GC.start
         rescue StandardError => e
           klass.on_error(e) if defined?(klass)
-          logger.warn "[#{message['id']}] \e[31m#{e.class}: #{e.message}\e[0m"
-          e.backtrace.each do |line|
-            logger.warn "[#{message['id']}]    " + line
-          end
+          logger.exception(e)
           if defined?(Sentry)
             Sentry.capture_exception(e, extra: { job_id: message["id"] })
           end
         ensure
-          logger.info "[#{message['id']}] Finished processing \e[34m#{message['class_name']}\e[0m job in #{Time.now - start_time}s"
+          logger.info "Finished job", time: (Time.now - start_time).to_i
         end
       end
     ensure
@@ -92,13 +84,21 @@ module Postal
 
     def join_queue(queue)
       if @active_queues[queue]
-        logger.info "Attempted to join queue #{queue} but already joined."
+        logger.error "attempted to join queue but already joined", queue: queue
       else
         consumer = self.class.job_queue(queue).subscribe(manual_ack: true) do |delivery_info, properties, body|
-          receive_job(delivery_info, properties, body)
+          message = begin
+            JSON.parse(body)
+          rescue StandardError
+            nil
+          end
+
+          logger.tagged(job_id: message["id"], queue: queue, job_class: message["class_name"]) do
+            receive_job(delivery_info, properties, message)
+          end
         end
         @active_queues[queue] = consumer
-        logger.info "Joined \e[32m#{queue}\e[0m queue"
+        logger.info "joined queue", queue: queue
       end
     end
 
@@ -106,9 +106,9 @@ module Postal
       if consumer = @active_queues[queue]
         consumer.cancel
         @active_queues.delete(queue)
-        logger.info "Left \e[32m#{queue}\e[0m queue"
+        logger.info "left queue", queue: queue
       else
-        logger.info "Not joined #{queue} so cannot leave"
+        logger.error "requested to leave queue, but not joined", queue: queue
       end
     end
 
@@ -198,7 +198,7 @@ module Postal
     class << self
 
       def logger
-        Postal.logger_for(:worker)
+        Postal.logger
       end
 
       def job_channel