Change SQL charset for message_db to utf8mb4

Squashed commit of the following:

commit c434c868d111fd091f638216e2dec2685c7ac296
Author: Charlie Smurthwaite <charlie@krystal.uk>
Date:   Thu Oct 1 10:37:17 2020 +0100

    change utf8mb4_bin to utf8mb4_unicode_ci

commit 68031c470b41edd9906e3b10ccbb6d2c71a2c9a7
Merge: 9f277d4 9b98339
Author: Charlie Smurthwaite <charlie@krystal.uk>
Date:   Thu Oct 1 10:29:02 2020 +0100

    Merge remote-tracking branch 'origin/master' into mb4

commit 9f277d4ce2
Author: Petter Helset <petter@helset.eu>
Date:   Thu Oct 26 15:15:16 2017 +0200

    Migrate message database with main database

commit 3be337e265
Author: Petter Helset <petter@helset.eu>
Date:   Thu Oct 26 14:45:26 2017 +0200

    Migrations for existing messagedbs to utf8mb4

commit 3d8d3956fe
Author: Petter Helset <petter@helset.eu>
Date:   Thu Oct 26 13:49:57 2017 +0200

    Change SQL charset for message_db to utf8mb4

lib/postal/message_db/migrations/19_convert_database_to_utf8mb4.rb

@@ -0,0 +1,26 @@
+module Postal
+  module MessageDB
+    module Migrations
+      class ConvertDatabaseToUtf8mb4 < Postal::MessageDB::Migration
+        def up
+          @database.query("ALTER DATABASE `#{@database.database_name}` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
+          @database.query("ALTER TABLE `#{@database.database_name}`.`clicks` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
+          @database.query("ALTER TABLE `#{@database.database_name}`.`deliveries` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
+          @database.query("ALTER TABLE `#{@database.database_name}`.`links` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
+          @database.query("ALTER TABLE `#{@database.database_name}`.`live_stats` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
+          @database.query("ALTER TABLE `#{@database.database_name}`.`loads` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
+          @database.query("ALTER TABLE `#{@database.database_name}`.`messages` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
+          @database.query("ALTER TABLE `#{@database.database_name}`.`migrations` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
+          @database.query("ALTER TABLE `#{@database.database_name}`.`raw_message_sizes` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
+          @database.query("ALTER TABLE `#{@database.database_name}`.`spam_checks` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
+          @database.query("ALTER TABLE `#{@database.database_name}`.`stats_daily` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
+          @database.query("ALTER TABLE `#{@database.database_name}`.`stats_hourly` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
+          @database.query("ALTER TABLE `#{@database.database_name}`.`stats_monthly` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
+          @database.query("ALTER TABLE `#{@database.database_name}`.`stats_yearly` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
+          @database.query("ALTER TABLE `#{@database.database_name}`.`suppressions` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
+          @database.query("ALTER TABLE `#{@database.database_name}`.`webhook_requests` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci")
+        end
+      end
+    end
+  end
+end

lib/postal/message_db/mysql.rb

@@ -7,7 +7,7 @@ module Postal
       # it would be undesirable as we'd just end up with lots of connections.
 
       def self.new_client
-        Mysql2::Client.new(:host => Postal.config.message_db.host, :username => Postal.config.message_db.username, :password => Postal.config.message_db.password, :port => Postal.config.message_db.port, :reconnect => true)
+        Mysql2::Client.new(:host => Postal.config.message_db.host, :username => Postal.config.message_db.username, :password => Postal.config.message_db.password, :port => Postal.config.message_db.port, :reconnect => true, :encoding => Postal.config.message_db.encoding || 'utf8mb4')
       end
 
       @free_clients = []

lib/postal/message_db/provisioner.rb

@@ -33,7 +33,7 @@ module Postal
       # Creates a new empty database
       #
       def create
-        @database.query("CREATE DATABASE `#{@database.database_name}` CHARSET utf8 COLLATE UTF8_UNICODE_CI;")
+        @database.query("CREATE DATABASE `#{@database.database_name}` CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci;")
         true
       rescue Mysql2::Error => e
         e.message =~ /database exists/ ? false : raise
@@ -185,7 +185,7 @@ module Postal
             s << ", PRIMARY KEY (`id`)"
           end
 
-          s << ") ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;"
+          s << ") ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4;"
         end
       end
 

lib/tasks/postal.rake

@@ -34,3 +34,7 @@ namespace :postal do
   end
 
 end
+
+Rake::Task['db:migrate'].enhance do
+  Rake::Task['postal:migrate_message_databases'].invoke
+end