Database migration Sqlite->Postgres (#2156)

Add a migration tool and a step-by-step guide for users who want to switch from Sqlite to Postgres.

docs/PostgreSQL.md

@@ -111,4 +111,34 @@ Eclair stores the latest database settings in the `${data-dir}/last_jdbcurl` fil
 
 The node operator can force Eclair to accept new database 
 connection settings by removing the `last_jdbcurl` file. 
-
+
+### Migrating from Sqlite to Postgres
+
+Eclair supports migrating your existing node from Sqlite to Postgres. Note that the opposite (from Postgres to Sqlite) is not supported.
+
+:warning: Once you have migrated from Sqlite to Postgres there is no going back!
+
+To migrate from Sqlite to Postgres, follow these steps:
+1. Stop Eclair
+2. Edit `eclair.conf`
+   1. Set `eclair.db.postgres.*` as explained in the section [Connection Settings](#connection-settings).
+   2. Set `eclair.db.driver=dual-sqlite-primary`. This will make Eclair use both databases backends. All calls to sqlite will be replicated in postgres.
+   3. Set `eclair.db.dual.migrate-on-restart=true`. This will make Eclair migrate the data from Sqlite to Postgres at startup.
+   4. Set `eclair.db.dual.compare-on-restart=true`. This will make Eclair compare Sqlite and Postgres at startup. The result of the comparison is displayed in the logs.
+3. Delete the file `~/.eclair/last_jdbcurl`. The purpose of this file is to prevent accidental change in the database backend.
+4. Start Eclair. You should see in the logs:
+   1. `migrating all tables...`
+   2. `migration complete`
+   3. `comparing all tables...`
+   4. `comparison complete identical=true` (NB: if `identical=false`, contact support)
+5. Eclair should then finish startup and operate normally. Data has been migrated to Postgres, and Sqlite/Postgres will be maintained in sync going forward.
+6. Edit `eclair.conf` and set `eclair.db.dual.migrate-on-restart=false` but do not restart Eclair yet.
+7. We recommend that you leave Eclair in dual db mode for a while, to make sure that you don't have issues with your new Postgres database. This a good time to set up [Backups and replication](#backups-and-replication).
+8. After some time has passed, restart Eclair. You should see in the logs:
+   1. `comparing all tables...`
+   2. `comparison complete identical=true` (NB: if `identical=false`, contact support)
+9. At this point we have confidence that the Postgres backend works normally, and we are ready to drop Sqlite for good.
+10. Edit `eclair.conf`
+    1. Set `eclair.db.driver=postgres`
+    2. Set `eclair.db.dual.compare-on-restart=false`
+11. Restart Eclair. From this moment, you cannot go back to Sqlite! If you try to do so, Eclair will refuse to start.

eclair-core/pom.xml

@@ -204,6 +204,12 @@
             <artifactId>scodec-core_${scala.version.short}</artifactId>
             <version>1.11.8</version>
         </dependency>
+        <dependency>
+            <!-- needed to fix "No implicit Ordering defined for scodec.bits.ByteVector" -->
+            <groupId>org.scodec</groupId>
+            <artifactId>scodec-bits_${scala.version.short}</artifactId>
+            <version>1.1.25</version>
+        </dependency>
         <dependency>
             <groupId>commons-codec</groupId>
             <artifactId>commons-codec</artifactId>

eclair-core/src/main/resources/reference.conf

@@ -311,7 +311,7 @@ eclair {
   }
 
   db {
-    driver = "sqlite" // sqlite, postgres
+    driver = "sqlite" // sqlite, postgres, dual-sqlite-primary, dual-postgres-primary
     postgres {
       database = "eclair"
       host = "localhost"
@@ -353,6 +353,10 @@ eclair {
         }
       }
     }
+    dual {
+      migrate-on-restart = false // migrate sqlite -> postgres on restart (only applies if sqlite is primary)
+      compare-on-restart = false // compare sqlite and postgres dbs on restart (only applies if sqlite is primary)
+    }
   }
 
   file-backup {

eclair-core/src/main/scala/fr/acinq/eclair/db/Databases.scala

@@ -21,6 +21,7 @@ import akka.actor.{ActorSystem, CoordinatedShutdown}
 import com.typesafe.config.Config
 import com.zaxxer.hikari.{HikariConfig, HikariDataSource}
 import fr.acinq.eclair.TimestampMilli
+import fr.acinq.eclair.db.migration.{CompareDb, MigrateDb}
 import fr.acinq.eclair.db.pg.PgUtils.PgLock.LockFailureHandler
 import fr.acinq.eclair.db.pg.PgUtils._
 import fr.acinq.eclair.db.pg._
@@ -267,13 +268,25 @@ object Databases extends Logging {
     db match {
       case Some(d) => d
       case None =>
+        val jdbcUrlFile = new File(chaindir, "last_jdbcurl")
         dbConfig.getString("driver") match {
-          case "sqlite" => Databases.sqlite(chaindir)
-          case "postgres" => Databases.postgres(dbConfig, instanceId, chaindir)
-          case "dual" =>
-            val sqlite = Databases.sqlite(chaindir)
-            val postgres = Databases.postgres(dbConfig, instanceId, chaindir)
-            DualDatabases(sqlite, postgres)
+          case "sqlite" => Databases.sqlite(chaindir, jdbcUrlFile_opt = Some(jdbcUrlFile))
+          case "postgres" => Databases.postgres(dbConfig, instanceId, chaindir, jdbcUrlFile_opt = Some(jdbcUrlFile))
+          case dual@("dual-sqlite-primary" | "dual-postgres-primary") =>
+            logger.info(s"using $dual database mode")
+            val sqlite = Databases.sqlite(chaindir, jdbcUrlFile_opt = None)
+            val postgres = Databases.postgres(dbConfig, instanceId, chaindir, jdbcUrlFile_opt = None)
+            val (primary, secondary) = if (dual == "dual-sqlite-primary") (sqlite, postgres) else (postgres, sqlite)
+            val dualDb = DualDatabases(primary, secondary)
+            if (primary == sqlite) {
+              if (dbConfig.getBoolean("dual.migrate-on-restart")) {
+                MigrateDb.migrateAll(dualDb)
+              }
+              if (dbConfig.getBoolean("dual.compare-on-restart")) {
+                CompareDb.compareAll(dualDb)
+              }
+            }
+            dualDb
           case driver => throw new RuntimeException(s"unknown database driver `$driver`")
         }
     }
@@ -282,18 +295,17 @@ object Databases extends Logging {
   /**
    * Given a parent folder it creates or loads all the databases from a JDBC connection
    */
-  def sqlite(dbdir: File): SqliteDatabases = {
+  def sqlite(dbdir: File, jdbcUrlFile_opt: Option[File]): SqliteDatabases = {
     dbdir.mkdirs()
-    val jdbcUrlFile = new File(dbdir, "last_jdbcurl")
     SqliteDatabases(
       eclairJdbc = SqliteUtils.openSqliteFile(dbdir, "eclair.sqlite", exclusiveLock = true, journalMode = "wal", syncFlag = "full"), // there should only be one process writing to this file
       networkJdbc = SqliteUtils.openSqliteFile(dbdir, "network.sqlite", exclusiveLock = false, journalMode = "wal", syncFlag = "normal"), // we don't need strong durability guarantees on the network db
       auditJdbc = SqliteUtils.openSqliteFile(dbdir, "audit.sqlite", exclusiveLock = false, journalMode = "wal", syncFlag = "full"),
-      jdbcUrlFile_opt = Some(jdbcUrlFile)
+      jdbcUrlFile_opt = jdbcUrlFile_opt
     )
   }
 
-  def postgres(dbConfig: Config, instanceId: UUID, dbdir: File, lockExceptionHandler: LockFailureHandler = LockFailureHandler.logAndStop)(implicit system: ActorSystem): PostgresDatabases = {
+  def postgres(dbConfig: Config, instanceId: UUID, dbdir: File, jdbcUrlFile_opt: Option[File], lockExceptionHandler: LockFailureHandler = LockFailureHandler.logAndStop)(implicit system: ActorSystem): PostgresDatabases = {
     dbdir.mkdirs()
     val database = dbConfig.getString("postgres.database")
     val host = dbConfig.getString("postgres.host")
@@ -328,8 +340,6 @@ object Databases extends Logging {
       case unknownLock => throw new RuntimeException(s"unknown postgres lock type: `$unknownLock`")
     }
 
-    val jdbcUrlFile = new File(dbdir, "last_jdbcurl")
-
     val safetyChecks_opt = if (dbConfig.getBoolean("postgres.safety-checks.enabled")) {
       Some(PostgresDatabases.SafetyChecks(
         localChannelsMaxAge = FiniteDuration(dbConfig.getDuration("postgres.safety-checks.max-age.local-channels").getSeconds, TimeUnit.SECONDS),
@@ -345,7 +355,7 @@ object Databases extends Logging {
       hikariConfig = hikariConfig,
       instanceId = instanceId,
       lock = lock,
-      jdbcUrlFile_opt = Some(jdbcUrlFile),
+      jdbcUrlFile_opt = jdbcUrlFile_opt,
       readOnlyUser_opt = readOnlyUser_opt,
       resetJsonColumns = resetJsonColumns,
       safetyChecks_opt = safetyChecks_opt