concord-server: generate admin token on start (#1559)

Author: brig

examples/ansible_project/README.md

@@ -45,7 +45,7 @@ and necessary runtime dependencies.
 ```
 curl -v \
 -H "Content-Type: application/json" \
--H "Authorization: auBy4eDWrKWsyhiDp3AQiw" \
+-H "Authorization: API_TOKEN" \
 -d '{ "name": "myProject", "cfg": { "template": "ansible" }, "repositories": { "myRepo": {"url": "[email protected]:my/repo.git", "secret": "mySecret" } } }' \
 http://localhost:8001/api/v1/org/Default/project
 ```
@@ -64,7 +64,7 @@ The `secret` parameters is the name of the key created on the step 2.
 ```
 curl -v \
 -H "Content-Type: application/json" \
--H "Authorization: auBy4eDWrKWsyhiDp3AQiw" \
+-H "Authorization: API_TOKEN" \
 -d '{ "username": "myUser" }' \
 http://localhost:8001/api/v1/user
 ```
@@ -83,15 +83,15 @@ Use the `username` value of the user created in the previous step.
 ```
 curl -v \
 -H "Content-Type: application/json" \
--H "Authorization: auBy4eDWrKWsyhiDp3AQiw" \
+-H "Authorization: API_TOKEN" \
 -d '{ "username": "myUser" }' \
 http://localhost:8001/api/v1/apikey
 ```
 
 ```json
 {
     "ok": true,
-    "key": "auBy4eDWrKWsyhiDp3AQiw"
+    "key": "API_TOKEN"
 }
 ```
 
@@ -124,7 +124,7 @@ Make a call:
 
 ```
 curl -v \
--H "Authorization: auBy4eDWrKWsyhiDp3AQiw" \
+-H "Authorization: API_TOKEN" \
 -F org=Default \
 -F project=myProject \
 -F repo=myRepo \
@@ -154,7 +154,7 @@ You can download Ansible play's statistics with this request:
 
 ```
 curl -v \
--H "Authorization: auBy4eDWrKWsyhiDp3AQiw" \
+-H "Authorization: API_TOKEN" \
 http://localhost:8001/api/v1/process/33c8f91e-db14-11e6-8d94-a3efec7ccd7b/attachment/ansible_stats.json
 ```
 

examples/ansible_remote/README.md

@@ -6,7 +6,7 @@ Example of running an Ansible playbook on a remote host without creating a proje
 
 1. Upload the remote ssh key to the server:
 ```
-curl -H "Authorization: auBy4eDWrKWsyhiDp3AQiw" \
+curl -H "Authorization: API_TOKEN" \
 -F private=@/path/to/id_rsa \
 -F public=@/path/to/id_rsa.pub \
 -F storePassword=mySecretPassword \

examples/secrets/README.md

@@ -6,7 +6,7 @@ Example of using "secrets" in processes.
 
 1. create a new SSH key pair:
 ```
-$ curl -H "Authorization: auBy4eDWrKWsyhiDp3AQiw" -F storePassword=12345678 -F name=myKey -F type=KEY_PAIR 'http://localhost:8001/api/v1/org/Default/secret'
+$ curl -H "Authorization: API_TOKEN" -F storePassword=12345678 -F name=myKey -F type=KEY_PAIR 'http://localhost:8001/api/v1/org/Default/secret'
 {
   "name" : "myKey",
   "publicKey" : "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCmV6+q6Uh8j8GYl0nzcwTGpjBwY1Dvv3QIAfmdwC8N6HredMl5hV3RCtpplYR7aItTorWVUYF1MMmXYKr6tjgU9hha2N2NogRjgPWzSVuR8GVa7CF155NB4nlUxt5cGidLj5Uwmy/uQm4Mni5pg/kZMGyIf+gMmcuQXDG3TOHwmJ48HOrpqxkUKaft3SYYOy7F8TjWFnmyXNlMCskSEJd5XdLDhyuDhDEXGpDSsT1brsq0WRXtFyBDjjNeYfI4J9jyOCpAXzbDCt7eYoYK+kod/b6RV8nbaxWALx2fwJS0bDhV3a9chwEyat24Ml66Z5LfCabCE7SGpFhTas56xYkH concord-server",
@@ -19,7 +19,7 @@ The `storePassword` value is the password you must use to decrypt/export the sec
 
 2. create a new username/password pair:
 ```
-$ curl -H "Authorization: auBy4eDWrKWsyhiDp3AQiw" -F username=myUser -F password=myPassword -F storePassword=12345678 -F name=myCreds -F type=USERNAME_PASSWORD 'http://localhost:8001/api/v1/org/Default/secret'
+$ curl -H "Authorization: API_TOKEN" -F username=myUser -F password=myPassword -F storePassword=12345678 -F name=myCreds -F type=USERNAME_PASSWORD 'http://localhost:8001/api/v1/org/Default/secret'
 {
   "exportPassword" : "12345678",
   "ok" : true
@@ -30,7 +30,7 @@ For the sake of the example, the same `storePassword` value is used.
 
 3. create a plain value secret:
 ```
-$ curl -H "Authorization: auBy4eDWrKWsyhiDp3AQiw" -F secret='my horrible secret' -F storePassword=12345678 -F name=myValue -F type=DATA 'http://localhost:8001/api/v1/org/Default/secret'
+$ curl -H "Authorization: API_TOKEN" -F secret='my horrible secret' -F storePassword=12345678 -F name=myValue -F type=DATA 'http://localhost:8001/api/v1/org/Default/secret'
 {
   "exportPassword" : "12345678",
   "ok" : true

it/common/src/main/java/com/walmartlabs/concord/it/common/ServerClient.java

@@ -39,7 +39,7 @@
 public class ServerClient {
 
     /**
-     * As defined in db/src/main/resources/com/walmartlabs/concord/server/db/v0.0.1.xml
+     * As defined in server.conf
      */
     public static final String DEFAULT_API_KEY = "auBy4eDWrKWsyhiDp3AQiw";
 

it/console/src/test/resources/server.conf

@@ -1,4 +1,10 @@
 concord-server {
+    db {
+        changeLogParameters {
+            defaultAdminToken = "auBy4eDWrKWsyhiDp3AQiw"
+        }
+    }
+
     secretStore {
         serverPassword = "aXRpdGl0"
         secretStoreSalt = "aXRpdGl0"

it/server/src/test/resources/server.conf

@@ -1,4 +1,10 @@
 concord-server {
+    db {
+        changeLogParameters {
+            defaultAdminToken = "auBy4eDWrKWsyhiDp3AQiw"
+        }
+    }
+
     secretStore {
         serverPassword = "aXRpdGl0"
         secretStoreSalt = "aXRpdGl0"

server/db/pom.xml

@@ -27,6 +27,10 @@
             <groupId>com.walmartlabs.concord.server</groupId>
             <artifactId>concord-server-sdk</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.walmartlabs.concord.server</groupId>
+            <artifactId>liquibase-ext</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.postgresql</groupId>
             <artifactId>postgresql</artifactId>
@@ -163,6 +167,7 @@
                     <username>${db.username}</username>
                     <password>${db.password}</password>
                     <promptOnNonLocalDatabase>false</promptOnNonLocalDatabase>
+                    <contexts>codegen</contexts>
                 </configuration>
             </plugin>
             <plugin>

server/db/src/main/java/com/walmartlabs/concord/db/DataSourceUtils.java

@@ -39,6 +39,7 @@
 
 import javax.sql.DataSource;
 import java.sql.Connection;
+import java.util.Map;
 
 public final class DataSourceUtils {
 
@@ -47,7 +48,6 @@ public final class DataSourceUtils {
     private static final int MIGRATION_MAX_RETRIES = 10;
     private static final int MIGRATION_RETRY_DELAY = 10000;
 
-
     public static DataSource createDataSource(DatabaseConfiguration cfg,
                                               String poolName,
                                               String username,
@@ -70,11 +70,28 @@ public static DataSource createDataSource(DatabaseConfiguration cfg,
     }
 
     public static void migrateDb(DataSource ds, DatabaseChangeLogProvider p) {
+        migrateDb(ds, p, null);
+    }
+
+    /**
+     * Migrate a database using the provided changelog and Liquibase parameters.
+     *
+     * @param dataSource        datasource to use for migration
+     * @param changeLogProvider provider of the changelog
+     * @param changeLogParams   Liquibase parameters to use during the migration
+     */
+    public static void migrateDb(DataSource dataSource,
+                                 DatabaseChangeLogProvider changeLogProvider,
+                                 Map<String, Object> changeLogParams) {
+
         int retries = MIGRATION_MAX_RETRIES;
         for (int i = 0; i < retries; i++) {
-            try (Connection c = ds.getConnection()) {
-                log.info("get -> performing '{}' migration...", p);
-                migrateDb(c, p.getChangeLogPath(), p.getChangeLogTable(), p.getLockTable());
+            try (Connection c = dataSource.getConnection()) {
+                log.info("get -> performing '{}' migration...", changeLogProvider);
+                String logPath = changeLogProvider.getChangeLogPath();
+                String logTable = changeLogProvider.getChangeLogTable();
+                String lockTable = changeLogProvider.getLockTable();
+                migrateDb(c, logPath, logTable, lockTable, changeLogParams);
                 log.info("get -> done");
                 break;
             } catch (Exception e) {
@@ -93,17 +110,6 @@ public static void migrateDb(DataSource ds, DatabaseChangeLogProvider p) {
         }
     }
 
-    private static void migrateDb(Connection conn, String logPath, String logTable, String lockTable) throws Exception {
-        LogFactory.getInstance().setDefaultLoggingLevel(LogLevel.WARNING);
-
-        Database db = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(new JdbcConnection(conn));
-        db.setDatabaseChangeLogTableName(logTable);
-        db.setDatabaseChangeLogLockTableName(lockTable);
-
-        Liquibase lb = new Liquibase(logPath, new ClassLoaderResourceAccessor(), db);
-        lb.update((String) null);
-    }
-
     public static Configuration createJooqConfiguration(DataSource ds) {
         Settings settings = new Settings();
         settings.setRenderSchema(false);
@@ -115,6 +121,27 @@ public static Configuration createJooqConfiguration(DataSource ds) {
                 .set(SQLDialect.POSTGRES);
     }
 
+    private static void migrateDb(Connection conn,
+                                  String logPath,
+                                  String logTable,
+                                  String lockTable,
+                                  Map<String, Object> params) throws Exception {
+
+        LogFactory.getInstance().setDefaultLoggingLevel(LogLevel.WARNING);
+
+        Database db = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(new JdbcConnection(conn));
+        db.setDatabaseChangeLogTableName(logTable);
+        db.setDatabaseChangeLogLockTableName(lockTable);
+
+        Liquibase lb = new Liquibase(logPath, new ClassLoaderResourceAccessor(), db);
+
+        if (params != null) {
+            params.forEach(lb::setChangeLogParameter);
+        }
+
+        lb.update((String) null);
+    }
+
     private DataSourceUtils() {
     }
 }

server/db/src/main/java/com/walmartlabs/concord/db/DatabaseConfiguration.java

@@ -20,6 +20,9 @@
  * =====
  */
 
+import java.util.Collections;
+import java.util.Map;
+
 public interface DatabaseConfiguration {
 
     default String driverClassName() {
@@ -35,4 +38,8 @@ default String driverClassName() {
     int maxPoolSize();
 
     long maxLifetime();
+
+    default Map<String, Object> changeLogParameters() {
+        return Collections.emptyMap();
+    }
 }

server/db/src/main/java/com/walmartlabs/concord/db/DatabaseModule.java

@@ -49,7 +49,7 @@ public DataSource appDataSource(@MainDB DatabaseConfiguration cfg,
                 // can't inject a set of objects with the same qualifier, filter manually
                 .filter(p -> p.getClass().getAnnotation(MainDB.class) != null)
                 .sorted(Comparator.comparingInt(DatabaseChangeLogProvider::order))
-                .forEach(p -> DataSourceUtils.migrateDb(ds, p));
+                .forEach(p -> DataSourceUtils.migrateDb(ds, p, cfg.changeLogParameters()));
 
         return ds;
     }

server/db/src/main/resources/com/walmartlabs/concord/server/db/liquibase.xml

@@ -2,7 +2,7 @@
 <databaseChangeLog
         xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.5.xsd">
 
     <include file="v0.0.1.xml" relativeToChangelogFile="true"/>
     <include file="v0.2.0.xml" relativeToChangelogFile="true"/>
@@ -82,5 +82,6 @@
     <include file="v1.40.0.xml" relativeToChangelogFile="true"/>
     <include file="v1.41.0.xml" relativeToChangelogFile="true"/>
     <include file="v1.43.0.xml" relativeToChangelogFile="true"/>
+    <include file="v1.45.0.xml" relativeToChangelogFile="true"/>
 
 </databaseChangeLog>

server/db/src/main/resources/com/walmartlabs/concord/server/db/v1.45.0.xml

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+        xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <changeSet id="1450000" author="[email protected]" runInTransaction="false" context="!codegen">
+        <sql>
+            delete from API_KEYS where KEY_ID = 'd5165ca8-e8de-11e6-9bf5-136b5db23c32'
+        </sql>
+
+        <customChange class="com.walmartlabs.concord.server.liquibase.ext.ApiTokenCreator">
+            <!-- default admin user ID -->
+            <param name="userId" value="230c5c9c-d9a7-11e6-bcfd-bb681c07b26c"/>
+            <!-- value from concord-server.conf -->
+            <param name="token" value="${defaultAdminToken}"/>
+        </customChange>
+    </changeSet>
+</databaseChangeLog>

server/dist/src/main/resources/concord-server.conf

@@ -52,6 +52,15 @@ concord-server {
 
         # maximum lifetime of a connection in the pool, ms
         maxLifetime = 300000 # 5 min
+
+        # parameters using during the DB schema migration
+        changeLogParameters {
+            # the default admin API token value
+            # must be a valid base64 value
+            # if empty a new random token will be generated
+            # applied only once on the first DB migration
+            defaultAdminToken = ""
+        }
     }
 
     # "remember me" cookie support

server/impl/src/main/java/com/walmartlabs/concord/server/cfg/MainDBConfiguration.java

@@ -9,9 +9,9 @@
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
- * 
+ *
  *      http://www.apache.org/licenses/LICENSE-2.0
- * 
+ *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -27,6 +27,7 @@
 import javax.inject.Inject;
 import javax.inject.Named;
 import javax.inject.Singleton;
+import java.util.Map;
 
 @Named
 @Singleton
@@ -53,6 +54,10 @@ public class MainDBConfiguration implements DatabaseConfiguration {
     @Config("db.maxLifetime")
     private long maxLifetime;
 
+    @Inject
+    @Config("db.changeLogParameters")
+    private Map<String, Object> changeLogParameters;
+
     @Override
     public String url() {
         return url;
@@ -77,4 +82,9 @@ public int maxPoolSize() {
     public long maxLifetime() {
         return maxLifetime;
     }
+
+    @Override
+    public Map<String, Object> changeLogParameters() {
+        return changeLogParameters;
+    }
 }

server/liquibase-ext/README.md

@@ -0,0 +1,3 @@
+# Liquibase Extensions
+
+This module contains Concord-specific [Liquibase](https://www.liquibase.org/) extensions. 

server/liquibase-ext/pom.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>com.walmartlabs.concord.server</groupId>
+        <artifactId>parent</artifactId>
+        <version>1.44.1-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <artifactId>liquibase-ext</artifactId>
+    <packaging>takari-jar</packaging>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.liquibase</groupId>
+            <artifactId>liquibase-core</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>io.takari.maven.plugins</groupId>
+                <artifactId>takari-lifecycle-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+</project>