Change Teletraan DB test setups

commit-id:0e28cb39
pinterest · May 14, 2024 · f8fe3f5 · f8fe3f5
1 parent 4609501
commit f8fe3f5
Show file tree

Hide file tree

Showing 43 changed files with 179 additions and 113 deletions.
diff --git a/deploy-service/common/pom.xml b/deploy-service/common/pom.xml
@@ -20,6 +20,12 @@
     </properties>
 
     <build>
+        <testResources>
+            <testResource>
+                <directory>src/main/resources</directory>
+                <filtering>true</filtering>
+            </testResource>
+        </testResources>
         <plugins>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>

diff --git a/deploy-service/common/src/main/resources/sql/cleanup.sql b/deploy-service/common/src/main/resources/sql/cleanup.sql
@@ -24,4 +24,8 @@ DROP TABLE IF EXISTS tags;
 DROP TABLE IF EXISTS tokens_and_roles;
 DROP TABLE IF EXISTS user_ratings;
 DROP TABLE IF EXISTS users_and_roles;
-DROP TABLE IF EXISTS schedules
+DROP TABLE IF EXISTS schedules;
+DROP TABLE IF EXISTS hosts_and_agents;
+DROP TABLE IF EXISTS deploy_constraints;
+DROP TABLE IF EXISTS agent_counts;
+DROP TABLE IF EXISTS pindeploy;
diff --git a/deploy-service/common/src/main/resources/sql/schema-update-1.sql b/deploy-service/common/src/main/resources/sql/schema-update-1.sql
@@ -0,0 +1 @@
+../../../../../../tools/mysql/schema-update-1.sql
diff --git a/deploy-service/common/src/main/resources/sql/schema-update-10.sql b/deploy-service/common/src/main/resources/sql/schema-update-10.sql
@@ -0,0 +1 @@
+../../../../../../tools/mysql/schema-update-10.sql
diff --git a/deploy-service/common/src/main/resources/sql/schema-update-11.sql b/deploy-service/common/src/main/resources/sql/schema-update-11.sql
@@ -0,0 +1 @@
+../../../../../../tools/mysql/schema-update-11.sql
diff --git a/deploy-service/common/src/main/resources/sql/schema-update-12.sql b/deploy-service/common/src/main/resources/sql/schema-update-12.sql
@@ -0,0 +1 @@
+../../../../../../tools/mysql/schema-update-12.sql
diff --git a/deploy-service/common/src/main/resources/sql/schema-update-13.sql b/deploy-service/common/src/main/resources/sql/schema-update-13.sql
@@ -0,0 +1 @@
+../../../../../../tools/mysql/schema-update-13.sql
diff --git a/deploy-service/common/src/main/resources/sql/schema-update-14.sql b/deploy-service/common/src/main/resources/sql/schema-update-14.sql
@@ -0,0 +1 @@
+../../../../../../tools/mysql/schema-update-14.sql
diff --git a/deploy-service/common/src/main/resources/sql/schema-update-15.sql b/deploy-service/common/src/main/resources/sql/schema-update-15.sql
@@ -0,0 +1 @@
+../../../../../../tools/mysql/schema-update-15.sql
diff --git a/deploy-service/common/src/main/resources/sql/schema-update-16.sql b/deploy-service/common/src/main/resources/sql/schema-update-16.sql
@@ -0,0 +1 @@
+../../../../../../tools/mysql/schema-update-16.sql
diff --git a/deploy-service/common/src/main/resources/sql/schema-update-17.sql b/deploy-service/common/src/main/resources/sql/schema-update-17.sql
@@ -0,0 +1 @@
+../../../../../../tools/mysql/schema-update-17.sql
diff --git a/deploy-service/common/src/main/resources/sql/schema-update-18.sql b/deploy-service/common/src/main/resources/sql/schema-update-18.sql
@@ -0,0 +1 @@
+../../../../../../tools/mysql/schema-update-18.sql
diff --git a/deploy-service/common/src/main/resources/sql/schema-update-19.sql b/deploy-service/common/src/main/resources/sql/schema-update-19.sql
@@ -0,0 +1 @@
+../../../../../../tools/mysql/schema-update-19.sql
diff --git a/deploy-service/common/src/main/resources/sql/schema-update-2.sql b/deploy-service/common/src/main/resources/sql/schema-update-2.sql
@@ -0,0 +1 @@
+../../../../../../tools/mysql/schema-update-2.sql
diff --git a/deploy-service/common/src/main/resources/sql/schema-update-3.sql b/deploy-service/common/src/main/resources/sql/schema-update-3.sql
@@ -0,0 +1 @@
+../../../../../../tools/mysql/schema-update-3.sql
diff --git a/deploy-service/common/src/main/resources/sql/schema-update-4.sql b/deploy-service/common/src/main/resources/sql/schema-update-4.sql
@@ -0,0 +1 @@
+../../../../../../tools/mysql/schema-update-4.sql
diff --git a/deploy-service/common/src/main/resources/sql/schema-update-5.sql b/deploy-service/common/src/main/resources/sql/schema-update-5.sql
@@ -0,0 +1 @@
+../../../../../../tools/mysql/schema-update-5.sql
diff --git a/deploy-service/common/src/main/resources/sql/schema-update-6.sql b/deploy-service/common/src/main/resources/sql/schema-update-6.sql
@@ -0,0 +1 @@
+../../../../../../tools/mysql/schema-update-6.sql
diff --git a/deploy-service/common/src/main/resources/sql/schema-update-7.sql b/deploy-service/common/src/main/resources/sql/schema-update-7.sql
@@ -0,0 +1 @@
+../../../../../../tools/mysql/schema-update-7.sql
diff --git a/deploy-service/common/src/main/resources/sql/schema-update-8.sql b/deploy-service/common/src/main/resources/sql/schema-update-8.sql
@@ -0,0 +1 @@
+../../../../../../tools/mysql/schema-update-8.sql
diff --git a/deploy-service/common/src/main/resources/sql/schema-update-9.sql b/deploy-service/common/src/main/resources/sql/schema-update-9.sql
@@ -0,0 +1 @@
+../../../../../../tools/mysql/schema-update-9.sql
diff --git a/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBDAOTest.java b/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBDAOTest.java
@@ -16,7 +16,6 @@
 package com.pinterest.deployservice.db;
 
 import com.pinterest.deployservice.bean.AcceptanceStatus;
-import com.pinterest.deployservice.bean.AcceptanceType;
 import com.pinterest.deployservice.bean.AgentBean;
 import com.pinterest.deployservice.bean.AgentErrorBean;
 import com.pinterest.deployservice.bean.AgentState;
@@ -26,19 +25,15 @@
 import com.pinterest.deployservice.bean.DataBean;
 import com.pinterest.deployservice.bean.DeployBean;
 import com.pinterest.deployservice.bean.DeployFilterBean;
-import com.pinterest.deployservice.bean.DeployPriority;
 import com.pinterest.deployservice.bean.DeployQueryResultBean;
 import com.pinterest.deployservice.bean.DeployStage;
 import com.pinterest.deployservice.bean.DeployState;
 import com.pinterest.deployservice.bean.DeployType;
-import com.pinterest.deployservice.bean.EnvState;
 import com.pinterest.deployservice.bean.EnvironBean;
-import com.pinterest.deployservice.bean.EnvironState;
 import com.pinterest.deployservice.bean.GroupRolesBean;
 import com.pinterest.deployservice.bean.HostBean;
 import com.pinterest.deployservice.bean.HostState;
 import com.pinterest.deployservice.bean.HostTagBean;
-import com.pinterest.deployservice.bean.OverridePolicy;
 import com.pinterest.deployservice.bean.PromoteBean;
 import com.pinterest.deployservice.bean.PromoteType;
 import com.pinterest.deployservice.bean.RatingBean;
@@ -70,16 +65,15 @@
 import com.pinterest.deployservice.dao.TokenRolesDAO;
 import com.pinterest.deployservice.dao.UserRolesDAO;
 import com.pinterest.deployservice.dao.UtilDAO;
+import com.pinterest.deployservice.fixture.EnvironBeanFixture;
 import com.pinterest.teletraan.universal.security.bean.AuthZResource;
 import org.apache.commons.dbcp.BasicDataSource;
 import org.apache.commons.lang.builder.EqualsBuilder;
 import org.joda.time.DateTime;
 import org.joda.time.Interval;
-
-import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
-import org.testcontainers.containers.MySQLContainer;
 
 import java.sql.Connection;
 import java.util.ArrayList;
@@ -115,38 +109,35 @@ public class DBDAOTest {
     private static TagDAO tagDAO;
     private static ScheduleDAO scheduleDAO;
     private static UtilDAO utilDAO;
-
-    public static final MySQLContainer mysql = DBUtils.getContainer();
+    private static BasicDataSource dataSource;
 
     @BeforeAll
     public static void setUpClass() throws Exception {
-        mysql.start();
-        BasicDataSource DATASOURCE = DatabaseUtil.createLocalDataSource(mysql.getJdbcUrl());
-        DBUtils.runMigrations(DATASOURCE);
-
-        buildDAO = new DBBuildDAOImpl(DATASOURCE);
-        agentDAO = new DBAgentDAOImpl(DATASOURCE);
-        agentErrorDAO = new DBAgentErrorDAOImpl(DATASOURCE);
-        dataDAO = new DBDataDAOImpl(DATASOURCE);
-        deployDAO = new DBDeployDAOImpl(DATASOURCE);
-        environDAO = new DBEnvironDAOImpl(DATASOURCE);
-        promoteDAO = new DBPromoteDAOImpl(DATASOURCE);
-        hostDAO = new DBHostDAOImpl(DATASOURCE);
-        hostTagDAO = new DBHostTagDAOImpl(DATASOURCE);
-        groupDAO = new DBGroupDAOImpl(DATASOURCE);
-        ratingDAO = new DBRatingsDAOImpl(DATASOURCE);
-        userRolesDAO = new DBUserRolesDAOImpl(DATASOURCE);
-        groupRolesDAO = new DBGroupRolesDAOImpl(DATASOURCE);
-        tokenRolesDAO = new DBTokenRolesDAOImpl(DATASOURCE);
-        configHistoryDAO = new DBConfigHistoryDAOImpl(DATASOURCE);
-        tagDAO = new DBTagDAOImpl(DATASOURCE);
-        scheduleDAO = new DBScheduleDAOImpl(DATASOURCE);
-        utilDAO = new DBUtilDAOImpl(DATASOURCE);
+        dataSource = DBUtils.createTestDataSource();
+
+        buildDAO = new DBBuildDAOImpl(dataSource);
+        agentDAO = new DBAgentDAOImpl(dataSource);
+        agentErrorDAO = new DBAgentErrorDAOImpl(dataSource);
+        dataDAO = new DBDataDAOImpl(dataSource);
+        deployDAO = new DBDeployDAOImpl(dataSource);
+        environDAO = new DBEnvironDAOImpl(dataSource);
+        promoteDAO = new DBPromoteDAOImpl(dataSource);
+        hostDAO = new DBHostDAOImpl(dataSource);
+        hostTagDAO = new DBHostTagDAOImpl(dataSource);
+        groupDAO = new DBGroupDAOImpl(dataSource);
+        ratingDAO = new DBRatingsDAOImpl(dataSource);
+        userRolesDAO = new DBUserRolesDAOImpl(dataSource);
+        groupRolesDAO = new DBGroupRolesDAOImpl(dataSource);
+        tokenRolesDAO = new DBTokenRolesDAOImpl(dataSource);
+        configHistoryDAO = new DBConfigHistoryDAOImpl(dataSource);
+        tagDAO = new DBTagDAOImpl(dataSource);
+        scheduleDAO = new DBScheduleDAOImpl(dataSource);
+        utilDAO = new DBUtilDAOImpl(dataSource);
     }
 
-    @AfterAll
-    public static void tearDownClass() throws Exception {
-        mysql.stop();
+    @AfterEach
+    void tearDown() throws Exception {
+        DBUtils.truncateAllTables(dataSource);
     }
 
     @Test
@@ -580,7 +571,7 @@ public void testEnvDAO() throws Exception {
         assertTrue(EqualsBuilder.reflectionEquals(envBean, envBean22));
 
         // Test Watcher Column
-        assertTrue(envBean2.getWatch_recipients().equals("watcher"));
+        assertEquals(envBean.getWatch_recipients(), envBean2.getWatch_recipients());
 
         // Test update
         EnvironBean envBean3 = new EnvironBean();
@@ -1127,35 +1118,11 @@ public void testUtilDAO() throws Exception {
 
     private EnvironBean genDefaultEnvBean(
             String envId, String envName, String envStage, String deployId) {
-        EnvironBean envBean = new EnvironBean();
+        EnvironBean envBean = EnvironBeanFixture.createRandomEnvironBean();
         envBean.setEnv_id(envId);
         envBean.setEnv_name(envName);
         envBean.setStage_name(envStage);
-        envBean.setEnv_state(EnvState.NORMAL);
-        envBean.setMax_parallel(1);
-        envBean.setPriority(DeployPriority.NORMAL);
-        envBean.setStuck_th(100);
-
-        // To keep the precision, the default success_th value should be 10000 in DB.
-        envBean.setSuccess_th(10000);
-        envBean.setDescription("foo");
         envBean.setDeploy_id(deployId);
-        envBean.setAdv_config_id("config_id_1");
-        envBean.setSc_config_id("envvar_id_1");
-        envBean.setLast_operator("bar");
-        envBean.setLast_update(System.currentTimeMillis());
-        envBean.setAccept_type(AcceptanceType.AUTO);
-        envBean.setNotify_authors(false);
-        envBean.setWatch_recipients("watcher");
-        envBean.setMax_deploy_num(5100);
-        envBean.setMax_deploy_day(366);
-        envBean.setIs_docker(false);
-        envBean.setMax_parallel_pct(0);
-        envBean.setState(EnvironState.NORMAL);
-        envBean.setMax_parallel_rp(1);
-        envBean.setOverride_policy(OverridePolicy.OVERRIDE);
-        envBean.setAllow_private_build(false);
-        envBean.setEnsure_trusted_build(false);
         return envBean;
     }
 

diff --git a/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBUtils.java b/deploy-service/common/src/test/java/com/pinterest/deployservice/db/DBUtils.java
@@ -17,25 +17,62 @@
 
 import com.ibatis.common.jdbc.ScriptRunner;
 import java.io.BufferedReader;
+import java.io.IOException;
 import java.io.InputStreamReader;
 import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
 import org.apache.commons.dbcp.BasicDataSource;
 import org.testcontainers.containers.MySQLContainer;
 
 public class DBUtils {
-
+    private static final String MYSQL_IMAGE_NAME = "mysql:8.0-oracle";
+    private static final String MYSQL_URL = "jdbc:mysql://0.0.0.0:3303/deploy?useSSL=false";
     public static String DATABASE_NAME = "deploy";
     public static String DATABASE_USER = "root";
     public static String DATABASE_PASSWORD = "";
+    private static MySQLContainer container;
+    private static BasicDataSource dataSource;
+
+    public static BasicDataSource createTestDataSource() throws IOException, SQLException {
+        if (dataSource == null) {
+            setUpDataSource();
+        }
+        runMigrations(dataSource);
+        return dataSource;
+    }
+
+    public static void truncateAllTables(BasicDataSource dataSource) throws Exception {
+        try (Connection conn = dataSource.getConnection();
+                Statement query = conn.createStatement();
+                Statement stmt = conn.createStatement(); ) {
+            ResultSet rs =
+                    query.executeQuery(
+                            "SELECT table_name FROM information_schema.tables WHERE table_schema = SCHEMA()");
+            stmt.addBatch("SET FOREIGN_KEY_CHECKS=0");
+            while (rs.next()) {
+                String sqlStatement = String.format("TRUNCATE `%s`", rs.getString(1));
+                stmt.addBatch(sqlStatement);
+            }
+            stmt.addBatch("SET FOREIGN_KEY_CHECKS=1");
+            stmt.executeBatch();
+        }
+    }
 
-    public static MySQLContainer getContainer() {
-        return new MySQLContainer<>()
-                .withDatabaseName(DATABASE_NAME)
-                .withUsername(DATABASE_USER)
-                .withPassword(DATABASE_PASSWORD);
+    private static MySQLContainer getContainer() {
+        if (container == null) {
+            container =
+                    new MySQLContainer(MYSQL_IMAGE_NAME)
+                            .withDatabaseName(DATABASE_NAME)
+                            .withUsername(DATABASE_USER)
+                            .withPassword(DATABASE_PASSWORD);
+            container.start();
+        }
+        return container;
     }
 
-    public static void runMigrations(BasicDataSource dataSource) throws Exception {
+    private static void runMigrations(BasicDataSource dataSource) throws IOException, SQLException {
         Connection conn = dataSource.getConnection();
         ScriptRunner runner = new ScriptRunner(conn, false, true);
         runner.runScript(
@@ -48,6 +85,34 @@ public static void runMigrations(BasicDataSource dataSource) throws Exception {
                                 DBUtils.class.getResourceAsStream("/sql/deploy.sql"))));
         conn.prepareStatement("SET sql_mode=(SELECT REPLACE(@@sql_mode,'ONLY_FULL_GROUP_BY',''));")
                 .execute();
+
+        int version = 7;
+        int executed = 0;
+        while (true) {
+            String scriptName = String.format("/sql/schema-update-%d.sql", version);
+            try {
+                runner.runScript(
+                        new InputStreamReader(DBUtils.class.getResourceAsStream(scriptName)));
+            } catch (Exception e) {
+                if (executed == 0) {
+                    throw new RuntimeException(
+                            "Could not run a single update script. Check starting version is correct");
+                }
+                break;
+            }
+            version++;
+            executed++;
+        }
         conn.close();
     }
+
+    private static void setUpDataSource() throws IOException, SQLException {
+        boolean userLocalMySQLInstance =
+                Boolean.parseBoolean(System.getenv("USE_LOCAL_MYSQL_INSTANCE"));
+        if (userLocalMySQLInstance) {
+            dataSource = DatabaseUtil.createLocalDataSource(MYSQL_URL);
+        } else {
+            dataSource = DatabaseUtil.createLocalDataSource(getContainer().getJdbcUrl());
+        }
+    }
 }