Migrate Store Loader module

giis-uniovi · Jun 23, 2024 · 5bcf00f · 5bcf00f
1 parent c584e6b
commit 5bcf00f
Show file tree

Hide file tree

Showing 63 changed files with 5,429 additions and 17 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -83,7 +83,7 @@ jobs:
         if: ${{ matrix.scope != 'Core' }}
         run: >
           mvn test surefire-report:report -Daggregate=true
-          -Dtest=Test${{ matrix.scope }}* -pl tdrules-client-rdb,tdrules-store-rdb -am -Dsurefire.failIfNoSpecifiedTests=false
+          -Dtest=Test${{ matrix.scope }}* -pl tdrules-client-rdb,tdrules-store-rdb,tdrules-store-loader -am -Dsurefire.failIfNoSpecifiedTests=false
           -Dmaven.test.failure.ignore=true -U --no-transfer-progress
           -Duser.timezone=Europe/Madrid
         # NOTE: must specify a timezone to avoid oracle error ORA-01882: timezone region not found

diff --git a/README.md b/README.md
@@ -8,47 +8,61 @@
 # TdRules - Test Data Coverage Evaluation
 
 This repository contains a set of components to generate *Full Predicate Coverage Rules* and *SQL Mutants*
-to assess the coverage of the test data in relation to a query:
+to assess the coverage of the test data in relation to a query and load test data:
 
 - Generate the FPC Rules and SQL Mutants ([TdRules Service](https://in2test.lsi.uniovi.es/tdrules/)).
-- Discover the data store schema from an open JDBC connection to a relational database.
+- Discover the data store schema from an OpenApi specification or from a JDBC connection to a relational database.
 - Models to manipulate the rules (both FPC Rules or SQL Mutants) and the schema.
+- Load test data through a REST API or a JDBC connection.
 - Available for Java 8 and higher, and .NET (netstandard 2.0).
 
 NOTE: The name *TdRules* (Test Data Rules) is replacing the former *SQLRules* 
 to enable generation of rules for data stores other than relational databases.
 
 ## Quick Start
 
-- On Java, include the client api dependencies
+- On Java, include the client API dependencies
 [tdrules-client](https://central.sonatype.com/artifact/io.github.giis-uniovi/tdrules-client)
-and 
+and either 
+[tdrules-client-oa](https://central.sonatype.com/artifact/io.github.giis-uniovi/tdrules-client-oa)
+or
 [tdrules-client-rdb](https://central.sonatype.com/artifact/io.github.giis-uniovi/tdrules-client-rdb)
-available in Maven Central.
+that are available in Maven Central.
 A bom is also available:
 [tdrules-bom](https://central.sonatype.com/artifact/io.github.giis-uniovi/tdrules-bom).
 
 - On .NET, include the `TdRules` package
 [TdRules](https://www.nuget.org/packages/TdRules/)
 available in NuGet.
 
-**Example:** To generate the FPC Rules for a query `query`
-that executes against a relational database that can be reached by an open JDBC Connection `conn`,
+**Example:** To generate the FPC Rules for a query (`query`)
+that executes against 
+an OpenApi schema specification indicated by file or url (`spec`)
+or a relational database that can be reached by an open JDBC Connection (`conn`),
 you first get the data store schema model and then the rules model as follows:
 
-<details open><summary><strong>Java</strong></summary>
+<details open><summary><strong>Java (OpenApi)</strong></summary>
 
 ```Java
-TdSchema schemaModel = new DbSchemaApi(conn).getDbSchema();
+TdSchema schemaModel = new OaSchemaApi(spec).getSchema();
 TdRules rulesModel = new TdRulesApi().getRules(schemaModel, query, "");
 ```
 
 </details>
 
-<details><summary><strong>.NET</strong></summary>
+<details open><summary><strong>Java (RDB)</strong></summary>
+
+```Java
+TdSchema schemaModel = new DbSchemaApi(conn).getSchema();
+TdRules rulesModel = new TdRulesApi().getRules(schemaModel, query, "");
+```
+
+</details>
+
+<details><summary><strong>.NET (RDB only)</strong></summary>
 
 ```C#
-TdSchema schemaModel = new DbSchemaApi(conn).GetDbSchema();
+TdSchema schemaModel = new DbSchemaApi(conn).GetSchema();
 TdRules rulesModel = new TdRulesApi().GetRules(schemaModel, query, "");
 ```
 
@@ -71,20 +85,25 @@ See the general contribution policies and guidelines for *giis-uniovi* at
 Modules currently available in this repo are:
 
 - `tdrules-bom`: The bill of materials of all TdRules components.
-- `tdrules-client`: Client api to generate FPC Rules and SQL Mutants.
+- `tdrules-client`: Client API to generate FPC Rules and SQL Mutants.
 - `tdrules-model`: Models of the FPC Rules, SQL Mutants and the data store schema.
-- `tdrules-client-rdb`: Client api to generate the data store schema from a live JDBC connection.
+- `tdrules-client-oa`: Client API to generate the data store schema from an OpenApi specification.
+- `tdrules-client-rdb`: Client API to generate the data store schema from a live JDBC connection.
 - `tdrules-store-rdb`: Core component used to discover the schema of relational databases.
 - `tdrules-store-shared`: Shared components for all data stores.
+- `tdrules-store-loader`: Load test data through a REST API or a JDBC connection.
 - `setup`: A folder with the configuration of test database containers to use in your development environment.
 - `net`: A folder with the source of the .NET implementation.
 
 ```mermaid
 flowchart TD
-    client(client) --> model(model)
+    client(client) ---> model(model)
+    clientoa(client-oa) ---> model
     clientrdb(client-rdb) --> model
     clientrdb --> storerdb(store-rdb)
     storerdb --> storeshared(store-shared)
+    storeloader(store-loader) ---> model
+    storeloader ---> storeshared
 ```
 
 To set-up the test database containers in a local development environment, see the `setup` folder.

diff --git a/pom.xml b/pom.xml
@@ -40,6 +40,7 @@
 		<module>tdrules-store-rdb</module>
 		<module>tdrules-client-rdb</module>
 		<module>tdrules-client-oa</module>
+		<module>tdrules-store-loader</module>
 	</modules>
 
 	<dependencyManagement>
@@ -70,6 +71,11 @@
 				<artifactId>httpclient</artifactId>
 				<version>${httpclient4-version}</version>
 			</dependency>
+			<dependency>
+			    <groupId>org.apache.commons</groupId>
+			    <artifactId>commons-lang3</artifactId>
+			    <version>3.14.0</version>
+			</dependency>
 			<!-- otras dependencias -->
 			<dependency>
 				<groupId>io.swagger.parser.v3</groupId>
@@ -166,6 +172,8 @@
 			<dependency>
 				<groupId>com.oracle.database.jdbc</groupId>
 				<artifactId>ojdbc8</artifactId>
+				<!-- latest: 23.4.0.24.05 does not work on windows with gvenzl/oracle-xe:21.3.0-slim-faststart
+				     error: ORA-01005 null password given, use 21.14.0.0 -->
 				<version>23.4.0.24.05</version>
 				<scope>test</scope>
 			</dependency>

diff --git a/setup/database.properties b/setup/database.properties
@@ -7,6 +7,9 @@ tdrules.java.tdclirdb.postgres.user=tdclirdb
 tdrules.java.tdstorerdb2.postgres.url=jdbc:postgresql://localhost:5432/tdstorerdb2
 tdrules.java.tdstorerdb2.postgres.user=tdstorerdb2
 
+tdrules.java.tdloadrdb.postgres.url=jdbc:postgresql://localhost:5432/tdloadrdb
+tdrules.java.tdloadrdb.postgres.user=tdloadrdb
+
 tdrules.java.tdstorerdb2.postgres.meta.dbms=Database: PostgreSQL - Version: 14.
 tdrules.java.tdstorerdb2.postgres.meta.driver=Driver: PostgreSQL JDBC Driver - Version: 42.
 
@@ -17,14 +20,20 @@ tdrules.java.tdclirdb.sqlserver.user=tdclirdb
 tdrules.java.tdstorerdb2.sqlserver.url=jdbc:sqlserver://localhost:1433;databaseName=tdstorerdb2;encrypt=true;trustServerCertificate=true
 tdrules.java.tdstorerdb2.sqlserver.user=tdstorerdb2
 
+tdrules.java.tdloadrdb.sqlserver.url=jdbc:sqlserver://localhost:1433;databaseName=tdloadrdb;encrypt=true;trustServerCertificate=true
+tdrules.java.tdloadrdb.sqlserver.user=tdloadrdb
+
 tdrules.java.tdstorerdb2.sqlserver.meta.dbms=Database: Microsoft SQL Server - Version: 14.
 tdrules.java.tdstorerdb2.sqlserver.meta.driver=Driver: Microsoft JDBC Driver 12.
 
-# Oracle is not iested in the client (tdclirdb) 
+# Oracle is not tested in the client-rdb (tdclirdb) 
 # and needs two additional databases for TestOracleSchemaMulti (see internal doc)
 tdrules.java.tdstorerdb2.oracle.url=jdbc:oracle:thin:@localhost:1521:XE
 tdrules.java.tdstorerdb2.oracle.user=tdstorerdb2
 
+tdrules.java.tdloadrdb.oracle.url=jdbc:oracle:thin:@localhost:1521:XE
+tdrules.java.tdloadrdb.oracle.user=tdloadrdb
+
 tdrules.java.tdstorerdb0.oracle.url=jdbc:oracle:thin:@localhost:1521:XE
 tdrules.java.tdstorerdb0.oracle.user=tdstorerdb0
 tdrules.java.tdstorerdb1.oracle.url=jdbc:oracle:thin:@localhost:1521:XE

diff --git a/setup/oracle/oracle-setup.sh b/setup/oracle/oracle-setup.sh
@@ -32,5 +32,15 @@ sqlplus system/${TEST_ORACLE_PWD}@XE  <<-EOSQL
   grant create view  to tdstorerdb1; 
   grant dba to tdstorerdb1;
   grant unlimited tablespace to tdstorerdb1;
+----
+  create user tdloadrdb identified by "$TEST_ORACLE_PWD" ACCOUNT UNLOCK;
+  grant connect to tdloadrdb;
+  grant create session to tdloadrdb;
+  grant resource to tdloadrdb;
+  grant create table to tdloadrdb; 
+  grant create procedure to tdloadrdb; 
+  grant create view to tdloadrdb;
+  grant unlimited tablespace to tdloadrdb;
+ 
 EOSQL
 echo "-- End setup"
diff --git a/setup/postgres/postgres-setup.sh b/setup/postgres/postgres-setup.sh
@@ -7,5 +7,8 @@ psql -v ON_ERROR_STOP=1   <<-EOSQL
 
   CREATE USER tdstorerdb2 with encrypted password '$TEST_POSTGRES_PWD';
   CREATE DATABASE tdstorerdb2;
+
+  CREATE USER tdloadrdb with encrypted password '$TEST_POSTGRES_PWD';
+  CREATE DATABASE tdloadrdb;
 EOSQL
 echo "-- END SETUP!"
diff --git a/setup/sqlserver/sqlserver-setup.sh b/setup/sqlserver/sqlserver-setup.sh
@@ -26,5 +26,14 @@ echo "-- Begin setup"
   CREATE USER [tdstorerdb2] FOR LOGIN [tdstorerdb2] --WITH DEFAULT_SCHEMA=[dbo]
   EXEC sp_addrolemember 'db_owner', 'tdstorerdb2'
   GO
+  ----
+  CREATE LOGIN tdloadrdb WITH PASSWORD = '$TEST_SQLSERVER_PWD', CHECK_POLICY=OFF, CHECK_EXPIRATION=OFF, DEFAULT_LANGUAGE=spanish;
+  GO
+  CREATE DATABASE tdloadrdb
+  GO
+  USE [tdloadrdb]
+  CREATE USER [tdloadrdb] FOR LOGIN [tdloadrdb] --WITH DEFAULT_SCHEMA=[dbo]
+  EXEC sp_addrolemember 'db_owner', 'tdloadrdb'
+  GO
 EOSQL
 echo "-- END SETUP!"
diff --git a/sonar-project.properties b/sonar-project.properties
@@ -3,7 +3,7 @@ sonar.projectName=tdrules
 sonar.organization=giis
 sonar.projectVersion=1.0
 sonar.java.source=1.8
-sonar.modules=tdrules-model,tdrules-client,tdrules-store-shared,tdrules-store-rdb,tdrules-client-rdb,tdrules-client-oa
+sonar.modules=tdrules-model,tdrules-client,tdrules-store-shared,tdrules-store-rdb,tdrules-client-rdb,tdrules-client-oa,tdrules-store-loader
 sonar.sources=src/main/java
 sonar.sourceEncoding=UTF-8
 sonar.tests=src/test/java

diff --git a/tdrules-bom/pom.xml b/tdrules-bom/pom.xml
@@ -48,6 +48,11 @@
 				<artifactId>tdrules-client-oa</artifactId>
 				<version>${project.version}</version>
 			</dependency>
+			<dependency>
+				<groupId>io.github.giis-uniovi</groupId>
+				<artifactId>tdrules-store-loader</artifactId>
+				<version>${project.version}</version>
+			</dependency>
 		</dependencies>
 	</dependencyManagement>
 

diff --git a/tdrules-store-loader/.classpath b/tdrules-store-loader/.classpath
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="src" output="target/classes" path="src/main/java">
+		<attributes>
+			<attribute name="optional" value="true"/>
+			<attribute name="maven.pomderived" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="src" output="target/test-classes" path="src/test/java">
+		<attributes>
+			<attribute name="optional" value="true"/>
+			<attribute name="maven.pomderived" value="true"/>
+			<attribute name="test" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources">
+		<attributes>
+			<attribute name="maven.pomderived" value="true"/>
+			<attribute name="test" value="true"/>
+			<attribute name="optional" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
+		<attributes>
+			<attribute name="maven.pomderived" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
+		<attributes>
+			<attribute name="maven.pomderived" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
+		<attributes>
+			<attribute name="maven.pomderived" value="true"/>
+			<attribute name="optional" value="true"/>
+		</attributes>
+	</classpathentry>
+	<classpathentry kind="output" path="target/classes"/>
+</classpath>
diff --git a/tdrules-store-loader/.project b/tdrules-store-loader/.project
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>tdrules-store-loader</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.eclipse.jdt.core.javabuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.eclipse.m2e.core.maven2Builder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.eclipse.jdt.core.javanature</nature>
+		<nature>org.eclipse.m2e.core.maven2Nature</nature>
+	</natures>
+</projectDescription>
diff --git a/tdrules-store-loader/.settings/org.eclipse.core.resources.prefs b/tdrules-store-loader/.settings/org.eclipse.core.resources.prefs
@@ -0,0 +1,6 @@
+eclipse.preferences.version=1
+encoding//src/main/java=UTF-8
+encoding//src/main/resources=UTF-8
+encoding//src/test/java=UTF-8
+encoding//src/test/resources=UTF-8
+encoding/<project>=UTF-8
diff --git a/tdrules-store-loader/.settings/org.eclipse.jdt.core.prefs b/tdrules-store-loader/.settings/org.eclipse.jdt.core.prefs
@@ -0,0 +1,8 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8
+org.eclipse.jdt.core.compiler.compliance=1.8
+org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled
+org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
+org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore
+org.eclipse.jdt.core.compiler.release=disabled
+org.eclipse.jdt.core.compiler.source=1.8
diff --git a/tdrules-store-loader/.settings/org.eclipse.m2e.core.prefs b/tdrules-store-loader/.settings/org.eclipse.m2e.core.prefs
@@ -0,0 +1,4 @@
+activeProfiles=
+eclipse.preferences.version=1
+resolveWorkspaceProjects=true
+version=1