Skip to content

Commit

Permalink
All the review comments have been addressed, and the required changes…
Browse files Browse the repository at this point in the history
… have been committed. The build has been successfully tested locally.
  • Loading branch information
Praveena2607 committed Jan 13, 2025
1 parent 645c24c commit 8f29e31
Show file tree
Hide file tree
Showing 4 changed files with 15 additions and 18 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -94,14 +94,14 @@ public void getCountOfNoOfRecordsTransferredToTargetBigQueryTable() throws IOExc
}

@Then("Validate records transferred to target table is equal to number of records from source table " +
"with filter {string}")
"with filter {string}")
public void validateRecordsTransferredToTargetTableIsEqualToNumberOfRecordsFromSourceTableWithFilter(String filter)
throws IOException, InterruptedException {
throws IOException, InterruptedException {
String projectId = (PluginPropertyUtils.pluginProp("projectId"));
String datasetName = (PluginPropertyUtils.pluginProp("dataset"));
int countRecordsTarget = BigQueryClient.countBqQuery(TestSetupHooks.bqTargetTable);
String selectQuery = "SELECT count(*) FROM `" + projectId + "." + datasetName + "." +
TestSetupHooks.bqTargetTable + "` WHERE " + PluginPropertyUtils.pluginProp(filter);
TestSetupHooks.bqTargetTable + "` WHERE " + PluginPropertyUtils.pluginProp(filter);
Optional<String> result = BigQueryClient.getSoleQueryResult(selectQuery);
int count = result.map(Integer::parseInt).orElse(0);
BeforeActions.scenario.write("Number of records transferred with respect to filter:" + count);
Expand All @@ -110,7 +110,7 @@ public void validateRecordsTransferredToTargetTableIsEqualToNumberOfRecordsFromS

@Then("Validate partition date in output partitioned table")
public void validatePartitionDateInOutputPartitionedTable()
throws IOException, InterruptedException {
throws IOException, InterruptedException {
Optional<String> result = BigQueryClient
.getSoleQueryResult("SELECT distinct _PARTITIONDATE as pt FROM `" +
(PluginPropertyUtils.pluginProp("projectId")) + "." +
Expand Down Expand Up @@ -233,10 +233,10 @@ public void verifyTheBigQueryValidationErrorMessageForInvalidProperty(String pro
.errorProp(E2ETestConstants.ERROR_MSG_BQ_INCORRECT_CHUNKSIZE);
} else if (property.equalsIgnoreCase("bucket")) {
expectedErrorMessage = PluginPropertyUtils
.errorProp(E2ETestConstants.ERROR_MSG_BQ_INCORRECT_TEMPORARY_BUCKET);
.errorProp(E2ETestConstants.ERROR_MSG_BQ_INCORRECT_TEMPORARY_BUCKET);
} else if (property.equalsIgnoreCase("table")) {
expectedErrorMessage = PluginPropertyUtils
.errorProp(E2ETestConstants.ERROR_MSG_INCORRECT_TABLE_NAME);
.errorProp(E2ETestConstants.ERROR_MSG_INCORRECT_TABLE_NAME);
} else {
expectedErrorMessage = PluginPropertyUtils.errorProp(E2ETestConstants.ERROR_MSG_BQ_INCORRECT_PROPERTY).
replaceAll("PROPERTY", property.substring(0, 1).toUpperCase() + property.substring(1));
Expand All @@ -260,5 +260,4 @@ public void validateRecordsTransferredToTargetTableIsEqualToNumberOfRecordsFromS
BeforeActions.scenario.write("Number of records transferred from source table to target table:" + count);
Assert.assertEquals(count, countRecordsTarget);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,7 @@ public enum CdfPluginPropertyLocator {
GCS_MOVE_DESTINATION_PATH("destPath"),
PARTITION_START_DATE("partitionFrom"),
PARTITION_END_DATE("partitionTo"),
FILTER("filter"),
OUTPUT_SCHEMA("Output Schema-macro-input");
FILTER("filter");

public String pluginProperty;
CdfPluginPropertyLocator(String property) {
Expand Down Expand Up @@ -79,7 +78,6 @@ public enum CdfPluginPropertyLocator {
.put("gcsMoveSourcePath", CdfPluginPropertyLocator.GCS_MOVE_SOURCE_PATH)
.put("gcsMoveDestinationPath", CdfPluginPropertyLocator.GCS_MOVE_DESTINATION_PATH)
.put("filter", CdfPluginPropertyLocator.FILTER)
.put("Output Schema-macro-input", CdfPluginPropertyLocator.OUTPUT_SCHEMA)
.put("partitionFrom", CdfPluginPropertyLocator.PARTITION_START_DATE)
.put("partitionTo", CdfPluginPropertyLocator.PARTITION_END_DATE)
.build();
Expand Down
4 changes: 2 additions & 2 deletions src/e2e-test/resources/errorMessage.properties
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ errorMessageMultipleFileWithoutClearDefaultSchema=Found a row with 4 fields when
errorMessageInvalidSourcePath=Invalid bucket name in path 'abc@'. Bucket name should
errorMessageInvalidDestPath=Invalid bucket name in path 'abc@'. Bucket name should
errorMessageInvalidEncryptionKey=CryptoKeyName.parse: formattedString not in valid format: Parameter "abc@" must be
errorMessageIncorrectPartitionStartDate=10-01-2025 is not in a valid format. Enter valid date in format: yyyy-MM-dd
errorMessageIncorrectPartitionEndDate=11-01-2025 is not in a valid format. Enter valid date in format: yyyy-MM-dd
errorMessageIncorrectPartitionStartDate=13-01-2025 is not in a valid format. Enter valid date in format: yyyy-MM-dd
errorMessageIncorrectPartitionEndDate=14-01-2025 is not in a valid format. Enter valid date in format: yyyy-MM-dd
errorMessageIncorrectReferenceName=Invalid reference name 'invalidRef&^*&&*'. Supported characters are: letters, numbers, and '_', '-', '.', or '$'.
errorLogsMessageInvalidFilter=Spark Program 'phase-1' failed.
12 changes: 6 additions & 6 deletions src/e2e-test/resources/pluginParameters.properties
Original file line number Diff line number Diff line change
Expand Up @@ -370,17 +370,17 @@ bqTargetTable=dummy
bqTargetTable2=dummy
bqmtTargetTable=tabA
bqmtTargetTable2=tabB
bqStartDate=2025-01-10
bqEndDate=2025-01-11
partitionFrom=2025-01-10
partitionTo=2025-01-11
bqStartDate=2025-01-13
bqEndDate=2025-01-14
partitionFrom=2025-01-113
partitionTo=2025-01-14
filter=Id=20
bqInvalidReferenceName=invalidRef&^*&&*
OutputSchema={ "type": "record", "name": "text", "fields": [{ "name": "Id", "type": "long" }, { "name": "Value", "type": "long" }, \
{ "name": "UID", "type": "string" } ] }
incorrectFilter=%%%%
bqIncorrectFormatStartDate=10-01-2025
bqIncorrectFormatEndDate=11-01-2025
bqIncorrectFormatStartDate=13-01-2025
bqIncorrectFormatEndDate=14-01-2025
## BQMT-PLUGIN-PROPERTIES-END

##CLOUDBIGTABLE-PLUGIN-PROPERTIES-START
Expand Down

0 comments on commit 8f29e31

Please sign in to comment.