From 1b4043fe4decee8652beaf0dc1b11d386ce9d0bf Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 26 Mar 2024 16:36:17 -0400 Subject: [PATCH 1/3] add append transformation --- .../transform/AppendToFileTransformation.java | 76 +++++++++++++++++++ .../models/transform/FeedTransformation.java | 3 +- .../jobs/ArbitraryTransformJobTest.java | 36 +++++++++ 3 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java new file mode 100644 index 000000000..535ec6eed --- /dev/null +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java @@ -0,0 +1,76 @@ + +package com.conveyal.datatools.manager.models.transform; + +import com.conveyal.datatools.common.status.MonitorableJob; +import com.conveyal.datatools.manager.models.TableTransformResult; +import com.conveyal.datatools.manager.models.TransformType; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; + +public class AppendToFileTransformation extends ZipTransformation { + + public static AppendToFileTransformation create(String csvData, String table) { + AppendToFileTransformation transformation = new AppendToFileTransformation(); + transformation.csvData = csvData; + transformation.table = table; + return transformation; + } + + @Override + public void validateParameters(MonitorableJob.Status status) { + if (csvData == null) { + status.fail("CSV data must not be null"); + } + } + + @Override + public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status status) { + String tableName = table + ".txt"; + Path targetZipPath = Paths.get(zipTarget.gtfsFile.getAbsolutePath()); + try ( + FileSystem targetZipFs = FileSystems.newFileSystem(targetZipPath, (ClassLoader) null); + InputStream inputStream = new ByteArrayInputStream(csvData.getBytes(StandardCharsets.UTF_8)); + ) { + TransformType type = TransformType.TABLE_MODIFIED; + + Path targetTxtFilePath = getTablePathInZip(tableName, targetZipFs); + + final File tempFile = File.createTempFile(tableName + "-temp", ".txt"); + Files.copy(targetTxtFilePath, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING); + + // Append CSV data into the target file in the temporary copy of file + try (OutputStream os = new FileOutputStream(tempFile, true)) { + os.write(inputStream.readAllBytes()); + } catch (Exception e) { + status.fail("Failed to write to target file", e); + } + + // Copy modified file into zip + Files.copy(tempFile.toPath(), targetTxtFilePath, StandardCopyOption.REPLACE_EXISTING); + + final int NEW_LINE_CHARACTER_CODE = 10; + int lineCount = (int) csvData.chars().filter(c -> c == NEW_LINE_CHARACTER_CODE).count(); + zipTarget.feedTransformResult.tableTransformResults.add(new TableTransformResult( + tableName, + type, + 0, + 0, + lineCount + 1, + 0 + )); + } catch (Exception e) { + status.fail("Unknown error encountered while transforming zip file", e); + } + } +} diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java index e63001b37..bed22f506 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/FeedTransformation.java @@ -32,7 +32,8 @@ @JsonSubTypes.Type(value = ReplaceFileFromVersionTransformation.class, name = "ReplaceFileFromVersionTransformation"), @JsonSubTypes.Type(value = ReplaceFileFromStringTransformation.class, name = "ReplaceFileFromStringTransformation"), @JsonSubTypes.Type(value = PreserveCustomFieldsTransformation.class, name = "PreserveCustomFieldsTransformation"), - @JsonSubTypes.Type(value = AddCustomFileFromStringTransformation.class, name = "AddCustomFileTransformation") + @JsonSubTypes.Type(value = AddCustomFileFromStringTransformation.class, name = "AddCustomFileTransformation"), + @JsonSubTypes.Type(value = AppendToFileTransformation.class, name = "AppendToFileTransformation") }) public abstract class FeedTransformation implements Serializable { private static final long serialVersionUID = 1L; diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java index 9303bf821..8541aa9ef 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java @@ -11,6 +11,7 @@ import com.conveyal.datatools.manager.models.Snapshot; import com.conveyal.datatools.manager.models.TableTransformResult; import com.conveyal.datatools.manager.models.transform.AddCustomFileFromStringTransformation; +import com.conveyal.datatools.manager.models.transform.AppendToFileTransformation; import com.conveyal.datatools.manager.models.transform.DeleteRecordsTransformation; import com.conveyal.datatools.manager.models.transform.FeedTransformRules; import com.conveyal.datatools.manager.models.transform.FeedTransformation; @@ -192,6 +193,37 @@ void replaceGtfsPlusFileFailsIfSourceIsMissing() throws IOException { assertThat(targetVersion.validationResult, Matchers.nullValue()); } + @Test + void canAppendToStops() throws SQLException, IOException { + sourceVersion = createFeedVersion( + feedSource, + zipFolderFiles("fake-agency-with-only-calendar") + ); + FeedTransformation transformation = AppendToFileTransformation.create(generateStopRow(), "stops"); + FeedTransformRules transformRules = new FeedTransformRules(transformation); + feedSource.transformRules.add(transformRules); + Persistence.feedSources.replace(feedSource.id, feedSource); + // Create new target version (note: the folder has no stop_attributes.txt file) + targetVersion = createFeedVersion( + feedSource, + zipFolderFiles("fake-agency-with-only-calendar-dates") + ); + LOG.info("Checking assertions."); + assertEquals( + 6, // Magic number should match row count in string produced by generateFeedInfo + targetVersion.feedLoadResult.stops.rowCount, + "stops.txt row count should equal input csv data # of rows + 1 extra row" + ); + // Check for presence of new stop id in database (one record). + assertThatSqlCountQueryYieldsExpectedCount( + String.format( + "SELECT count(*) FROM %s.stops WHERE stop_id = '%s'", + targetVersion.namespace, + "new" + ), + 1 + ); + } @Test void canReplaceFeedInfo() throws SQLException, IOException { // Generate random UUID for feedId, which gets placed into the csv data. @@ -282,6 +314,10 @@ private static String generateStopsWithCustomFields() { + "\n1234567,customValue3,customValue4"; } + private static String generateStopRow() { + return "\nnew,new,appended stop,,37.06668,-122.07781,,,0,123,,"; + } + private static String generateCustomCsvData() { return "custom_column1,custom_column2,custom_column3" + "\ncustomValue1,customValue2,customValue3" From ac604859233e9c41257faf66dc825f753d939321 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Tue, 26 Mar 2024 16:39:33 -0400 Subject: [PATCH 2/3] correct comment --- .../datatools/manager/jobs/ArbitraryTransformJobTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java index 8541aa9ef..01d57be88 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java @@ -210,7 +210,7 @@ void canAppendToStops() throws SQLException, IOException { ); LOG.info("Checking assertions."); assertEquals( - 6, // Magic number should match row count in string produced by generateFeedInfo + 6, // Magic number should match row count of stops.txt with one extra targetVersion.feedLoadResult.stops.rowCount, "stops.txt row count should equal input csv data # of rows + 1 extra row" ); From 8082faceae934234d3be91b1adb7eec868cc80a8 Mon Sep 17 00:00:00 2001 From: miles-grant-ibi Date: Wed, 27 Mar 2024 16:07:11 -0400 Subject: [PATCH 3/3] refactor: address pr feedback --- .../transform/AppendToFileTransformation.java | 9 +++++++-- .../manager/jobs/ArbitraryTransformJobTest.java | 16 +++++----------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java b/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java index 535ec6eed..d290afd66 100644 --- a/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java +++ b/src/main/java/com/conveyal/datatools/manager/models/transform/AppendToFileTransformation.java @@ -38,9 +38,11 @@ public void validateParameters(MonitorableJob.Status status) { public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status status) { String tableName = table + ".txt"; Path targetZipPath = Paths.get(zipTarget.gtfsFile.getAbsolutePath()); + try ( - FileSystem targetZipFs = FileSystems.newFileSystem(targetZipPath, (ClassLoader) null); - InputStream inputStream = new ByteArrayInputStream(csvData.getBytes(StandardCharsets.UTF_8)); + FileSystem targetZipFs = FileSystems.newFileSystem(targetZipPath, (ClassLoader) null); + InputStream newLineStream = new ByteArrayInputStream("\n".getBytes(StandardCharsets.UTF_8)); + InputStream inputStream = new ByteArrayInputStream(csvData.getBytes(StandardCharsets.UTF_8)); ) { TransformType type = TransformType.TABLE_MODIFIED; @@ -51,6 +53,9 @@ public void transform(FeedTransformZipTarget zipTarget, MonitorableJob.Status st // Append CSV data into the target file in the temporary copy of file try (OutputStream os = new FileOutputStream(tempFile, true)) { + // Append a newline in case our data doesn't include one + // Having an extra newline is not a problem! + os.write(newLineStream.readAllBytes()); os.write(inputStream.readAllBytes()); } catch (Exception e) { status.fail("Failed to write to target file", e); diff --git a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java index 01d57be88..eb25b619c 100644 --- a/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java +++ b/src/test/java/com/conveyal/datatools/manager/jobs/ArbitraryTransformJobTest.java @@ -4,7 +4,6 @@ import com.conveyal.datatools.UnitTest; import com.conveyal.datatools.manager.auth.Auth0Connection; import com.conveyal.datatools.manager.auth.Auth0UserProfile; -import com.conveyal.datatools.manager.models.FeedRetrievalMethod; import com.conveyal.datatools.manager.models.FeedSource; import com.conveyal.datatools.manager.models.FeedVersion; import com.conveyal.datatools.manager.models.Project; @@ -27,17 +26,11 @@ import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.supercsv.io.CsvMapReader; -import org.supercsv.prefs.CsvPreference; -import java.io.File; -import java.io.InputStream; import java.io.IOException; -import java.io.InputStreamReader; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.UUID; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; @@ -47,7 +40,6 @@ import static com.conveyal.datatools.TestUtils.createFeedVersion; import static com.conveyal.datatools.TestUtils.zipFolderFiles; import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.MANUALLY_UPLOADED; -import static com.conveyal.datatools.manager.models.FeedRetrievalMethod.VERSION_CLONE; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -210,9 +202,9 @@ void canAppendToStops() throws SQLException, IOException { ); LOG.info("Checking assertions."); assertEquals( - 6, // Magic number should match row count of stops.txt with one extra + 5 + 3, // Magic number should match row count of stops.txt with three extra targetVersion.feedLoadResult.stops.rowCount, - "stops.txt row count should equal input csv data # of rows + 1 extra row" + "stops.txt row count should equal input csv data # of rows + 3 extra rows" ); // Check for presence of new stop id in database (one record). assertThatSqlCountQueryYieldsExpectedCount( @@ -315,7 +307,9 @@ private static String generateStopsWithCustomFields() { } private static String generateStopRow() { - return "\nnew,new,appended stop,,37.06668,-122.07781,,,0,123,,"; + return "new3,new3,appended stop,,37,-122,,,0,123,," + + "\nnew2,new2,appended stop,,37,-122,,,0,123,," + + "\nnew,new,appended stop,,37.06668,-122.07781,,,0,123,,"; } private static String generateCustomCsvData() {