Update output for Data Preparation processing (#1808)

* Updated data preparation proto to only carry binary encoded data preparation contents after processing the data preparation definition. * Addressed PR Comments. * Addressed final PR comments * Fixed compilation issue.
dataform-co · Aug 15, 2024 · 2531b12 · 2531b12
1 parent ae85965
commit 2531b12
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 66 deletions.
diff --git a/core/actions/data_preparation.ts b/core/actions/data_preparation.ts
@@ -34,9 +34,9 @@ export class DataPreparation extends ActionBuilder<dataform.DataPreparation> {
     }
 
     config.filename = resolveActionsConfigFilename(config.filename, configPath);
-    const dataPreparationContents = nativeRequire(config.filename).asJson;
-    const dataPreparationDefinition = parseDataPreparationDefinitionJson(dataPreparationContents);
-    this.proto.dataPreparation = dataPreparationDefinition;
+    const dataPreparationAsJson = nativeRequire(config.filename).asJson;
+    const dataPreparationDefinition = parseDataPreparationDefinitionJson(dataPreparationAsJson);
+    this.proto.dataPreparationContents = dataform.DataPreparationDefinition.encode(dataPreparationDefinition).finish();
 
     // Find targets
     const targets = getTargets(dataPreparationDefinition);

diff --git a/core/main_test.ts b/core/main_test.ts
@@ -1,17 +1,18 @@
 // tslint:disable tsr-detect-non-literal-fs-filename
 import { expect } from "chai";
 import * as fs from "fs-extra";
-import { dump as dumpYaml } from "js-yaml";
+import { dump as dumpYaml, load as loadYaml } from "js-yaml";
 import * as path from "path";
 import { CompilerFunction, NodeVM } from "vm2";
 
-import { decode64, encode64 } from "df/common/protos";
+import { decode64, encode64, verifyObjectMatchesProto } from "df/common/protos";
 import { compile } from "df/core/compilers";
 import { version } from "df/core/version";
 import { dataform } from "df/protos/ts";
 import { asPlainObject, suite, test } from "df/testing";
 import { TmpDirFixture } from "df/testing/fixtures";
 
+
 const SOURCE_EXTENSIONS = ["js", "sql", "sqlx", "yaml", "ipynb"];
 
 const VALID_WORKFLOW_SETTINGS_YAML = `
@@ -877,9 +878,7 @@ actions:
 
     test(`data preparations can be loaded via an actions config file`, () => {
       const projectDir = createSimpleDataPreparationProject();
-      fs.writeFileSync(
-        path.join(projectDir, "definitions/data_preparation.yaml"),
-        `
+      const dataPreparationYaml = `
 nodes:
 - id: node1
   source:
@@ -912,6 +911,23 @@ nodes:
           type: STRING
           mode: NULLABLE
 `
+
+      fs.writeFileSync(
+        path.join(projectDir, "definitions/data_preparation.yaml"),
+          dataPreparationYaml
+      );
+
+      // Generate Base64 encoded representation of the YAML.
+      const dataPreparationAsObject = loadYaml(dataPreparationYaml);
+      const dataPreparationDefinition = verifyObjectMatchesProto(
+          dataform.DataPreparationDefinition,
+          dataPreparationAsObject as {
+            [key: string]: any;
+          }
+      );
+      const base64encodedContents = encode64(
+          dataform.DataPreparationDefinition,
+          dataPreparationDefinition
       );
 
       const result = runMainInVm(coreExecutionRequestFromPath(projectDir));
@@ -945,62 +961,8 @@ nodes:
               }
             ],
             fileName: "definitions/data_preparation.yaml",
-            dataPreparation: {
-              nodes: [
-                {
-                  id: "node1",
-                  source: {
-                    table: {
-                      dataset: "ds",
-                      project: "prj",
-                      table: "src"
-                    }
-                  },
-                  destination: {
-                    table: {
-                      dataset: "ds",
-                      project: "prj",
-                      table: "dest"
-                    }
-                  },
-                  generated: {
-                    destinationGenerated: {
-                      schema: {
-                        field: [
-                          {
-                            mode: "NULLABLE",
-                            name: "a",
-                            type: "STRING"
-                          }
-                        ]
-                      }
-                    },
-                    outputSchema: {
-                      field: [
-                        {
-                          mode: "NULLABLE",
-                          name: "a",
-                          type: "INT64"
-                        }
-                      ]
-                    },
-                    sourceGenerated: {
-                      sourceSchema: {
-                        tableSchema: {
-                          field: [
-                            {
-                              mode: "NULLABLE",
-                              name: "a",
-                              type: "STRING"
-                            }
-                          ]
-                        }
-                      }
-                    }
-                  }
-                }
-              ]
-            }
+            // Base64 encoded representation of the data preparation definition proto.
+            dataPreparationContents: base64encodedContents
           }
         ])
       );

diff --git a/protos/core.proto b/protos/core.proto
@@ -1,5 +1,4 @@
 syntax = "proto3";
-import "data_preparation.proto";
 
 package dataform;
 
@@ -268,7 +267,10 @@ message DataPreparation {
 
   bool disabled = 6;
 
-  DataPreparationDefinition data_preparation = 7;
+  // Binary encoded contents of the Data preparation proto
+  bytes data_preparation_contents = 10;
+
+  reserved 7;
 }
 
 message CompiledGraph {