Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add assert config option for the JS api #1878

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+151 −112
Open

Add assert config option for the JS api #1878

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion core/actions/assertion.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import { dataform } from "df/protos/ts";
* This maintains backwards compatability with older versions.
* TODO(ekrekr): consider breaking backwards compatability of these in v4.
*/
interface ILegacyAssertionConfig extends dataform.ActionConfig.AssertionConfig {
export interface ILegacyAssertionConfig extends dataform.ActionConfig.AssertionConfig {
dependencies: Resolvable[];
database: string;
schema: string;
Expand Down Expand Up @@ -76,6 +76,10 @@ export class Assertion extends ActionBuilder<dataform.Assertion> {
this.query(nativeRequire(config.filename).query);
}

if (config.query) {
this.query(config.query);
}

if (config.dependencyTargets) {
this.dependencies(
config.dependencyTargets.map(dependencyTarget =>
Expand Down Expand Up @@ -226,6 +230,10 @@ export class Assertion extends ActionBuilder<dataform.Assertion> {
delete unverifiedConfig.type;
}

if (!!unverifiedConfig.filename && !!unverifiedConfig.query) {
this.session.compileError("Both filename and query cannot be defined in the same config");
}

return verifyObjectMatchesProto(
dataform.ActionConfig.AssertionConfig,
unverifiedConfig,
Expand Down
240 changes: 132 additions & 108 deletions core/main_test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,18 @@ import { dump as dumpYaml, load as loadYaml } from "js-yaml";
import * as path from "path";
import { CompilerFunction, NodeVM } from "vm2";

import { decode64, encode64, verifyObjectMatchesProto, VerifyProtoErrorBehaviour } from "df/common/protos";
import {
decode64,
encode64,
verifyObjectMatchesProto,
VerifyProtoErrorBehaviour
} from "df/common/protos";
import { compile } from "df/core/compilers";
import { version } from "df/core/version";
import { dataform } from "df/protos/ts";
import { asPlainObject, suite, test } from "df/testing";
import { TmpDirFixture } from "df/testing/fixtures";


const SOURCE_EXTENSIONS = ["js", "sql", "sqlx", "yaml", "ipynb"];

const VALID_WORKFLOW_SETTINGS_YAML = `
Expand Down Expand Up @@ -910,47 +914,47 @@ nodes:
- name: a
type: STRING
mode: NULLABLE
`
`;

fs.writeFileSync(
path.join(projectDir, "definitions/data_preparation.yaml"),
dataPreparationYaml
path.join(projectDir, "definitions/data_preparation.yaml"),
dataPreparationYaml
);

const result = runMainInVm(coreExecutionRequestFromPath(projectDir));

expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]);
expect(asPlainObject(result.compile.compiledGraph.dataPreparations)).deep.equals(
asPlainObject([
{
target: {
asPlainObject([
{
target: {
database: "prj",
schema: "ds",
name: "dest"
},
canonicalTarget: {
database: "prj",
schema: "ds",
name: "dest"
},
targets: [
{
database: "prj",
schema: "ds",
name: "dest"
},
canonicalTarget: {
}
],
canonicalTargets: [
{
database: "prj",
schema: "ds",
name: "dest"
},
targets: [
{
database: "prj",
schema: "ds",
name: "dest"
}
],
canonicalTargets: [
{
database: "prj",
schema: "ds",
name: "dest"
}
],
fileName: "definitions/data_preparation.yaml",
dataPreparationYaml: dumpYaml(loadYaml(dataPreparationYaml))
}
])
}
],
fileName: "definitions/data_preparation.yaml",
dataPreparationYaml: dumpYaml(loadYaml(dataPreparationYaml))
}
])
);
});

Expand Down Expand Up @@ -1010,11 +1014,11 @@ nodes:
- name: a
type: STRING
mode: NULLABLE
`
`;

fs.writeFileSync(
path.join(projectDir, "definitions/data_preparation.yaml"),
dataPreparationYaml
path.join(projectDir, "definitions/data_preparation.yaml"),
dataPreparationYaml
);

const resolvedYaml = `
Expand Down Expand Up @@ -1070,42 +1074,42 @@ nodes:
- name: a
type: STRING
mode: NULLABLE
`
`;

const result = runMainInVm(coreExecutionRequestFromPath(projectDir));

expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]);
expect(asPlainObject(result.compile.compiledGraph.dataPreparations)).deep.equals(
asPlainObject([
{
target: {
asPlainObject([
{
target: {
database: "defaultProject_projectSuffix",
schema: "defaultDataset_datasetSuffix",
name: "tablePrefix_dest"
},
canonicalTarget: {
database: "defaultProject",
schema: "defaultDataset",
name: "dest"
},
targets: [
{
database: "defaultProject_projectSuffix",
schema: "defaultDataset_datasetSuffix",
name: "tablePrefix_dest"
},
canonicalTarget: {
}
],
canonicalTargets: [
{
database: "defaultProject",
schema: "defaultDataset",
name: "dest"
},
targets: [
{
database: "defaultProject_projectSuffix",
schema: "defaultDataset_datasetSuffix",
name: "tablePrefix_dest"
}
],
canonicalTargets: [
{
database: "defaultProject",
schema: "defaultDataset",
name: "dest"
}
],
fileName: "definitions/data_preparation.yaml",
dataPreparationYaml: dumpYaml(loadYaml(resolvedYaml))
}
])
}
],
fileName: "definitions/data_preparation.yaml",
dataPreparationYaml: dumpYaml(loadYaml(resolvedYaml))
}
])
);
});
});
Expand Down Expand Up @@ -1768,20 +1772,8 @@ actions:
] as dataform.IAssertion[]
};

test(`for assertions`, () => {
const projectDir = tmpDirFixture.createNewTmpDir();
fs.writeFileSync(
path.join(projectDir, "workflow_settings.yaml"),
VALID_WORKFLOW_SETTINGS_YAML
);
fs.mkdirSync(path.join(projectDir, "definitions"));
fs.writeFileSync(path.join(projectDir, "definitions/operation.sqlx"), "SELECT 1");
fs.writeFileSync(
path.join(projectDir, "definitions/assertion.sqlx"),
// If change, then change test "action configs assertions can be loaded".
`
config {
type: "assertion",
// If change, then change test "action configs assertions can be loaded".
let assertionConfig = `
name: "name",
schema: "dataset",
database: "project",
Expand All @@ -1790,45 +1782,77 @@ config {
disabled: true,
description: "description",
hermetic: true,
dependOnDependencyAssertions: true,
dependOnDependencyAssertions: true`;
[
{
testNameSuffix: "SQLX",
filename: "assertion.sqlx",
fileContents: `
config {
type: "assertion",
${assertionConfig}
}
SELECT 1`
);
},
{
testNameSuffix: "the javascript API",
filename: "assertion.js",
fileContents: `assert(
{
${assertionConfig},
query: "SELECT 1"
}
)`
}
].forEach(testParameters => {
test(`for assertions configured with ${testParameters.testNameSuffix}`, () => {
const projectDir = tmpDirFixture.createNewTmpDir();
fs.writeFileSync(
path.join(projectDir, "workflow_settings.yaml"),
VALID_WORKFLOW_SETTINGS_YAML
);
fs.mkdirSync(path.join(projectDir, "definitions"));
fs.writeFileSync(path.join(projectDir, "definitions/operation.sqlx"), "SELECT 1");
fs.writeFileSync(
path.join(projectDir, `definitions/${testParameters.filename}`),
testParameters.fileContents
);

const result = runMainInVm(coreExecutionRequestFromPath(projectDir));
const result = runMainInVm(coreExecutionRequestFromPath(projectDir));

expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]);
expect(asPlainObject(result.compile.compiledGraph.assertions)).deep.equals(
asPlainObject([
{
target: {
database: "project",
schema: "dataset",
name: "name"
},
canonicalTarget: {
database: "project",
schema: "dataset",
name: "name"
},
actionDescriptor: {
description: "description"
},
dependencyTargets: [
{
database: "defaultProject",
schema: "defaultDataset",
name: "operation"
}
],
disabled: true,
fileName: "definitions/assertion.sqlx",
hermeticity: "HERMETIC",
tags: ["tagA", "tagB"],
query: "\n\nSELECT 1"
}
])
);
expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]);
expect(asPlainObject(result.compile.compiledGraph.assertions)).deep.equals(
asPlainObject([
{
target: {
database: "project",
schema: "dataset",
name: "name"
},
canonicalTarget: {
database: "project",
schema: "dataset",
name: "name"
},
actionDescriptor: {
description: "description"
},
dependencyTargets: [
{
database: "defaultProject",
schema: "defaultDataset",
name: "operation"
}
],
disabled: true,
fileName: "definitions/assertion.sqlx",
hermeticity: "HERMETIC",
tags: ["tagA", "tagB"],
query: "\n\nSELECT 1"
}
])
);
});
});

test(`for declarations`, () => {
Expand Down Expand Up @@ -2103,7 +2127,7 @@ config {
${assertions}
hermetic: true,
}
SELECT 1`
SELECT 1`;
const expectedCompiledTable = (name: string, fileName: string) => ({
target: {
database: "project",
Expand Down Expand Up @@ -2153,7 +2177,7 @@ SELECT 1`
key: "val"
}
}
})
});
fs.writeFileSync(
path.join(projectDir, "workflow_settings.yaml"),
VALID_WORKFLOW_SETTINGS_YAML
Expand Down
3 changes: 2 additions & 1 deletion core/session.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import * as utils from "df/core/utils";
import { toResolvable } from "df/core/utils";
import { version as dataformCoreVersion } from "df/core/version";
import { dataform } from "df/protos/ts";
import { ILegacyAssertionConfig } from "df/core/actions/assertion";

const DEFAULT_CONFIG = {
defaultSchema: "dataform",
Expand Down Expand Up @@ -275,7 +276,7 @@ export class Session {
return newTable;
}

public assert(name: string, query?: AContextable<string>): Assertion {
public assert(name: string, query?: AContextable<string> | ILegacyAssertionConfig): Assertion {
const assertion = new Assertion();
assertion.session = this;
utils.setNameAndTarget(this, assertion.proto, name, this.projectConfig.assertionSchema);
Expand Down
1 change: 1 addition & 0 deletions core/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ export function setNameAndTarget(
overrideSchema,
overrideDatabase
);
console.log("ACTION NAME TARGET:", action.target.name);
if (action.target.name.includes(".")) {
session.compileError(
new Error("Action target names cannot include '.'"),
Expand Down
Loading
Loading