From db55cb428b4bcc29a2185beb291091ef85d886a3 Mon Sep 17 00:00:00 2001 From: Breck Yunits Date: Sun, 29 Dec 2024 12:14:08 -0800 Subject: [PATCH] Better table transforms --- package.json | 2 +- parsers/groupBy.parsers | 7 ++++--- parsers/root.parsers | 2 +- parsers/tables.parsers | 28 ++++++++++++++++++++++------ releaseNotes.scroll | 8 ++++++++ tests/tables.scroll | 15 +++++++++++++++ 6 files changed, 51 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index d59654a64..09747a0e1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "scroll-cli", - "version": "164.10.0", + "version": "164.11.0", "description": "A language for scientists of all ages. A curated collection of tools for refining and sharing thoughts.", "main": "scroll.js", "engines": { diff --git a/parsers/groupBy.parsers b/parsers/groupBy.parsers index 0a5d2dd6c..6849b54cf 100644 --- a/parsers/groupBy.parsers +++ b/parsers/groupBy.parsers @@ -86,12 +86,13 @@ scrollGroupByParser colsToReduce.forEach((col) => { const sourceColName = col.source const reduction = col.reduction + const newColName = col.name if (reduction === "concat") { - newRow[col.name] = group.map((row) => row[sourceColName]).join(" ") + newRow[newColName] = group.map((row) => row[sourceColName]).join(" ") return } if (reduction === "first") { - newRow[col.name] = group[0][sourceColName] + newRow[newColName] = group.find((row) => row[sourceColName] !== "")?.[sourceColName] return } const values = group.map((row) => row[sourceColName]).filter((val) => typeof val === "number" && !isNaN(val)) @@ -100,7 +101,7 @@ scrollGroupByParser if (reduction === "max") reducedValue = Math.max(...values) if (reduction === "min") reducedValue = Math.min(...values) if (reduction === "mean") reducedValue = values.reduce((prev, current) => prev + current, 0) / values.length - newRow[col.name] = reducedValue + newRow[newColName] = reducedValue }) rows.push(newRow) } diff --git a/parsers/root.parsers b/parsers/root.parsers index 5f2f61c79..61ec92a43 100644 --- a/parsers/root.parsers +++ b/parsers/root.parsers @@ -346,7 +346,7 @@ scrollParser } get scrollVersion() { // currently manually updated - return "164.10.0" + return "164.11.0" } // Use the first paragraph for the description // todo: add a particle method version of get that gets you the first particle. (actulaly make get return array?) diff --git a/parsers/tables.parsers b/parsers/tables.parsers index 8aa4c6cb5..38334c652 100644 --- a/parsers/tables.parsers +++ b/parsers/tables.parsers @@ -1,6 +1,6 @@ abstractTableTransformParser atoms cueAtom - inScope abstractTableVisualizationParser abstractTableTransformParser h1Parser h2Parser scrollQuestionParser htmlInlineParser scrollBrParser + inScope abstractTableVisualizationParser abstractTableTransformParser h1Parser h2Parser scrollQuestionParser htmlInlineParser scrollBrParser slashCommentParser javascript get coreTable() { return this.parent.coreTable @@ -216,7 +216,7 @@ scrollTableDelimiterParser abstractDatatableProviderParser description A datatable. extends abstractScrollParser - inScope scrollTableDataParser scrollTableDelimiterParser abstractTableVisualizationParser abstractTableTransformParser h1Parser h2Parser scrollQuestionParser htmlInlineParser scrollBrParser + inScope scrollTableDataParser scrollTableDelimiterParser abstractTableVisualizationParser abstractTableTransformParser h1Parser h2Parser scrollQuestionParser htmlInlineParser scrollBrParser slashCommentParser javascript get visualizations() { return this.topDownArray.filter(particle => particle.isTableVisualization || particle.isHeader || particle.isHtml) @@ -354,7 +354,8 @@ scrollWhereParser extends abstractTableTransformParser description Filter rows by condition. cue where - atoms cueAtom columnNameAtom comparisonAtom constantAtom + atoms cueAtom columnNameAtom comparisonAtom + catchAllAtomType constantAtom example table iris.csv where Species = setosa @@ -366,7 +367,7 @@ scrollWhereParser let untypedScalarValue = this.getAtom(3) const typedValue = isNaN(parseFloat(untypedScalarValue)) ? untypedScalarValue : parseFloat(untypedScalarValue) const coreTable = this.parent.coreTable - if (!columnName || !operator || untypedScalarValue === undefined) return coreTable + if (!columnName || !operator || (untypedScalarValue === undefined && !operator.includes("mpty"))) return coreTable const filterFn = row => { const atom = row[columnName] const typedAtom = atom === null ? undefined : atom // convert nulls to undefined @@ -380,8 +381,8 @@ scrollWhereParser else if (operator === "<") return typedAtom < typedValue else if (operator === ">=") return typedAtom >= typedValue else if (operator === "<=") return typedAtom <= typedValue - else if (operator === "empty") return atom === "" || atom === undefined - else if (operator === "notEmpty") return !(atom === "" || atom === undefined) + else if (operator === "empty") return typedAtom === "" || typedAtom === undefined + else if (operator === "notEmpty") return typedAtom !== "" && typedAtom !== undefined } return coreTable.filter(filterFn) } @@ -604,6 +605,21 @@ scrollOrderByParser return this.root.lodash.orderBy(this.parent.coreTable.slice(), orderBy[0], orderBy[1]) } +assertRowCountParser + extends abstractTableTransformParser + description Test row count is expected value. + atoms cueAtom integerAtom + cueFromId + javascript + getErrors() { + const errors = super.getErrors() + const actualRows = this.coreTable.length + const expectedRows = parseInt(this.content) + if (actualRows !== expectedRows) + return errors.concat(this.makeError(`Expected '${expectedRows}' rows but got '${actualRows}'.`)) + return errors + } + scrollRenameParser // todo: add support in Parsers for tuple catch alls catchAllAtomType columnNameAtom newColumnNameAtom diff --git a/releaseNotes.scroll b/releaseNotes.scroll index 9c16abde3..5e825955d 100644 --- a/releaseNotes.scroll +++ b/releaseNotes.scroll @@ -22,9 +22,17 @@ ciBadges.scroll br thinColumns +📦 164.11.0 12/29/2024 +🎉 added slashComment support to table flows +🎉 added assertRowCount parser for fast testing of table transformers +🏥 fix regression where empty and nonEmpty filters were not working +🏥 fix "first" reduction to select the first non-blank value +🏥 fix syntax highlighting in table flows + 📦 164.10.0 12/29/2024 🎉 added scrollModalParser 🎉 added click afterText parser +🎉 renamed `linkParser` to `scrollLinkParser` to clean up linkParser for userspace 📦 164.9.0 12/24/2024 🎉 added `belowAsHtml` and `aboveAsHtml` from btheado diff --git a/tests/tables.scroll b/tests/tables.scroll index 45ee963f4..7abac23a3 100644 --- a/tests/tables.scroll +++ b/tests/tables.scroll @@ -138,3 +138,18 @@ iris iris summarize printTable + +--- +# Empty and non empty +table + where name notEmpty + assertRowCount 2 + where score empty + assertRowCount 0 + where score notEmpty + assertRowCount 3 + data + name,score + ,1 + joe,2 + frank,4