Better table transforms

package.json

@@ -1,6 +1,6 @@
 {
   "name": "scroll-cli",
-  "version": "164.10.0",
+  "version": "164.11.0",
   "description": "A language for scientists of all ages. A curated collection of tools for refining and sharing thoughts.",
   "main": "scroll.js",
   "engines": {

parsers/groupBy.parsers

@@ -86,12 +86,13 @@ scrollGroupByParser
          colsToReduce.forEach((col) => {
            const sourceColName = col.source
            const reduction = col.reduction
+           const newColName = col.name
            if (reduction === "concat") {
-              newRow[col.name] = group.map((row) => row[sourceColName]).join(" ")
+              newRow[newColName] = group.map((row) => row[sourceColName]).join(" ")
               return 
            }
            if (reduction === "first") {
-              newRow[col.name] = group[0][sourceColName]
+              newRow[newColName] = group.find((row) => row[sourceColName] !== "")?.[sourceColName]
               return 
            }
            const values = group.map((row) => row[sourceColName]).filter((val) => typeof val === "number" && !isNaN(val))
@@ -100,7 +101,7 @@ scrollGroupByParser
            if (reduction === "max") reducedValue = Math.max(...values)
            if (reduction === "min") reducedValue = Math.min(...values)
            if (reduction === "mean") reducedValue = values.reduce((prev, current) => prev + current, 0) / values.length
-           newRow[col.name] = reducedValue
+           newRow[newColName] = reducedValue
          })
          rows.push(newRow)
        }

parsers/root.parsers

@@ -346,7 +346,7 @@ scrollParser
   }
   get scrollVersion() {
     // currently manually updated
-    return "164.10.0"
+    return "164.11.0"
   }
   // Use the first paragraph for the description
   // todo: add a particle method version of get that gets you the first particle. (actulaly make get return array?)

parsers/tables.parsers

@@ -1,6 +1,6 @@
 abstractTableTransformParser
  atoms cueAtom
- inScope abstractTableVisualizationParser abstractTableTransformParser h1Parser h2Parser scrollQuestionParser htmlInlineParser scrollBrParser
+ inScope abstractTableVisualizationParser abstractTableTransformParser h1Parser h2Parser scrollQuestionParser htmlInlineParser scrollBrParser slashCommentParser
  javascript
   get coreTable() {
     return this.parent.coreTable
@@ -216,7 +216,7 @@ scrollTableDelimiterParser
 abstractDatatableProviderParser
  description A datatable.
  extends abstractScrollParser
- inScope scrollTableDataParser scrollTableDelimiterParser abstractTableVisualizationParser abstractTableTransformParser h1Parser h2Parser scrollQuestionParser htmlInlineParser scrollBrParser
+ inScope scrollTableDataParser scrollTableDelimiterParser abstractTableVisualizationParser abstractTableTransformParser h1Parser h2Parser scrollQuestionParser htmlInlineParser scrollBrParser slashCommentParser
  javascript
   get visualizations() {
     return this.topDownArray.filter(particle => particle.isTableVisualization || particle.isHeader || particle.isHtml)
@@ -354,7 +354,8 @@ scrollWhereParser
  extends abstractTableTransformParser
  description Filter rows by condition.
  cue where
- atoms cueAtom columnNameAtom comparisonAtom constantAtom
+ atoms cueAtom columnNameAtom comparisonAtom
+ catchAllAtomType constantAtom
  example
   table iris.csv
    where Species = setosa
@@ -366,7 +367,7 @@ scrollWhereParser
    let untypedScalarValue = this.getAtom(3)
    const typedValue = isNaN(parseFloat(untypedScalarValue)) ? untypedScalarValue : parseFloat(untypedScalarValue)
    const coreTable = this.parent.coreTable
-   if (!columnName || !operator || untypedScalarValue === undefined) return coreTable
+   if (!columnName || !operator || (untypedScalarValue === undefined && !operator.includes("mpty"))) return coreTable
     const filterFn = row => {
       const atom = row[columnName]
       const typedAtom = atom === null ? undefined : atom // convert nulls to undefined
@@ -380,8 +381,8 @@ scrollWhereParser
       else if (operator === "<") return typedAtom < typedValue
       else if (operator === ">=") return typedAtom >= typedValue
       else if (operator === "<=") return typedAtom <= typedValue
-      else if (operator === "empty") return atom === "" || atom === undefined
-      else if (operator === "notEmpty") return !(atom === "" || atom === undefined)
+      else if (operator === "empty") return typedAtom === "" || typedAtom === undefined
+      else if (operator === "notEmpty") return typedAtom !== "" && typedAtom !== undefined
     }
    return coreTable.filter(filterFn)
   }
@@ -604,6 +605,21 @@ scrollOrderByParser
    return this.root.lodash.orderBy(this.parent.coreTable.slice(), orderBy[0], orderBy[1])
   }
 
+assertRowCountParser
+ extends abstractTableTransformParser
+ description Test row count is expected value.
+ atoms cueAtom integerAtom
+ cueFromId
+ javascript
+  getErrors() {
+    const errors = super.getErrors()
+    const actualRows = this.coreTable.length
+    const expectedRows = parseInt(this.content)
+    if (actualRows !== expectedRows)
+      return errors.concat(this.makeError(`Expected '${expectedRows}' rows but got '${actualRows}'.`))
+    return errors
+  }
+
 scrollRenameParser
  // todo: add support in Parsers for tuple catch alls
   catchAllAtomType columnNameAtom newColumnNameAtom

releaseNotes.scroll

@@ -22,9 +22,17 @@ ciBadges.scroll
 br
 thinColumns
 
+📦 164.11.0 12/29/2024
+🎉 added slashComment support to table flows
+🎉 added assertRowCount parser for fast testing of table transformers
+🏥 fix regression where empty and nonEmpty filters were not working
+🏥 fix "first" reduction to select the first non-blank value
+🏥 fix syntax highlighting in table flows
+
 📦 164.10.0 12/29/2024
 🎉 added scrollModalParser
 🎉 added click afterText parser
+🎉 renamed `linkParser` to `scrollLinkParser` to clean up linkParser for userspace
 
 📦 164.9.0 12/24/2024
 🎉 added `belowAsHtml` and `aboveAsHtml` from btheado

tests/tables.scroll

@@ -138,3 +138,18 @@ iris
 iris
  summarize
   printTable
+
+---
+# Empty and non empty
+table
+ where name notEmpty
+  assertRowCount 2
+ where score empty
+  assertRowCount 0
+ where score notEmpty
+  assertRowCount 3
+ data
+  name,score
+  ,1
+  joe,2
+  frank,4