OpenFn · josephjclark · Jul 14, 2023 · Jul 14, 2023 · Jul 14, 2023 · Jul 14, 2023
diff --git a/packages/bigquery/CHANGELOG.md b/packages/bigquery/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/language-bigquery
 
+## 2.0.1
+
+### Patch Changes
+
+- Updated dependencies [26a303e]
+  - @openfn/[email protected]
+
 ## 2.0.0
 
 ### Major Changes

diff --git a/packages/bigquery/package.json b/packages/bigquery/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-bigquery",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "description": "A Google BigQuery language package for use with Open Function",
   "main": "dist/index.cjs",
   "scripts": {
@@ -21,7 +21,7 @@
   ],
   "dependencies": {
     "@google-cloud/bigquery": "^5.12.0",
-    "@openfn/language-common": "workspace:1.10.1",
+    "@openfn/language-common": "workspace:1.10.2",
     "csv-parse": "^4.16.3",
     "import": "0.0.6",
     "json2csv": "^5.0.7",

diff --git a/packages/commcare/CHANGELOG.md b/packages/commcare/CHANGELOG.md
@@ -1,5 +1,13 @@
 # @openfn/language-commcare
 
+## 1.6.5
+
+### Patch Changes
+
+- Updated dependencies [26a303e]
+  - @openfn/[email protected]
+  - @openfn/[email protected]
+
 ## 1.6.4
 
 ### Patch Changes

diff --git a/packages/commcare/package.json b/packages/commcare/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-commcare",
-  "version": "1.6.4",
+  "version": "1.6.5",
   "description": "Commcare Language Pack for OpenFn",
   "homepage": "https://docs.openfn.org",
   "repository": {
@@ -25,8 +25,8 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "workspace:1.10.1",
-    "@openfn/language-http": "workspace:^5.0.0",
+    "@openfn/language-common": "workspace:1.10.2",
+    "@openfn/language-http": "workspace:^5.0.1",
     "JSONPath": "^0.10.0",
     "form-data": "^4.0.0",
     "js2xmlparser": "^1.0.0",

diff --git a/packages/common/CHANGELOG.md b/packages/common/CHANGELOG.md
@@ -1,5 +1,11 @@
 v0.4.0
 
+## 1.10.2
+
+### Patch Changes
+
+- 26a303e: add expandReferences for csvData and parsingOptions
+
 ## 1.10.1
 
 ### Patch Changes

diff --git a/packages/common/package.json b/packages/common/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-common",
-  "version": "1.10.1",
+  "version": "1.10.2",
   "description": "Common Expressions for OpenFn",
   "homepage": "https://docs.openfn.org",
   "repository": {

diff --git a/packages/common/src/Adaptor.js b/packages/common/src/Adaptor.js
@@ -4,6 +4,8 @@ import fromPairs from 'lodash/fp/fromPairs.js';
 import { JSONPath } from 'jsonpath-plus';
 import { parse } from 'csv-parse';
 
+import { expandReferences as newExpandReferences } from './util';
+
 export * as beta from './beta';
 export * as http from './http';
 export * as dateFns from './dateFns';
@@ -588,23 +590,31 @@ export function parseCsv(csvData, parsingOptions = {}, callback) {
     skip_empty_lines: true,
   };
 
-  const filteredOptions = Object.fromEntries(
-    Object.entries(parsingOptions).filter(([key]) => key in defaultOptions)
-  );
+  return async state => {
+    const [resolvedCsvData, resolvedParsingOptions] = newExpandReferences(
+      state,
+      csvData,
+      parsingOptions
+    );
 
-  const options = { ...defaultOptions, ...filteredOptions };
+    const filteredOptions = Object.fromEntries(
+      Object.entries(resolvedParsingOptions).filter(
+        ([key]) => key in defaultOptions
+      )
+    );
 
-  if (options.chunkSize < 1) {
-    throw new Error('chunkSize must be at least 1');
-  }
+    const options = { ...defaultOptions, ...filteredOptions };
+
+    if (options.chunkSize < 1) {
+      throw new Error('chunkSize must be at least 1');
+    }
 
-  return async state => {
     let buffer = [];
 
     const parser =
-      typeof csvData === 'string'
-        ? parse(csvData, options)
-        : csvData.pipe(parse(options));
+      typeof resolvedCsvData === 'string'
+        ? parse(resolvedCsvData, options)
+        : resolvedCsvData.pipe(parse(options));
 
     const flushBuffer = async currentState => {
       const nextState = callback

diff --git a/packages/dynamics/CHANGELOG.md b/packages/dynamics/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/language-dynamics
 
+## 0.4.5
+
+### Patch Changes
+
+- Updated dependencies [26a303e]
+  - @openfn/[email protected]
+
 ## 0.4.4
 
 ### Patch Changes

diff --git a/packages/dynamics/package.json b/packages/dynamics/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-dynamics",
-  "version": "0.4.4",
+  "version": "0.4.5",
   "description": "A Microsoft Dynamics Language Pack for OpenFn",
   "main": "dist/index.cjs",
   "scripts": {
@@ -20,7 +20,7 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "1.10.1",
+    "@openfn/language-common": "1.10.2",
     "request": "^2.72.0"
   },
   "devDependencies": {

diff --git a/packages/http/CHANGELOG.md b/packages/http/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/language-http
 
+## 5.0.1
+
+### Patch Changes
+
+- Updated dependencies [26a303e]
+  - @openfn/[email protected]
+
 ## 5.0.0
 
 ### Major Changes

diff --git a/packages/http/package.json b/packages/http/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-http",
-  "version": "5.0.0",
+  "version": "5.0.1",
   "description": "An HTTP request language package for use with Open Function",
   "homepage": "https://docs.openfn.org",
   "repository": {
@@ -33,7 +33,7 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "workspace:1.10.1",
+    "@openfn/language-common": "workspace:1.10.2",
     "cheerio": "^1.0.0-rc.10",
     "cheerio-tableparser": "^1.0.1",
     "csv-parse": "^4.10.1",

diff --git a/packages/mssql/CHANGELOG.md b/packages/mssql/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/language-mssql
 
+## 4.1.5
+
+### Patch Changes
+
+- Updated dependencies [26a303e]
+  - @openfn/[email protected]
+
 ## 4.1.4
 
 ### Patch Changes

diff --git a/packages/mssql/package.json b/packages/mssql/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-mssql",
-  "version": "4.1.4",
+  "version": "4.1.5",
   "description": "A Microsoft SQL language pack for OpenFn",
   "exports": {
     ".": {
@@ -31,7 +31,7 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "workspace:1.10.1",
+    "@openfn/language-common": "workspace:1.10.2",
     "tedious": "15.1.0"
   },
   "devDependencies": {

diff --git a/packages/mysql/CHANGELOG.md b/packages/mysql/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/language-mysql
 
+## 1.4.5
+
+### Patch Changes
+
+- Updated dependencies [26a303e]
+  - @openfn/[email protected]
+
 ## 1.4.4
 
 ### Patch Changes

diff --git a/packages/mysql/package.json b/packages/mysql/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-mysql",
-  "version": "1.4.4",
+  "version": "1.4.5",
   "description": "A MySQL Language Pack for OpenFn",
   "homepage": "https://docs.openfn.org",
   "main": "dist/index.cjs",
@@ -25,7 +25,7 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "1.10.1",
+    "@openfn/language-common": "1.10.2",
     "json-sql": "^0.3.10",
     "mysql": "^2.13.0",
     "squel": "^5.8.0",

diff --git a/packages/ocl/CHANGELOG.md b/packages/ocl/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/language-ocl
 
+## 1.1.3
+
+### Patch Changes
+
+- Updated dependencies [26a303e]
+  - @openfn/[email protected]
+
 ## 1.1.2
 
 ### Patch Changes

diff --git a/packages/ocl/package.json b/packages/ocl/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-ocl",
-  "version": "1.1.2",
+  "version": "1.1.3",
   "description": "An OCL language package for use with Open Function",
   "main": "dist/index.cjs",
   "scripts": {
@@ -20,7 +20,7 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "1.10.1"
+    "@openfn/language-common": "1.10.2"
   },
   "devDependencies": {
     "@openfn/buildtools": "workspace:^1.0.2",

diff --git a/packages/openfn/CHANGELOG.md b/packages/openfn/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/language-openfn
 
+## 1.3.5
+
+### Patch Changes
+
+- Updated dependencies [26a303e]
+  - @openfn/[email protected]
+
 ## 1.3.4
 
 ### Patch Changes

diff --git a/packages/openfn/package.json b/packages/openfn/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-openfn",
-  "version": "1.3.4",
+  "version": "1.3.5",
   "description": "An (experimental) adaptor for accessing the OpenFn web API",
   "homepage": "https://docs.openfn.org",
   "repository": {
@@ -25,7 +25,7 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "1.10.1",
+    "@openfn/language-common": "1.10.2",
     "axios": "^0.21.1"
   },
   "devDependencies": {

diff --git a/packages/openmrs/CHANGELOG.md b/packages/openmrs/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/language-openmrs
 
+## 2.0.5
+
+### Patch Changes
+
+- Updated dependencies [26a303e]
+  - @openfn/[email protected]
+
 ## 2.0.4
 
 ### Patch Changes

diff --git a/packages/openmrs/package.json b/packages/openmrs/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-openmrs",
-  "version": "2.0.4",
+  "version": "2.0.5",
   "description": "OpenMRS Language Pack for OpenFn",
   "homepage": "https://docs.openfn.org",
   "repository": {
@@ -25,7 +25,7 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "1.10.1",
+    "@openfn/language-common": "1.10.2",
     "superagent": "^8.0.9"
   },
   "devDependencies": {

diff --git a/packages/postgresql/CHANGELOG.md b/packages/postgresql/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/language-postgresql
 
+## 4.1.5
+
+### Patch Changes
+
+- Updated dependencies [26a303e]
+  - @openfn/[email protected]
+
 ## 4.1.4
 
 ### Patch Changes

diff --git a/packages/postgresql/package.json b/packages/postgresql/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@openfn/language-postgresql",
-  "version": "4.1.4",
+  "version": "4.1.5",
   "description": "A PostgreSQL Language Pack for OpenFn",
   "homepage": "https://docs.openfn.org",
   "exports": {
@@ -31,7 +31,7 @@
     "configuration-schema.json"
   ],
   "dependencies": {
-    "@openfn/language-common": "1.10.1",
+    "@openfn/language-common": "1.10.2",
     "pg": "^8.3.2",
     "pg-format": "^1.0.4"
   },

diff --git a/packages/primero/CHANGELOG.md b/packages/primero/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @openfn/language-primero
 
+## 2.11.5
+
+### Patch Changes
+
+- Updated dependencies [26a303e]
+  - @openfn/[email protected]
+
 ## 2.11.4
 
 ### Patch Changes