Merge pull request #32 from data-depot/staging

chore(csv-support): csv support release
data-depo · Jun 15, 2020 · 2eb5dcf · 2eb5dcf
2 parents a00c6ec + bcfbfdc
commit 2eb5dcf
Show file tree

Hide file tree

Showing 6 changed files with 140 additions and 34 deletions.
diff --git a/package.json b/package.json
@@ -90,6 +90,7 @@
   },
   "dependencies": {
     "camelcase-keys": "^6.2.2",
+    "csv-parse": "^4.10.1",
     "debug": "^4.1.1",
     "dotenv": "^8.2.0",
     "got": "^11.0.2",

diff --git a/src/manager.ts b/src/manager.ts
@@ -6,7 +6,7 @@ import { Subject, from, Observable, defer } from 'rxjs'
 // local
 import { AuthOpts, Query } from './types'
 import { limit, offset } from './clauses'
-import { createRunner } from './runner'
+import { createJsonRunner } from './runner'
 
 const logger = debug('soda-ts:manager')
 
@@ -91,7 +91,7 @@ export const createManagerCreator = <T>(
     offset(paginationOpts.offset)
   )(query)
 
-  const runner = createRunner<T[]>(opts.authOpts)
+  const runner = createJsonRunner<T[]>(opts.authOpts)
 
   const run = (): Promise<T[]> => {
     return runner(paginatedQuery)

diff --git a/src/runner.ts b/src/runner.ts
@@ -4,7 +4,7 @@ import got from 'got'
 import debug from 'debug'
 import { defer, from } from 'rxjs'
 import camelCaseKeys from 'camelcase-keys'
-
+import parser from 'csv-parse'
 // local
 import { Query, AuthOpts } from './types'
 import { queryClauseTransformer } from './clauses'
@@ -36,15 +36,13 @@ const logger = debug('soda-ts:runner')
  *  runner
  * )(query)
  */
-export const createRunner = <T>({
+export const createRawRunner = ({
   appToken,
-  keysCamelCased,
   ext = 'json'
-}: AuthOpts = {}) => async (query: Query): Promise<T> => {
+}: AuthOpts = {}) => async (
+  query: Query
+): Promise<string> => {
   const url = `https://${query.domain}/${query.apiPath}/${query.src}.${ext}`
-
-  logger(`making req to url: ${url}`)
-
   // TODO: refactor: cleanup param generation into a fn
   // use pure fn to generate the search params
   const clauseParams = queryClauseTransformer(query.clauses)
@@ -70,21 +68,34 @@ export const createRunner = <T>({
     `making req with headers: ${JSON.stringify(headers)}`
   )
 
+  const options = {
+    headers,
+    searchParams
+  }
+
   try {
-    const res = await got
-      .get(url, {
-        headers,
-        searchParams
-        // hooks: {
-        //   beforeRequest: [(options) => logger(options)]
-        // }
-      })
-      .json<T>()
+    const res = await got.get(url, options)
+    return res.body
+  } catch (e) {
+    throw new Error(e.message)
+  }
+}
 
+export const createJsonRunner = <T>({
+  appToken,
+  keysCamelCased
+}: AuthOpts = {}) => async (query: Query): Promise<T> => {
+  try {
+    const res = await createRawRunner({
+      appToken,
+      keysCamelCased,
+      ext: 'json'
+    })(query)
+    const data: T = JSON.parse(res)
     if (keysCamelCased) {
-      return camelCaseKeys(res)
+      return camelCaseKeys(data)
     } else {
-      return res
+      return data
     }
     // return res
   } catch (e) {
@@ -99,4 +110,30 @@ export const createRunner = <T>({
  */
 export const createRunner$ = <T>(authOpts?: AuthOpts) => (
   query: Query
-) => defer(() => from(createRunner<T>(authOpts)(query)))
+) => defer(() => from(createJsonRunner<T>(authOpts)(query)))
+
+export const createCsvRunner = ({
+  appToken
+}: AuthOpts = {}) => async (
+  query: Query
+): Promise<string[][]> => {
+  const output: Array<[]> = []
+  const res = await createRawRunner({
+    appToken,
+    ext: 'csv'
+  })(query)
+  const parse = parser({
+    delimiter: ','
+  })
+  parse.on('readable', function () {
+    let record: []
+    while ((record = parse.read())) {
+      output.push(record)
+    }
+  })
+  const stringArr = res
+  const resBody: string[] = stringArr.split(' ')
+  resBody.forEach((a) => parse.write(a))
+  parse.end()
+  return output
+}
diff --git a/src/types.ts b/src/types.ts
@@ -23,13 +23,12 @@ export interface Query {
   /** rest path associated with api and domain */
   apiPath: 'resource' | 'api/views' | 'api/catalog/v1'
 }
-
 export interface AuthOpts {
   // apiToken?: string
   /** appToken userd to make authenticated req */
   appToken?: string
   /** whether to serialize response keys to camelCase */
   keysCamelCased?: boolean
   /** where to use json or csv files */
-  ext?: 'json' | 'csv'
+  ext?: string
 }
diff --git a/test/runner.test.ts b/test/runner.test.ts
@@ -1,5 +1,7 @@
 import {
-  createRunner,
+  createRawRunner,
+  createJsonRunner,
+  createCsvRunner,
   createQuery,
   Query,
   where,
@@ -13,18 +15,18 @@ import { RAW_DATA, RawData } from './fixtures'
 
 describe('createRunner', () => {
   let query: Query
-  let runner: ReturnType<typeof createRunner>
+  let runner: ReturnType<typeof createRawRunner>
 
   const authOpts = {
     appToken: process.env.APP_TOKEN
   }
 
-  let authenticatedRunner: ReturnType<typeof createRunner>
+  let authenticatedRunner: ReturnType<typeof createRawRunner>
 
   beforeAll(() => {
     query = createQuery({ src: 'w7w3-xahh' })
-    runner = createRunner()
-    authenticatedRunner = createRunner(authOpts)
+    runner = createRawRunner()
+    authenticatedRunner = createRawRunner(authOpts)
   })
 
   it('successfully grabs ', async () => {
@@ -47,7 +49,7 @@ describe('createRunner', () => {
         domain: 'soda.demo.socrata.com'
       })
 
-      const testRunner = createRunner<RawData[]>()
+      const testRunner = createJsonRunner<RawData[]>()
 
       const res = pipe(
         where('magnitude > 3.0'),
@@ -65,7 +67,7 @@ describe('createRunner', () => {
         domain: 'data.cityofnewyork.us'
       })
 
-      const testRunner = createRunner<RawData[]>()
+      const testRunner = createJsonRunner<RawData[]>()
 
       const res = pipe(
         where("license_nbr like '1232665-DCA'"),
@@ -83,7 +85,7 @@ describe('createRunner', () => {
         domain: 'data.cityofnewyork.us'
       })
 
-      const testRunner = createRunner<RawData[]>()
+      const testRunner = createJsonRunner<RawData[]>()
 
       const res = pipe(
         where`license_nbr = '1232665-DCA'`,
@@ -108,7 +110,9 @@ describe('createRunner', () => {
         domain: 'data.cityofnewyork.us'
       })
 
-      const testRunner = createRunner<RawData[]>(authOpts)
+      const testRunner = createJsonRunner<RawData[]>(
+        authOpts
+      )
 
       const res = pipe(
         where`license_creation_date between "${RAW_DATA.license_creation_date}" and "${dateToday}"`,
@@ -127,7 +131,9 @@ describe('createRunner', () => {
         domain: 'data.cityofnewyork.us'
       })
 
-      const testRunner = createRunner<RawData[]>(authOpts)
+      const testRunner = createJsonRunner<RawData[]>(
+        authOpts
+      )
 
       const res = pipe(
         limit('5'),
@@ -140,7 +146,7 @@ describe('createRunner', () => {
   })
 
   it('runner fails when invalid token provided', async () => {
-    const misAuthenticatedRunner = createRunner({
+    const misAuthenticatedRunner = createRawRunner({
       appToken: 'au7shiadu7fhdas7s'
     })
 
@@ -164,4 +170,62 @@ describe('createRunner', () => {
       }
     )
   })
+
+  it('json support', async () => {
+    const authOpts = {
+      appToken: process.env.App_Token,
+      keyCamelCased: false
+    }
+    query = createQuery({
+      src: 'w7w3-xahh',
+      domain: 'data.cityofnewyork.us',
+      apiPath: 'resource'
+    })
+    const testRunner = await createJsonRunner<RawData[]>(
+      authOpts
+    )
+    const res = await pipe(
+      limit('10'),
+      offset('0'),
+      testRunner
+    )(query)
+    expect(res.length).toBeGreaterThanOrEqual(5)
+  })
+
+  it('csv support', async () => {
+    const authOpts = {
+      appToken: process.env.App_Token
+    }
+    query = createQuery({
+      src: '3h2n-5cm9',
+      domain: 'data.cityofnewyork.us',
+      apiPath: 'resource'
+    })
+    const testRunner = await createCsvRunner(authOpts)
+    const res = await pipe(
+      limit('5'),
+      offset('0'),
+      testRunner
+    )(query)
+    expect(res).toBeTruthy()
+    expect(res.length).toBeGreaterThanOrEqual(5)
+  })
+
+  it('createRunnerRaw', async () => {
+    const authOpts = {
+      appToken: process.env.App_Token
+    }
+    query = createQuery({
+      src: '3h2n-5cm9',
+      domain: 'data.cityofnewyork.us',
+      apiPath: 'resource'
+    })
+    const testRunner = await createRawRunner(authOpts)
+    const res = await pipe(
+      limit('5'),
+      offset('0'),
+      testRunner
+    )(query)
+    expect(res).toBeTruthy()
+  })
 })
diff --git a/yarn.lock b/yarn.lock
@@ -2697,6 +2697,11 @@ cssstyle@^2.0.0:
   dependencies:
     cssom "~0.3.6"
 
+csv-parse@^4.10.1:
+  version "4.10.1"
+  resolved "https://registry.yarnpkg.com/csv-parse/-/csv-parse-4.10.1.tgz#1e26ba63d29c75e94d0eba6e9de9a8aaf89d72a6"
+  integrity sha512-gdDJVchi0oSLIcYXz1H/VSgLE6htHDqJyFsRU/vTkQgmVOZ3S0IR2LXnNbWUYG7VD76dYVwdfBLyx8AX9+An8A==
+
 currently-unhandled@^0.4.1:
   version "0.4.1"
   resolved "https://registry.yarnpkg.com/currently-unhandled/-/currently-unhandled-0.4.1.tgz#988df33feab191ef799a61369dd76c17adf957ea"