accept bare URL definitions for sql tables (#1098)

* accept bare URL definitions for sql tables:

---
sql:
  adresses: https://static.data.gouv.fr/resources/bureaux-de-vote-et-adresses-de-leurs-electeurs/20230626-135723/table-adresses-reu.parquet
  quakes: https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_day.csv
---

```sql echo
SELECT COUNT() FROM quakes;
```

closes #1071

* checkpoint url-based sources

* simplify

* remove live example

* VIEW instead of TABLE

* Update docs/sql.md

---------

Co-authored-by: Mike Bostock <[email protected]>

docs/sql.md

@@ -16,6 +16,17 @@ sql:
 ---
 ```
 
+To load externally-hosted data, you can use a full URL:
+
+```yaml
+---
+sql:
+  quakes: https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_day.csv
+---
+```
+
+<div class="tip">For performance and reliability, we recommend using local files rather than loading data from external servers at runtime. If needed, you can use a <a href="./loaders">data loader</a> to take a snapshot of a remote data during build.</div>
+
 ## SQL code blocks
 
 To run SQL queries, create a SQL fenced code block (<code>```sql</code>). For example, to query the first 10 rows from the `gaia` table:

src/client/preview.js

@@ -89,7 +89,7 @@ export function open({hash, eval: compile} = {}) {
             registerTable(name, null);
           }
           for (const table of message.tables.added) {
-            registerTable(table.name, FileAttachment(table.path));
+            registerTable(table.name, !/^(\w+:|#)/.test(table.path) ? FileAttachment(table.path) : table.path);
           }
         }
         if (message.tables.removed.length || message.tables.added.length) {

src/client/stdlib/duckdb.js

@@ -181,31 +181,33 @@ Object.defineProperty(DuckDBClient.prototype, "dialect", {
 
 async function insertSource(database, name, source) {
   source = await source;
-  if (isFileAttachment(source)) {
-    // bare file
-    await insertFile(database, name, source);
-  } else if (isArrowTable(source)) {
-    // bare arrow table
-    await insertArrowTable(database, name, source);
-  } else if (Array.isArray(source)) {
-    // bare array of objects
-    await insertArray(database, name, source);
-  } else if (isArqueroTable(source)) {
-    await insertArqueroTable(database, name, source);
-  } else if ("data" in source) {
-    // data + options
-    const {data, ...options} = source;
-    if (isArrowTable(data)) {
-      await insertArrowTable(database, name, data, options);
-    } else {
-      await insertArray(database, name, data, options);
+  if (isFileAttachment(source)) return insertFile(database, name, source);
+  if (isArrowTable(source)) return insertArrowTable(database, name, source);
+  if (Array.isArray(source)) return insertArray(database, name, source);
+  if (isArqueroTable(source)) return insertArqueroTable(database, name, source);
+  if (typeof source === "string") return insertUrl(database, name, source);
+  if (source && typeof source === "object") {
+    if ("data" in source) {
+      // data + options
+      const {data, ...options} = source;
+      if (isArrowTable(data)) return insertArrowTable(database, name, data, options);
+      return insertArray(database, name, data, options);
     }
-  } else if ("file" in source) {
-    // file + options
-    const {file, ...options} = source;
-    await insertFile(database, name, file, options);
-  } else {
-    throw new Error(`invalid source: ${source}`);
+    if ("file" in source) {
+      // file + options
+      const {file, ...options} = source;
+      return insertFile(database, name, file, options);
+    }
+  }
+  throw new Error(`invalid source: ${source}`);
+}
+
+async function insertUrl(database, name, url) {
+  const connection = await database.connect();
+  try {
+    await connection.query(`CREATE VIEW '${name}' AS FROM '${url}'`);
+  } finally {
+    await connection.close();
   }
 }
 

src/javascript/transpile.ts

@@ -2,7 +2,7 @@ import {join} from "node:path/posix";
 import type {CallExpression, Node} from "acorn";
 import type {ImportDeclaration, ImportDefaultSpecifier, ImportNamespaceSpecifier, ImportSpecifier} from "acorn";
 import {simple} from "acorn-walk";
-import {isPathImport, relativePath, resolvePath} from "../path.js";
+import {isPathImport, relativePath, resolvePath, resolveRelativePath} from "../path.js";
 import {getModuleResolver} from "../resolvers.js";
 import {Sourcemap} from "../sourcemap.js";
 import type {FileExpression} from "./files.js";
@@ -106,7 +106,7 @@ export async function transpileModule(
 function rewriteFileExpressions(output: Sourcemap, files: FileExpression[], path: string): void {
   for (const {name, node} of files) {
     const source = node.arguments[0];
-    const resolved = relativePath(path, resolvePath(path, name));
+    const resolved = resolveRelativePath(path, name);
     output.replaceLeft(source.start, source.end, JSON.stringify(resolved));
   }
 }

src/path.ts

@@ -65,3 +65,10 @@ export function isPathImport(specifier: string): boolean {
 export function isAssetPath(specifier: string): boolean {
   return !/^(\w+:|#)/.test(specifier);
 }
+
+/**
+ * Returns the relative path to the specified target from the given source.
+ */
+export function resolveRelativePath(source: string, target: string): string {
+  return relativePath(source, resolvePath(source, target));
+}

src/render.ts

@@ -8,7 +8,7 @@ import {transpileJavaScript} from "./javascript/transpile.js";
 import type {MarkdownPage} from "./markdown.js";
 import type {PageLink} from "./pager.js";
 import {findLink, normalizePath} from "./pager.js";
-import {relativePath, resolvePath} from "./path.js";
+import {isAssetPath, relativePath, resolvePath, resolveRelativePath} from "./path.js";
 import type {Resolvers} from "./resolvers.js";
 import {getResolvers} from "./resolvers.js";
 import {rollupClient} from "./rollup.js";
@@ -64,21 +64,15 @@ import ${preview || page.code.length ? `{${preview ? "open, " : ""}define} from
       : ""
   }${data?.sql ? `\nimport {registerTable} from ${JSON.stringify(resolveImport("npm:@observablehq/duckdb"))};` : ""}${
     files.size
-      ? `\n${renderFiles(
+      ? `\n${registerFiles(
           files,
           resolveFile,
           preview
             ? (name) => loaders.getSourceLastModified(resolvePath(path, name))
             : (name) => loaders.getOutputLastModified(resolvePath(path, name))
         )}`
       : ""
-  }${
-    data?.sql
-      ? `\n${Object.entries<string>(data.sql)
-          .map(([name, source]) => `registerTable(${JSON.stringify(name)}, FileAttachment(${JSON.stringify(source)}));`)
-          .join("\n")}`
-      : ""
-  }
+  }${data?.sql ? `\n${registerTables(data.sql, options)}` : ""}
 ${preview ? `\nopen({hash: ${JSON.stringify(resolvers.hash)}, eval: (body) => eval(body)});\n` : ""}${page.code
     .map(({node, id}) => `\n${transpileJavaScript(node, {id, path, resolveImport})}`)
     .join("")}`)}
@@ -92,18 +86,32 @@ ${html.unsafe(rewriteHtml(page.body, resolvers))}</main>${renderFooter(page.foot
 `);
 }
 
-function renderFiles(
+function registerTables(sql: Record<string, any>, options: RenderOptions): string {
+  return Object.entries(sql)
+    .map(([name, source]) => registerTable(name, source, options))
+    .join("\n");
+}
+
+function registerTable(name: string, source: any, {path}: RenderOptions): string {
+  return `registerTable(${JSON.stringify(name)}, ${
+    isAssetPath(source)
+      ? `FileAttachment(${JSON.stringify(resolveRelativePath(path, source))})`
+      : JSON.stringify(source)
+  });`;
+}
+
+function registerFiles(
   files: Iterable<string>,
   resolve: (name: string) => string,
   getLastModified: (name: string) => number | undefined
 ): string {
   return Array.from(files)
     .sort()
-    .map((f) => renderFile(f, resolve, getLastModified))
+    .map((f) => registerFile(f, resolve, getLastModified))
     .join("");
 }
 
-function renderFile(
+function registerFile(
   name: string,
   resolve: (name: string) => string,
   getLastModified: (name: string) => number | undefined