diff --git a/docs/sql.md b/docs/sql.md index 0c990713a..e87d1f26d 100644 --- a/docs/sql.md +++ b/docs/sql.md @@ -16,6 +16,17 @@ sql: --- ``` +To load externally-hosted data, you can use a full URL: + +```yaml +--- +sql: + quakes: https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_day.csv +--- +``` + +
For performance and reliability, we recommend using local files rather than loading data from external servers at runtime. If needed, you can use a data loader to take a snapshot of a remote data during build.
+ ## SQL code blocks To run SQL queries, create a SQL fenced code block (```sql). For example, to query the first 10 rows from the `gaia` table: diff --git a/src/client/preview.js b/src/client/preview.js index 73b4fa8d8..6bdb484c7 100644 --- a/src/client/preview.js +++ b/src/client/preview.js @@ -89,7 +89,7 @@ export function open({hash, eval: compile} = {}) { registerTable(name, null); } for (const table of message.tables.added) { - registerTable(table.name, FileAttachment(table.path)); + registerTable(table.name, !/^(\w+:|#)/.test(table.path) ? FileAttachment(table.path) : table.path); } } if (message.tables.removed.length || message.tables.added.length) { diff --git a/src/client/stdlib/duckdb.js b/src/client/stdlib/duckdb.js index c4d6093ac..711bdb301 100644 --- a/src/client/stdlib/duckdb.js +++ b/src/client/stdlib/duckdb.js @@ -181,31 +181,33 @@ Object.defineProperty(DuckDBClient.prototype, "dialect", { async function insertSource(database, name, source) { source = await source; - if (isFileAttachment(source)) { - // bare file - await insertFile(database, name, source); - } else if (isArrowTable(source)) { - // bare arrow table - await insertArrowTable(database, name, source); - } else if (Array.isArray(source)) { - // bare array of objects - await insertArray(database, name, source); - } else if (isArqueroTable(source)) { - await insertArqueroTable(database, name, source); - } else if ("data" in source) { - // data + options - const {data, ...options} = source; - if (isArrowTable(data)) { - await insertArrowTable(database, name, data, options); - } else { - await insertArray(database, name, data, options); + if (isFileAttachment(source)) return insertFile(database, name, source); + if (isArrowTable(source)) return insertArrowTable(database, name, source); + if (Array.isArray(source)) return insertArray(database, name, source); + if (isArqueroTable(source)) return insertArqueroTable(database, name, source); + if (typeof source === "string") return insertUrl(database, name, source); + if (source && typeof source === "object") { + if ("data" in source) { + // data + options + const {data, ...options} = source; + if (isArrowTable(data)) return insertArrowTable(database, name, data, options); + return insertArray(database, name, data, options); } - } else if ("file" in source) { - // file + options - const {file, ...options} = source; - await insertFile(database, name, file, options); - } else { - throw new Error(`invalid source: ${source}`); + if ("file" in source) { + // file + options + const {file, ...options} = source; + return insertFile(database, name, file, options); + } + } + throw new Error(`invalid source: ${source}`); +} + +async function insertUrl(database, name, url) { + const connection = await database.connect(); + try { + await connection.query(`CREATE VIEW '${name}' AS FROM '${url}'`); + } finally { + await connection.close(); } } diff --git a/src/javascript/transpile.ts b/src/javascript/transpile.ts index 311eededa..d54a1af55 100644 --- a/src/javascript/transpile.ts +++ b/src/javascript/transpile.ts @@ -2,7 +2,7 @@ import {join} from "node:path/posix"; import type {CallExpression, Node} from "acorn"; import type {ImportDeclaration, ImportDefaultSpecifier, ImportNamespaceSpecifier, ImportSpecifier} from "acorn"; import {simple} from "acorn-walk"; -import {isPathImport, relativePath, resolvePath} from "../path.js"; +import {isPathImport, relativePath, resolvePath, resolveRelativePath} from "../path.js"; import {getModuleResolver} from "../resolvers.js"; import {Sourcemap} from "../sourcemap.js"; import type {FileExpression} from "./files.js"; @@ -106,7 +106,7 @@ export async function transpileModule( function rewriteFileExpressions(output: Sourcemap, files: FileExpression[], path: string): void { for (const {name, node} of files) { const source = node.arguments[0]; - const resolved = relativePath(path, resolvePath(path, name)); + const resolved = resolveRelativePath(path, name); output.replaceLeft(source.start, source.end, JSON.stringify(resolved)); } } diff --git a/src/path.ts b/src/path.ts index cf4d86279..798181ef8 100644 --- a/src/path.ts +++ b/src/path.ts @@ -65,3 +65,10 @@ export function isPathImport(specifier: string): boolean { export function isAssetPath(specifier: string): boolean { return !/^(\w+:|#)/.test(specifier); } + +/** + * Returns the relative path to the specified target from the given source. + */ +export function resolveRelativePath(source: string, target: string): string { + return relativePath(source, resolvePath(source, target)); +} diff --git a/src/render.ts b/src/render.ts index 625967c83..447c9947a 100644 --- a/src/render.ts +++ b/src/render.ts @@ -8,7 +8,7 @@ import {transpileJavaScript} from "./javascript/transpile.js"; import type {MarkdownPage} from "./markdown.js"; import type {PageLink} from "./pager.js"; import {findLink, normalizePath} from "./pager.js"; -import {relativePath, resolvePath} from "./path.js"; +import {isAssetPath, relativePath, resolvePath, resolveRelativePath} from "./path.js"; import type {Resolvers} from "./resolvers.js"; import {getResolvers} from "./resolvers.js"; import {rollupClient} from "./rollup.js"; @@ -64,7 +64,7 @@ import ${preview || page.code.length ? `{${preview ? "open, " : ""}define} from : "" }${data?.sql ? `\nimport {registerTable} from ${JSON.stringify(resolveImport("npm:@observablehq/duckdb"))};` : ""}${ files.size - ? `\n${renderFiles( + ? `\n${registerFiles( files, resolveFile, preview @@ -72,13 +72,7 @@ import ${preview || page.code.length ? `{${preview ? "open, " : ""}define} from : (name) => loaders.getOutputLastModified(resolvePath(path, name)) )}` : "" - }${ - data?.sql - ? `\n${Object.entries(data.sql) - .map(([name, source]) => `registerTable(${JSON.stringify(name)}, FileAttachment(${JSON.stringify(source)}));`) - .join("\n")}` - : "" - } + }${data?.sql ? `\n${registerTables(data.sql, options)}` : ""} ${preview ? `\nopen({hash: ${JSON.stringify(resolvers.hash)}, eval: (body) => eval(body)});\n` : ""}${page.code .map(({node, id}) => `\n${transpileJavaScript(node, {id, path, resolveImport})}`) .join("")}`)} @@ -92,18 +86,32 @@ ${html.unsafe(rewriteHtml(page.body, resolvers))}${renderFooter(page.foot `); } -function renderFiles( +function registerTables(sql: Record, options: RenderOptions): string { + return Object.entries(sql) + .map(([name, source]) => registerTable(name, source, options)) + .join("\n"); +} + +function registerTable(name: string, source: any, {path}: RenderOptions): string { + return `registerTable(${JSON.stringify(name)}, ${ + isAssetPath(source) + ? `FileAttachment(${JSON.stringify(resolveRelativePath(path, source))})` + : JSON.stringify(source) + });`; +} + +function registerFiles( files: Iterable, resolve: (name: string) => string, getLastModified: (name: string) => number | undefined ): string { return Array.from(files) .sort() - .map((f) => renderFile(f, resolve, getLastModified)) + .map((f) => registerFile(f, resolve, getLastModified)) .join(""); } -function renderFile( +function registerFile( name: string, resolve: (name: string) => string, getLastModified: (name: string) => number | undefined