From ae0535d94580f14ef3338fb714280d25a1a2fba3 Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Sat, 22 May 2021 09:09:03 -0700 Subject: [PATCH 1/6] SQLite --- src/fileAttachment.js | 6 ++++ src/library.js | 8 +++-- src/md.js | 77 +++++++++++++++++++++---------------------- src/sqlite.js | 70 +++++++++++++++++++++++++++++++++++++++ src/tex.js | 36 ++++++++++---------- src/vegalite.js | 16 ++++----- test/index-test.js | 1 + 7 files changed, 143 insertions(+), 71 deletions(-) create mode 100644 src/sqlite.js diff --git a/src/fileAttachment.js b/src/fileAttachment.js index 5cbb2b59..a1da5bb9 100644 --- a/src/fileAttachment.js +++ b/src/fileAttachment.js @@ -1,4 +1,5 @@ import {require as requireDefault} from "d3-require"; +import sqlite, {SQLiteDatabaseClient} from "./sqlite.js"; async function remote_fetch(file) { const response = await fetch(await file.url()); @@ -56,6 +57,11 @@ class FileAttachment { i.src = url; }); } + async sqlite() { + const [SQL, buffer] = await Promise.all([sqlite(requireDefault), this.arrayBuffer()]); + const db = new SQL.Database(new Uint8Array(buffer)); + return new SQLiteDatabaseClient(db); + } } export function NoFileAttachments(name) { diff --git a/src/library.js b/src/library.js index 70d1e97a..6abe513c 100644 --- a/src/library.js +++ b/src/library.js @@ -10,6 +10,7 @@ import now from "./now.js"; import Promises from "./promises/index.js"; import resolve from "./resolve.js"; import requirer from "./require.js"; +import SQLite from "./sqlite.js"; import svg from "./svg.js"; import tex from "./tex.js"; import vegalite from "./vegalite.js"; @@ -26,18 +27,19 @@ export default Object.assign(function Library(resolver) { Mutable: () => Mutable, Plot: () => require("@observablehq/plot@0.1.0/dist/plot.umd.min.js"), Promises: () => Promises, + SQLite: () => SQLite(require), _: () => require("lodash@4.17.21/lodash.min.js"), d3: () => require("d3@6.7.0/dist/d3.min.js"), dot: () => require("@observablehq/graphviz@0.2.1/dist/graphviz.min.js"), htl: () => require("htl@0.2.5/dist/htl.min.js"), html: () => html, - md: md(require), + md: () => md(require), now: now, require: () => require, resolve: () => resolve, svg: () => svg, - tex: tex(require), - vl: vegalite(require), + tex: () => tex(require), + vl: () => vegalite(require), width: width })); }, {resolve: requireDefault.resolve}); diff --git a/src/md.js b/src/md.js index 027b7087..a09c0f99 100644 --- a/src/md.js +++ b/src/md.js @@ -1,47 +1,44 @@ import template from "./template.js"; -const HL_ROOT = - "https://cdn.jsdelivr.net/npm/@observablehq/highlight.js@2.0.0/"; +const HL_ROOT = "https://cdn.jsdelivr.net/npm/@observablehq/highlight.js@2.0.0/"; export default function(require) { - return function() { - return require("marked@0.3.12/marked.min.js").then(function(marked) { - return template( - function(string) { - var root = document.createElement("div"); - root.innerHTML = marked(string, {langPrefix: ""}).trim(); - var code = root.querySelectorAll("pre code[class]"); - if (code.length > 0) { - require(HL_ROOT + "highlight.min.js").then(function(hl) { - code.forEach(function(block) { - function done() { - hl.highlightBlock(block); - block.parentNode.classList.add("observablehq--md-pre"); - } - if (hl.getLanguage(block.className)) { - done(); - } else { - require(HL_ROOT + "async-languages/index.js") - .then(index => { - if (index.has(block.className)) { - return require(HL_ROOT + - "async-languages/" + - index.get(block.className)).then(language => { - hl.registerLanguage(block.className, language); - }); - } - }) - .then(done, done); - } - }); + return require("marked@0.3.12/marked.min.js").then(function(marked) { + return template( + function(string) { + var root = document.createElement("div"); + root.innerHTML = marked(string, {langPrefix: ""}).trim(); + var code = root.querySelectorAll("pre code[class]"); + if (code.length > 0) { + require(HL_ROOT + "highlight.min.js").then(function(hl) { + code.forEach(function(block) { + function done() { + hl.highlightBlock(block); + block.parentNode.classList.add("observablehq--md-pre"); + } + if (hl.getLanguage(block.className)) { + done(); + } else { + require(HL_ROOT + "async-languages/index.js") + .then(index => { + if (index.has(block.className)) { + return require(HL_ROOT + + "async-languages/" + + index.get(block.className)).then(language => { + hl.registerLanguage(block.className, language); + }); + } + }) + .then(done, done); + } }); - } - return root; - }, - function() { - return document.createElement("div"); + }); } - ); - }); - }; + return root; + }, + function() { + return document.createElement("div"); + } + ); + }); } diff --git a/src/sqlite.js b/src/sqlite.js new file mode 100644 index 00000000..5b91ac6c --- /dev/null +++ b/src/sqlite.js @@ -0,0 +1,70 @@ +import {require as requireDefault} from "d3-require"; + +export default async function sqlite(require) { + const sql = await require("sql.js@1.5.0/dist/sql-wasm.js"); + return sql({locateFile: file => `https://cdn.jsdelivr.net/npm/sql.js@1.5.0/dist/${file}`}); +} + +export class SQLiteDatabaseClient { + constructor(db) { + Object.defineProperties(this, { + _db: {value: db} + }); + } + async query(query, params) { + return await exec(this._db, query, params); + } + async queryRow(query, params) { + return (await this.query(query, params))[0]; + } + async explain(query, params) { + const rows = (await this.query(`EXPLAIN QUERY PLAN ${query}`, params)); + const text = rows.map(row => row.detail).join("\n"); + const pre = document.createElement("PRE"); + pre.className = "observablehq--inspect"; + pre.appendChild(document.createTextNode(text)); + return pre; + } + async describe(object) { + if (object !== undefined) { + const row = await this.queryRow(`SELECT * FROM '${object}' LIMIT 1`); + return table( + Object.entries(row).map(([column_name, value]) => ({ + column_name, + data_type: typeof value === "string" ? "character varying" + : typeof value === "number" ? "integer" + : undefined, + column_default: null, + is_nullable: "YES" + })) + ); + } else { + const rows = await this.query(`SELECT name FROM sqlite_master WHERE type = 'table'`); + return table(rows); + } + } +} + +async function exec(db, query, params) { + if (params !== undefined) { + let i = -1; + query = query.replace(/\?/g, () => { + const param = params[++i]; + return Array.isArray(param) + ? new Array(param.length).fill("?") + : "?"; + }); + params = params.flat(1); + } + const [result] = await db.exec(query, params); + if (!result) return []; + const {columns, values} = result; + const rows = values.map(row => Object.fromEntries(row.map((value, i) => [columns[i], value]))); + rows.columns = columns; + return rows; +} + +async function table(data, options) { + const Inputs = await requireDefault("@observablehq/inputs@0.8.0/dist/inputs.umd.min.js"); + return Inputs.table(data, options); +} diff --git a/src/tex.js b/src/tex.js index cb5bc007..c07cea43 100644 --- a/src/tex.js +++ b/src/tex.js @@ -11,25 +11,23 @@ function style(href) { }); } -export default function(require) { - return function() { - return Promise.all([ - require("@observablehq/katex@0.11.1/dist/katex.min.js"), - require.resolve("@observablehq/katex@0.11.1/dist/katex.min.css").then(style) - ]).then(function(values) { - var katex = values[0], tex = renderer(); +export default function tex(require) { + return Promise.all([ + require("@observablehq/katex@0.11.1/dist/katex.min.js"), + require.resolve("@observablehq/katex@0.11.1/dist/katex.min.css").then(style) + ]).then(function(values) { + var katex = values[0], tex = renderer(); - function renderer(options) { - return function() { - var root = document.createElement("div"); - katex.render(raw.apply(String, arguments), root, options); - return root.removeChild(root.firstChild); - }; - } + function renderer(options) { + return function() { + var root = document.createElement("div"); + katex.render(raw.apply(String, arguments), root, options); + return root.removeChild(root.firstChild); + }; + } - tex.options = renderer; - tex.block = renderer({displayMode: true}); - return tex; - }); - }; + tex.options = renderer; + tex.block = renderer({displayMode: true}); + return tex; + }); } diff --git a/src/vegalite.js b/src/vegalite.js index 96ddc914..652a77f1 100644 --- a/src/vegalite.js +++ b/src/vegalite.js @@ -1,10 +1,8 @@ -export default function vl(require) { - return async () => { - const [vega, vegalite, api] = await Promise.all([ - "vega@5.20.2/build/vega.min.js", - "vega-lite@5.1.0/build/vega-lite.min.js", - "vega-lite-api@5.0.0/build/vega-lite-api.min.js" - ].map(module => require(module))); - return api.register(vega, vegalite); - }; +export default async function vl(require) { + const [vega, vegalite, api] = await Promise.all([ + "vega@5.20.2/build/vega.min.js", + "vega-lite@5.1.0/build/vega-lite.min.js", + "vega-lite-api@5.0.0/build/vega-lite-api.min.js" + ].map(module => require(module))); + return api.register(vega, vegalite); } diff --git a/test/index-test.js b/test/index-test.js index 8ff32a60..006bd53e 100644 --- a/test/index-test.js +++ b/test/index-test.js @@ -11,6 +11,7 @@ test("new Library returns a library with the expected keys", async t => { "Mutable", "Plot", "Promises", + "SQLite", "_", "d3", "dot", From 2ca0124abdd9b73cfc396fd16a769b39a6b22342 Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Sat, 22 May 2021 13:32:36 -0700 Subject: [PATCH 2/6] return null from queryRow --- src/sqlite.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlite.js b/src/sqlite.js index 5b91ac6c..3e0546d9 100644 --- a/src/sqlite.js +++ b/src/sqlite.js @@ -15,7 +15,7 @@ export class SQLiteDatabaseClient { return await exec(this._db, query, params); } async queryRow(query, params) { - return (await this.query(query, params))[0]; + return (await this.query(query, params))[0] || null; } async explain(query, params) { const rows = (await this.query(`EXPLAIN QUERY PLAN ${query}`, params)); From 520b7fef0c28812d580c37e64edadb92b3fc54ff Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Sat, 22 May 2021 13:32:54 -0700 Subject: [PATCH 3/6] pass params as-is --- src/sqlite.js | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/sqlite.js b/src/sqlite.js index 3e0546d9..a750067f 100644 --- a/src/sqlite.js +++ b/src/sqlite.js @@ -46,16 +46,6 @@ export class SQLiteDatabaseClient { } async function exec(db, query, params) { - if (params !== undefined) { - let i = -1; - query = query.replace(/\?/g, () => { - const param = params[++i]; - return Array.isArray(param) - ? new Array(param.length).fill("?") - : "?"; - }); - params = params.flat(1); - } const [result] = await db.exec(query, params); if (!result) return []; const {columns, values} = result; From b3047f4426ac94c34a6134a1074091273fedc267 Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Sat, 22 May 2021 13:46:33 -0700 Subject: [PATCH 4/6] better describe --- src/sqlite.js | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/src/sqlite.js b/src/sqlite.js index a750067f..7e85238f 100644 --- a/src/sqlite.js +++ b/src/sqlite.js @@ -26,22 +26,11 @@ export class SQLiteDatabaseClient { return pre; } async describe(object) { - if (object !== undefined) { - const row = await this.queryRow(`SELECT * FROM '${object}' LIMIT 1`); - return table( - Object.entries(row).map(([column_name, value]) => ({ - column_name, - data_type: typeof value === "string" ? "character varying" - : typeof value === "number" ? "integer" - : undefined, - column_default: null, - is_nullable: "YES" - })) - ); - } else { - const rows = await this.query(`SELECT name FROM sqlite_master WHERE type = 'table'`); - return table(rows); - } + return table( + await (object === undefined + ? this.query(`SELECT name FROM sqlite_master WHERE type = 'table'`) + : this.query(`SELECT * FROM pragma_table_info(?)`, [object])) + ); } } From a4768f6d2bef930d282f8132ca2ceb94fa06e7c3 Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Mon, 24 May 2021 13:48:59 -0700 Subject: [PATCH 5/6] simpler table --- src/sqlite.js | 43 +++++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/src/sqlite.js b/src/sqlite.js index 7e85238f..24a92274 100644 --- a/src/sqlite.js +++ b/src/sqlite.js @@ -1,5 +1,3 @@ -import {require as requireDefault} from "d3-require"; - export default async function sqlite(require) { const sql = await require("sql.js@1.5.0/dist/sql-wasm.js"); return sql({locateFile: file => `https://cdn.jsdelivr.net/npm/sql.js@1.5.0/dist/${file}`}); @@ -18,15 +16,13 @@ export class SQLiteDatabaseClient { return (await this.query(query, params))[0] || null; } async explain(query, params) { - const rows = (await this.query(`EXPLAIN QUERY PLAN ${query}`, params)); - const text = rows.map(row => row.detail).join("\n"); - const pre = document.createElement("PRE"); - pre.className = "observablehq--inspect"; - pre.appendChild(document.createTextNode(text)); - return pre; + return formatText( + (await this.query(`EXPLAIN QUERY PLAN ${query}`, params)) + .map(row => row.detail).join("\n") + ); } async describe(object) { - return table( + return formatTable( await (object === undefined ? this.query(`SELECT name FROM sqlite_master WHERE type = 'table'`) : this.query(`SELECT * FROM pragma_table_info(?)`, [object])) @@ -43,7 +39,30 @@ async function exec(db, query, params) { return rows; } -async function table(data, options) { - const Inputs = await requireDefault("@observablehq/inputs@0.8.0/dist/inputs.umd.min.js"); - return Inputs.table(data, options); +function formatTable(rows, columns = rows.columns) { + if (!rows.length) throw new Error("Not found"); + const table = document.createElement("table"); + const thead = table.appendChild(document.createElement("thead")); + const tr = thead.appendChild(document.createElement("tr")); + for (const column of columns) { + const th = tr.appendChild(document.createElement("th")); + th.appendChild(document.createTextNode(column)); + } + const tbody = table.appendChild(document.createElement("tbody")); + for (const row of rows) { + const tr = tbody.appendChild(document.createElement("tr")); + for (const column of columns) { + const td = tr.appendChild(document.createElement("td")); + td.appendChild(document.createTextNode(row[column])); + } + } + table.value = rows; + return table; +} + +function formatText(text) { + const pre = document.createElement("pre"); + pre.className = "observablehq--inspect"; + pre.appendChild(document.createTextNode(text)); + return pre; } From 15a2fc67c6a8d733b55777c67d1b9027eb5cfaf7 Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Mon, 24 May 2021 14:17:58 -0700 Subject: [PATCH 6/6] hyperscript-ish --- src/sqlite.js | 54 ++++++++++++++++++++------------------------------- 1 file changed, 21 insertions(+), 33 deletions(-) diff --git a/src/sqlite.js b/src/sqlite.js index 24a92274..af36a68b 100644 --- a/src/sqlite.js +++ b/src/sqlite.js @@ -16,17 +16,21 @@ export class SQLiteDatabaseClient { return (await this.query(query, params))[0] || null; } async explain(query, params) { - return formatText( - (await this.query(`EXPLAIN QUERY PLAN ${query}`, params)) - .map(row => row.detail).join("\n") - ); + const rows = await this.query(`EXPLAIN QUERY PLAN ${query}`, params); + return element("pre", {className: "observablehq--inspect"}, [ + text(rows.map(row => row.detail).join("\n")) + ]); } async describe(object) { - return formatTable( - await (object === undefined - ? this.query(`SELECT name FROM sqlite_master WHERE type = 'table'`) - : this.query(`SELECT * FROM pragma_table_info(?)`, [object])) - ); + const rows = await (object === undefined + ? this.query(`SELECT name FROM sqlite_master WHERE type = 'table'`) + : this.query(`SELECT * FROM pragma_table_info(?)`, [object])); + if (!rows.length) throw new Error("Not found"); + const {columns} = rows; + return element("table", {value: rows}, [ + element("thead", [element("tr", columns.map(c => element("th", [text(c)])))]), + element("tbody", rows.map(r => element("tr", columns.map(c => element("td", [text(r[c])]))))) + ]); } } @@ -39,30 +43,14 @@ async function exec(db, query, params) { return rows; } -function formatTable(rows, columns = rows.columns) { - if (!rows.length) throw new Error("Not found"); - const table = document.createElement("table"); - const thead = table.appendChild(document.createElement("thead")); - const tr = thead.appendChild(document.createElement("tr")); - for (const column of columns) { - const th = tr.appendChild(document.createElement("th")); - th.appendChild(document.createTextNode(column)); - } - const tbody = table.appendChild(document.createElement("tbody")); - for (const row of rows) { - const tr = tbody.appendChild(document.createElement("tr")); - for (const column of columns) { - const td = tr.appendChild(document.createElement("td")); - td.appendChild(document.createTextNode(row[column])); - } - } - table.value = rows; - return table; +function element(name, props, children) { + if (arguments.length === 2) children = props, props = undefined; + const element = document.createElement(name); + if (props !== undefined) for (const p in props) element[p] = props[p]; + if (children !== undefined) for (const c of children) element.appendChild(c); + return element; } -function formatText(text) { - const pre = document.createElement("pre"); - pre.className = "observablehq--inspect"; - pre.appendChild(document.createTextNode(text)); - return pre; +function text(value) { + return document.createTextNode(value); }