Skip to content

SQLite #212

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
May 25, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions src/fileAttachment.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import {require as requireDefault} from "d3-require";
import sqlite, {SQLiteDatabaseClient} from "./sqlite.js";

async function remote_fetch(file) {
const response = await fetch(await file.url());
Expand Down Expand Up @@ -56,6 +57,11 @@ class FileAttachment {
i.src = url;
});
}
async sqlite() {
const [SQL, buffer] = await Promise.all([sqlite(requireDefault), this.arrayBuffer()]);
const db = new SQL.Database(new Uint8Array(buffer));
return new SQLiteDatabaseClient(db);
}
}

export function NoFileAttachments(name) {
Expand Down
8 changes: 5 additions & 3 deletions src/library.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import now from "./now.js";
import Promises from "./promises/index.js";
import resolve from "./resolve.js";
import requirer from "./require.js";
import SQLite from "./sqlite.js";
import svg from "./svg.js";
import tex from "./tex.js";
import vegalite from "./vegalite.js";
Expand All @@ -26,18 +27,19 @@ export default Object.assign(function Library(resolver) {
Mutable: () => Mutable,
Plot: () => require("@observablehq/[email protected]/dist/plot.umd.min.js"),
Promises: () => Promises,
SQLite: () => SQLite(require),
_: () => require("[email protected]/lodash.min.js"),
d3: () => require("[email protected]/dist/d3.min.js"),
dot: () => require("@observablehq/[email protected]/dist/graphviz.min.js"),
htl: () => require("[email protected]/dist/htl.min.js"),
html: () => html,
md: md(require),
md: () => md(require),
now: now,
require: () => require,
resolve: () => resolve,
svg: () => svg,
tex: tex(require),
vl: vegalite(require),
tex: () => tex(require),
vl: () => vegalite(require),
width: width
}));
}, {resolve: requireDefault.resolve});
Expand Down
77 changes: 37 additions & 40 deletions src/md.js
Original file line number Diff line number Diff line change
@@ -1,47 +1,44 @@
import template from "./template.js";

const HL_ROOT =
"https://cdn.jsdelivr.net/npm/@observablehq/[email protected]/";
const HL_ROOT = "https://cdn.jsdelivr.net/npm/@observablehq/[email protected]/";

export default function(require) {
return function() {
return require("[email protected]/marked.min.js").then(function(marked) {
return template(
function(string) {
var root = document.createElement("div");
root.innerHTML = marked(string, {langPrefix: ""}).trim();
var code = root.querySelectorAll("pre code[class]");
if (code.length > 0) {
require(HL_ROOT + "highlight.min.js").then(function(hl) {
code.forEach(function(block) {
function done() {
hl.highlightBlock(block);
block.parentNode.classList.add("observablehq--md-pre");
}
if (hl.getLanguage(block.className)) {
done();
} else {
require(HL_ROOT + "async-languages/index.js")
.then(index => {
if (index.has(block.className)) {
return require(HL_ROOT +
"async-languages/" +
index.get(block.className)).then(language => {
hl.registerLanguage(block.className, language);
});
}
})
.then(done, done);
}
});
return require("[email protected]/marked.min.js").then(function(marked) {
return template(
function(string) {
var root = document.createElement("div");
root.innerHTML = marked(string, {langPrefix: ""}).trim();
var code = root.querySelectorAll("pre code[class]");
if (code.length > 0) {
require(HL_ROOT + "highlight.min.js").then(function(hl) {
code.forEach(function(block) {
function done() {
hl.highlightBlock(block);
block.parentNode.classList.add("observablehq--md-pre");
}
if (hl.getLanguage(block.className)) {
done();
} else {
require(HL_ROOT + "async-languages/index.js")
.then(index => {
if (index.has(block.className)) {
return require(HL_ROOT +
"async-languages/" +
index.get(block.className)).then(language => {
hl.registerLanguage(block.className, language);
});
}
})
.then(done, done);
}
});
}
return root;
},
function() {
return document.createElement("div");
});
}
);
});
};
return root;
},
function() {
return document.createElement("div");
}
);
});
}
56 changes: 56 additions & 0 deletions src/sqlite.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
export default async function sqlite(require) {
const sql = await require("[email protected]/dist/sql-wasm.js");
return sql({locateFile: file => `https://cdn.jsdelivr.net/npm/[email protected]/dist/${file}`});
}

export class SQLiteDatabaseClient {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it going to be odd that this database client doesn't extend the worker's DatabaseClient function, prototype-chain-wise, even though it implements the same interface? It only occurred to me because queryRow matches exactly the worker's implementation and could be re-used here without copying.

I was thinking alternatively, this implementation could move to the worker and it could accept a file attachment for the constructor: DatabaseClient(FileAttachment("db.sqlite")). Though, there's a lot of assumption going into that expression since nothing references "sqlite" statically.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DatabaseClient isn’t currently part of the standard library, so I’ll need to move parts of it up here to make that work. I’ll take a crack at it.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought about some more and I don’t think this is worth the effort. It’s not easy to move the DatabaseClient class into the standard library because it hard-codes the known implementations, all of which currently require coordination with the editor (since they are remote databases that require a database connector, not an in-memory local database like SQLite).

constructor(db) {
Object.defineProperties(this, {
_db: {value: db}
});
}
async query(query, params) {
return await exec(this._db, query, params);
}
async queryRow(query, params) {
return (await this.query(query, params))[0] || null;
}
async explain(query, params) {
const rows = await this.query(`EXPLAIN QUERY PLAN ${query}`, params);
return element("pre", {className: "observablehq--inspect"}, [
text(rows.map(row => row.detail).join("\n"))
]);
}
async describe(object) {
const rows = await (object === undefined
? this.query(`SELECT name FROM sqlite_master WHERE type = 'table'`)
: this.query(`SELECT * FROM pragma_table_info(?)`, [object]));
if (!rows.length) throw new Error("Not found");
const {columns} = rows;
return element("table", {value: rows}, [
element("thead", [element("tr", columns.map(c => element("th", [text(c)])))]),
element("tbody", rows.map(r => element("tr", columns.map(c => element("td", [text(r[c])])))))
]);
}
}

async function exec(db, query, params) {
const [result] = await db.exec(query, params);
if (!result) return [];
const {columns, values} = result;
const rows = values.map(row => Object.fromEntries(row.map((value, i) => [columns[i], value])));
rows.columns = columns;
return rows;
}

function element(name, props, children) {
if (arguments.length === 2) children = props, props = undefined;
const element = document.createElement(name);
if (props !== undefined) for (const p in props) element[p] = props[p];
if (children !== undefined) for (const c of children) element.appendChild(c);
return element;
}

function text(value) {
return document.createTextNode(value);
}
36 changes: 17 additions & 19 deletions src/tex.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,25 +11,23 @@ function style(href) {
});
}

export default function(require) {
return function() {
return Promise.all([
require("@observablehq/[email protected]/dist/katex.min.js"),
require.resolve("@observablehq/[email protected]/dist/katex.min.css").then(style)
]).then(function(values) {
var katex = values[0], tex = renderer();
export default function tex(require) {
return Promise.all([
require("@observablehq/[email protected]/dist/katex.min.js"),
require.resolve("@observablehq/[email protected]/dist/katex.min.css").then(style)
]).then(function(values) {
var katex = values[0], tex = renderer();

function renderer(options) {
return function() {
var root = document.createElement("div");
katex.render(raw.apply(String, arguments), root, options);
return root.removeChild(root.firstChild);
};
}
function renderer(options) {
return function() {
var root = document.createElement("div");
katex.render(raw.apply(String, arguments), root, options);
return root.removeChild(root.firstChild);
};
}

tex.options = renderer;
tex.block = renderer({displayMode: true});
return tex;
});
};
tex.options = renderer;
tex.block = renderer({displayMode: true});
return tex;
});
}
16 changes: 7 additions & 9 deletions src/vegalite.js
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
export default function vl(require) {
return async () => {
const [vega, vegalite, api] = await Promise.all([
"[email protected]/build/vega.min.js",
"[email protected]/build/vega-lite.min.js",
"[email protected]/build/vega-lite-api.min.js"
].map(module => require(module)));
return api.register(vega, vegalite);
};
export default async function vl(require) {
const [vega, vegalite, api] = await Promise.all([
"[email protected]/build/vega.min.js",
"[email protected]/build/vega-lite.min.js",
"[email protected]/build/vega-lite-api.min.js"
].map(module => require(module)));
return api.register(vega, vegalite);
}
1 change: 1 addition & 0 deletions test/index-test.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ test("new Library returns a library with the expected keys", async t => {
"Mutable",
"Plot",
"Promises",
"SQLite",
"_",
"d3",
"dot",
Expand Down