diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..4a4833c --- /dev/null +++ b/.travis.yml @@ -0,0 +1,3 @@ +language: node_js +node: + - lts/* diff --git a/README.md b/README.md index 05b3ce5..25ebaff 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # random-access-chrome-file +[![Build Status](https://travis-ci.com/random-access-storage/random-access-chrome-file.svg?branch=master)](https://travis-ci.com/random-access-storage/random-access-chrome-file) +[![JavaScript Style Guide](https://img.shields.io/badge/code_style-standard-brightgreen.svg)](https://standardjs.com) + A [random-access-storage](https://github.com/random-access-storage/random-access-storage) instance backed by the Chrome file system api ``` diff --git a/bench.js b/bench.js index 88f29b6..7f4e401 100644 --- a/bench.js +++ b/bench.js @@ -37,7 +37,10 @@ function benchRead () { console.time('512mb read') st.read(0, 65536, function onread (err, buf) { if (err) throw err - if (offset >= 512 * 1024 * 1024) return console.timeEnd('512mb read') + if (offset >= 512 * 1024 * 1024) { + console.timeEnd('512mb read') + return process.exit(0) // we do this to close browser-runner + } st.read(offset += buf.length, 65536, onread) }) } diff --git a/browser-runner.config.js b/browser-runner.config.js new file mode 100644 index 0000000..c15a2e8 --- /dev/null +++ b/browser-runner.config.js @@ -0,0 +1,17 @@ +const finished = require('tap-finished') + +let stream = null + +module.exports = { + beforeAll ({ shutdown }) { + stream = finished(function (results) { + if (results.ok) { + return shutdown(0) + } + shutdown(1) + }) + }, + onMessage (msg) { + stream.write(msg + '\n') + } +} diff --git a/index.js b/index.js index 2763424..fadb037 100644 --- a/index.js +++ b/index.js @@ -1,4 +1,5 @@ const ras = require('random-access-storage') +const mutexify = require('mutexify') const TYPE = {type: 'octet/stream'} const requestFileSystem = window.requestFileSystem || window.webkitRequestFileSystem @@ -29,6 +30,7 @@ function createFile (name, opts) { var fs = null var entry = null + var file = null var toDestroy = null var readers = [] var writers = [] @@ -36,12 +38,12 @@ function createFile (name, opts) { return ras({read, write, open, stat, close, destroy}) function read (req) { - const r = readers.pop() || new ReadRequest(readers, entry, mutex) + const r = readers.pop() || new ReadRequest(readers, entry, file, mutex) r.run(req) } function write (req) { - const w = writers.pop() || new WriteRequest(writers, entry, mutex) + const w = writers.pop() || new WriteRequest(writers, entry, file, mutex) w.run(req) } @@ -51,9 +53,10 @@ function createFile (name, opts) { } function stat (req) { - entry.file(file => { + file.get((err, file) => { + if (err) return req.callback(err) req.callback(null, file) - }, err => req.callback(err)) + }) } function destroy (req) { @@ -78,7 +81,11 @@ function createFile (name, opts) { mkdirp(parentFolder(name), function () { fs.root.getFile(name, {create: true}, function (e) { entry = toDestroy = e - req.callback(null) + file = new EntryFile(entry) + file.get((err) => { + if (err) return onerror(err) + req.callback(null) + }) }, onerror) }) }, onerror) @@ -107,9 +114,10 @@ function parentFolder (path) { return /^\w:$/.test(p) ? '' : p } -function WriteRequest (pool, entry, mutex) { +function WriteRequest (pool, entry, file, mutex) { this.pool = pool this.entry = entry + this.file = file this.mutex = mutex this.writer = null this.req = null @@ -122,8 +130,8 @@ WriteRequest.prototype.makeWriter = function () { this.entry.createWriter(function (writer) { self.writer = writer - writer.onwriteend = function () { - self.onwrite(null) + writer.onwriteend = function (e) { + self.onwrite(null, e) } writer.onerror = function (err) { @@ -134,7 +142,7 @@ WriteRequest.prototype.makeWriter = function () { }) } -WriteRequest.prototype.onwrite = function (err) { +WriteRequest.prototype.onwrite = function (err, e) { const req = this.req this.req = null @@ -143,6 +151,10 @@ WriteRequest.prototype.onwrite = function (err) { this.mutex.release() } + if (!err) { + this.file.updateSize(e.currentTarget.length) + } + if (this.truncating) { this.truncating = false if (!err) return this.run(req) @@ -154,6 +166,7 @@ WriteRequest.prototype.onwrite = function (err) { WriteRequest.prototype.truncate = function () { this.truncating = true + this.file.truncate() this.writer.truncate(this.req.offset) } @@ -164,7 +177,9 @@ WriteRequest.prototype.lock = function () { } WriteRequest.prototype.run = function (req) { - this.entry.file(file => { + this.file.getWritableFile((err, file) => { + if (err) return req.callback(err) + this.req = req if (!this.writer || this.writer.length !== file.size) return this.makeWriter() @@ -178,7 +193,7 @@ WriteRequest.prototype.run = function (req) { this.writer.seek(req.offset) this.writer.write(new Blob([req.data], TYPE)) - }, err => req.callback(err)) + }) } function Mutex () { @@ -202,9 +217,10 @@ Mutex.prototype.lock = function (req) { return true } -function ReadRequest (pool, entry, mutex) { +function ReadRequest (pool, entry, file, mutex) { this.pool = pool this.entry = entry + this.file = file this.mutex = mutex this.reader = new FileReader() this.req = null @@ -233,8 +249,14 @@ ReadRequest.prototype.onread = function (err, buf) { const req = this.req if (err && this.retry) { - this.retry = false - if (this.lock(this)) this.run(req) + if (err.code !== 0) { + this.retry = false + } + + if (this.lock(this)) { + this.file.clearFile() + this.run(req) + } return } @@ -251,10 +273,69 @@ ReadRequest.prototype.onread = function (err, buf) { } ReadRequest.prototype.run = function (req) { - this.entry.file(file => { + this.file.get((err, file) => { + if (err) return req.callback(err) + const end = req.offset + req.size this.req = req if (end > file.size) return this.onread(new Error('Could not satisfy length'), null) this.reader.readAsArrayBuffer(file.slice(req.offset, end)) - }, err => req.callback(err)) + }) +} + +class EntryFile { + constructor (entry) { + this._entry = entry + this._lock = mutexify() + this._file = null + this._size = 0 + this._truncated = false + } + + get size () { + return this._size + } + + updateSize (size) { + if (!this._truncated && size > this._size) { + this._size = size + } + + this.clearFile() + } + + truncate () { + this._truncated = true + } + + clearFile () { + this._file = null + } + + get (cb) { + if (this._file && !this._truncated) { + return cb(null, this._file) + } + + this._lock(release => { + if (this._file && !this._truncated) { + return release(cb, null, this._file) + } + + this._entry.file(file => { + this._truncated = false + this._file = file + this._size = file.size + release(cb, null, file) + }, err => release(cb, err)) + }) + } + + getWritableFile (cb) { + if (!this._truncated) { + return cb(null, this) + } + + this.get(cb) + } } diff --git a/package.json b/package.json index 5f62f58..e95089d 100644 --- a/package.json +++ b/package.json @@ -4,13 +4,21 @@ "description": "random-access-storage instance backed by the Chrome file system api.", "main": "index.js", "dependencies": { + "mutexify": "^1.3.0", "random-access-storage": "^1.3.0" }, "devDependencies": { - "standard": "^11.0.1" + "@dxos/browser-runner": "^1.0.0-beta.9", + "random-access-test": "^1.0.0", + "standard": "^11.0.1", + "tap-finished": "0.0.1", + "tape": "^5.0.1" }, "scripts": { - "test": "standard" + "test": "browser-runner test.js", + "posttest": "npm run lint", + "lint": "standard", + "bench": "browser-runner bench.js --timeout 0" }, "repository": { "type": "git", diff --git a/test.js b/test.js new file mode 100644 index 0000000..1793044 --- /dev/null +++ b/test.js @@ -0,0 +1,67 @@ +const { promisify } = require('util') +const test = require('tape') +const randomAccessTest = require('random-access-test') +const racf = require('./') + +const createStorage = (root) => (file, opts) => racf(`${root}/${file}`, opts) + +const storage = createStorage('tests-' + Math.random()) + +randomAccessTest(function (name, options, callback) { + callback(storage(name, options)) +}, {}) + +test('write/read concurrent requests', async t => { + const st = storage('random') + + const rand = (min, max) => Math.floor(Math.random() * max) + min + const read = promisify(st.read.bind(st)) + const write = promisify(st.write.bind(st)) + + try { + await new Promise(resolve => st.open(() => resolve())) + + const buf = Buffer.alloc(1) + + await Promise.all([...Array(1000).keys()].map(from => { + return write(from, buf) + })) + + await Promise.all([...Array(1000).keys()].map(() => { + const row = rand(0, 2) + const from = rand(0, 1000) + const to = 1 + + if (row === 0) { + return read(from, to) + } + return write(from, buf) + })) + + t.pass('should work ok with random concurrent request') + t.end() + } catch (err) { + t.end(err) + } +}) + +test('write concurrent requests over the same offset different size', async t => { + const st = storage('random') + + const write = promisify(st.write.bind(st)) + + try { + await new Promise(resolve => st.open(() => resolve())) + + await Promise.all([ + write(0, Buffer.alloc(10)), + write(0, Buffer.alloc(1)), + write(0, Buffer.alloc(5)) + ]) + + t.pass('should write multiple requests over the same offset different size') + t.end() + } catch (err) { + t.end(err) + } +})