diff --git a/index.js b/index.js index 6684064..c4d5a4b 100644 --- a/index.js +++ b/index.js @@ -27,6 +27,11 @@ function Deps (opts) { if (!opts) opts = {}; this.basedir = opts.basedir || process.cwd(); + this.persistentCache = opts.persistentCache || function (file, id, pkg, fallback, cb) { + setImmediate(function () { + fallback(null, cb); + }); + }; this.cache = opts.cache; this.fileCache = opts.fileCache; this.pkgCache = opts.packageCache || {}; @@ -196,10 +201,7 @@ Deps.prototype.resolve = function (id, parent, cb) { Deps.prototype.readFile = function (file, id, pkg) { var self = this; if (xhas(this.fileCache, file)) { - var tr = through(); - tr.push(this.fileCache[file]); - tr.push(null); - return tr; + return toStream(this.fileCache[file]); } var rs = fs.createReadStream(file, { encoding: 'utf8' @@ -376,19 +378,44 @@ Deps.prototype.walk = function (id, parent, cb) { var c = self.cache && self.cache[file]; if (c) return fromDeps(file, c.source, c.package, fakePath, Object.keys(c.deps)); - self.readFile(file, id, pkg) - .pipe(self.getTransforms(fakePath || file, pkg, { - builtin: builtin, - inNodeModules: parent.inNodeModules - })) - .pipe(concat(function (body) { - fromSource(file, body.toString('utf8'), pkg, fakePath); - })) - ; + self.persistentCache(file, id, pkg, persistentCacheFallback, function (err, c) { + if (err) { + self.emit('error', err); + return; + } + fromDeps(file, c.source, c.package, fakePath, Object.keys(c.deps)); + }); + + function persistentCacheFallback (dataAsString, cb) { + var stream = dataAsString ? toStream(dataAsString) : self.readFile(file, id, pkg); + stream + .pipe(self.getTransforms(fakePath || file, pkg, { + builtin: builtin, + inNodeModules: parent.inNodeModules + })) + .pipe(concat(function (body) { + var src = body.toString('utf8'); + var deps = getDeps(file, src); + if (deps) { + cb(null, { + source: src, + package: pkg, + deps: deps.reduce(function (deps, dep) { + deps[dep] = true; + return deps; + }, {}) + }); + } + })); + } }); + function getDeps (file, src) { + return rec.noparse ? [] : self.parseDeps(file, src); + } + function fromSource (file, src, pkg, fakePath) { - var deps = rec.noparse ? [] : self.parseDeps(file, src); + var deps = getDeps(file, src); if (deps) fromDeps(file, src, pkg, fakePath, deps); } @@ -550,6 +577,13 @@ function xhas (obj) { return true; } +function toStream (dataAsString) { + var tr = through(); + tr.push(dataAsString); + tr.push(null); + return tr; +} + function has (obj, key) { return obj && Object.prototype.hasOwnProperty.call(obj, key); } diff --git a/readme.markdown b/readme.markdown index 8a1566b..d3327eb 100644 --- a/readme.markdown +++ b/readme.markdown @@ -94,6 +94,51 @@ contents for browser fields, main entries, and transforms * `opts.fileCache` - an object mapping filenames to raw source to avoid reading from disk. +* `opts.persistentCache` - a complex cache handler that allows async and persistent + caching of data. A `persistentCache` needs to follow this interface: + ``` + function persistentCache ( + file, // the path to the file that is loaded + id, // the id that is used to reference this file + pkg, // the package that this file belongs to fallback + fallback, // async fallback handler to be called if the cache doesn't hold the given file + cb // callback handler that receives the cache data + ) { + if (hasError()) { + return cb(error) // Pass any error to the callback + } + + var fileData = fs.readFileSync(file) + var key = keyFromFile(file, fileData) + + if (db.has(key)) { + return cb(null, { + source: db.get(key).toString(), + package: pkg, // The package for housekeeping + deps: { + 'id': // id that is used to reference a required file + 'file' // file path to the required file + } + }) + } + // + // The fallback will process the file in case the file is not + // in cache. + // + // Note that if your implementation doesn't need the file data + // then you can pass `null` instead of the source and the fallback will + // fetch the data by itself. + // + fallback(fileData, function (error, cacheableEntry) { + if (error) { + return cb(error) + } + db.addToCache(key, cacheableEntry) + cb(null, cacheableEntry) + }) + } + ``` + * `opts.paths` - array of global paths to search. Defaults to splitting on `':'` in `process.env.NODE_PATH` diff --git a/test/cache_persistent.js b/test/cache_persistent.js new file mode 100644 index 0000000..8c18734 --- /dev/null +++ b/test/cache_persistent.js @@ -0,0 +1,81 @@ +var parser = require('../'); +var test = require('tap').test; +var path = require('path'); +var fs = require('fs'); + +var files = { + foo: path.join(__dirname, '/files/foo.js'), + bar: path.join(__dirname, '/files/bar.js') +}; + +test('uses persistent cache', function (t) { + t.plan(1); + var p = parser({ + persistentCache: function (file, id, pkg, fallback, cb) { + if (file === files.bar) { + return fallback(null, cb) + } + cb(null, { + source: 'file at ' + file + '@' + id, + package: pkg, + deps: { './bar': files.bar } + }) + } + }); + p.end({ id: 'foo', file: files.foo, entry: false }); + + var rows = []; + p.on('data', function (row) { rows.push(row) }); + p.on('end', function () { + t.same(rows.sort(cmp), [ + { + id: files.bar, + file: files.bar, + source: fs.readFileSync(files.bar, 'utf8'), + deps: {} + }, + { + id: 'foo', + file: files.foo, + source: 'file at ' + files.foo + '@' + files.foo, + deps: { './bar': files.bar } + } + ].sort(cmp)); + }); +}); + +test('passes persistent cache error through', function (t) { + t.plan(1); + var p = parser({ + persistentCache: function (file, id, pkg, fallback, cb) { + cb(new Error('foo')) + } + }); + p.end({ id: 'foo', file: files.foo, entry: false }); + p.on('error', function (err) { t.equals(err.message, 'foo') }); +}); + +test('allow passing of the raw source as string', function (t) { + t.plan(1); + var p = parser({ + persistentCache: function (file, id, pkg, fallback, cb) { + fallback(fs.readFileSync(files.bar, 'utf8'), cb) + } + }); + p.end({ id: 'foo', file: files.foo, entry: false }); + + var rows = []; + p.on('data', function (row) { rows.push(row) }); + p.on('end', function () { + t.same(rows.sort(cmp), [ + { + id: 'foo', + file: files.foo, + source: fs.readFileSync(files.bar, 'utf8'), + deps: {} + } + ].sort(cmp)); + }); +}); + +function cmp (a, b) { return a.id < b.id ? -1 : 1 }