Support for an advanced “persistentCache”

index.js

@@ -27,6 +27,11 @@ function Deps (opts) {
     if (!opts) opts = {};
 
     this.basedir = opts.basedir || process.cwd();
+    this.persistentCache = opts.persistentCache || function (file, id, pkg, fallback, cb) {
+        setImmediate(function () {
+            fallback(null, cb);
+        });
+    };
     this.cache = opts.cache;
     this.fileCache = opts.fileCache;
     this.pkgCache = opts.packageCache || {};
@@ -196,10 +201,7 @@ Deps.prototype.resolve = function (id, parent, cb) {
 Deps.prototype.readFile = function (file, id, pkg) {
     var self = this;
     if (xhas(this.fileCache, file)) {
-        var tr = through();
-        tr.push(this.fileCache[file]);
-        tr.push(null);
-        return tr;
+        return toStream(this.fileCache[file]);
     }
     var rs = fs.createReadStream(file, {
         encoding: 'utf8'
@@ -376,19 +378,44 @@ Deps.prototype.walk = function (id, parent, cb) {
         var c = self.cache && self.cache[file];
         if (c) return fromDeps(file, c.source, c.package, fakePath, Object.keys(c.deps));
 
-        self.readFile(file, id, pkg)
-            .pipe(self.getTransforms(fakePath || file, pkg, {
-                builtin: builtin,
-                inNodeModules: parent.inNodeModules
-            }))
-            .pipe(concat(function (body) {
-                fromSource(file, body.toString('utf8'), pkg, fakePath);
-            }))
-        ;
+        self.persistentCache(file, id, pkg, persistentCacheFallback, function (err, c) {
+            if (err) {
+                self.emit('error', err);
+                return;
+            }
+            fromDeps(file, c.source, c.package, fakePath, Object.keys(c.deps));
+        });
+
+        function persistentCacheFallback (dataAsString, cb) {
+            var stream = dataAsString ? toStream(dataAsString) : self.readFile(file, id, pkg);
+            stream
+                .pipe(self.getTransforms(fakePath || file, pkg, {
+                    builtin: builtin,
+                    inNodeModules: parent.inNodeModules
+                }))
+                .pipe(concat(function (body) {
+                    var src = body.toString('utf8');
+                    var deps = getDeps(file, src);
+                    if (deps) {
+                        cb(null, {
+                            source: src,
+                            package: pkg,
+                            deps: deps.reduce(function (deps, dep) {
+                                deps[dep] = true;
+                                return deps;
+                            }, {})
+                        });
+                    }
+                }));
+        }
     });
 
+    function getDeps (file, src) {
+        return rec.noparse ? [] : self.parseDeps(file, src);
+    }
+
     function fromSource (file, src, pkg, fakePath) {
-        var deps = rec.noparse ? [] : self.parseDeps(file, src);
+        var deps = getDeps(file, src);
         if (deps) fromDeps(file, src, pkg, fakePath, deps);
     }
 
@@ -550,6 +577,13 @@ function xhas (obj) {
     return true;
 }
 
+function toStream (dataAsString) {
+    var tr = through();
+    tr.push(dataAsString);
+    tr.push(null);
+    return tr;
+}
+
 function has (obj, key) {
     return obj && Object.prototype.hasOwnProperty.call(obj, key);
 }

readme.markdown

@@ -94,6 +94,51 @@ contents for browser fields, main entries, and transforms
 * `opts.fileCache` - an object mapping filenames to raw source to avoid reading
 from disk.
 
+* `opts.persistentCache` - a complex cache handler that allows async and persistent
+    caching of data. A `persistentCache` needs to follow this interface:
+    ```
+    function persistentCache (
+        file, // the path to the file that is loaded
+        id,   // the id that is used to reference this file
+        pkg,  // the package that this file belongs to fallback
+        fallback, // async fallback handler to be called if the cache doesn't hold the given file 
+        cb    // callback handler that receives the cache data
+    ) {
+        if (hasError()) {
+            return cb(error) // Pass any error to the callback
+        }
+
+        var fileData = fs.readFileSync(file)
+        var key = keyFromFile(file, fileData)
+
+        if (db.has(key)) {
+            return cb(null, {
+                source: db.get(key).toString(),
+                package: pkg, // The package for housekeeping
+                deps: {
+                    'id':  // id that is used to reference a required file
+                    'file' // file path to the required file
+                }
+            })
+        }
+        //
+        // The fallback will process the file in case the file is not
+        // in cache.
+        //
+        // Note that if your implementation doesn't need the file data
+        // then you can pass `null` instead of the source and the fallback will
+        // fetch the data by itself.
+        //
+        fallback(fileData, function (error, cacheableEntry) {
+            if (error) {
+                return cb(error)
+            }
+            db.addToCache(key, cacheableEntry)
+            cb(null, cacheableEntry)
+        })
+    }
+    ```
+
 * `opts.paths` - array of global paths to search. Defaults to splitting on `':'`
 in `process.env.NODE_PATH`
 

test/cache_persistent.js

@@ -0,0 +1,81 @@
+var parser = require('../');
+var test = require('tap').test;
+var path = require('path');
+var fs = require('fs');
+
+var files = {
+    foo: path.join(__dirname, '/files/foo.js'),
+    bar: path.join(__dirname, '/files/bar.js')
+};
+
+test('uses persistent cache', function (t) {
+    t.plan(1);
+    var p = parser({
+        persistentCache: function (file, id, pkg, fallback, cb) {
+            if (file === files.bar) {
+                return fallback(null, cb)
+            }
+            cb(null, {
+                source: 'file at ' + file + '@' + id,
+                package: pkg,
+                deps: { './bar': files.bar }
+            })
+        }
+    });
+    p.end({ id: 'foo', file: files.foo, entry: false });
+
+    var rows = [];
+    p.on('data', function (row) { rows.push(row) });
+    p.on('end', function () {
+        t.same(rows.sort(cmp), [
+            {
+                id: files.bar,
+                file: files.bar,
+                source: fs.readFileSync(files.bar, 'utf8'),
+                deps: {}
+            },
+            {
+                id: 'foo',
+                file: files.foo,
+                source: 'file at ' + files.foo + '@' + files.foo,
+                deps: { './bar': files.bar }
+            }
+        ].sort(cmp));
+    });
+});
+
+test('passes persistent cache error through', function (t) {
+    t.plan(1);
+    var p = parser({
+        persistentCache: function (file, id, pkg, fallback, cb) {
+            cb(new Error('foo'))
+        }
+    });
+    p.end({ id: 'foo', file: files.foo, entry: false });
+    p.on('error', function (err) { t.equals(err.message, 'foo') });
+});
+
+test('allow passing of the raw source as string', function (t) {
+    t.plan(1);
+    var p = parser({
+        persistentCache: function (file, id, pkg, fallback, cb) {
+            fallback(fs.readFileSync(files.bar, 'utf8'), cb)
+        }
+    });
+    p.end({ id: 'foo', file: files.foo, entry: false });
+
+    var rows = [];
+    p.on('data', function (row) { rows.push(row) });
+    p.on('end', function () {
+        t.same(rows.sort(cmp), [
+            {
+                id: 'foo',
+                file: files.foo,
+                source: fs.readFileSync(files.bar, 'utf8'),
+                deps: {}
+            }
+        ].sort(cmp));
+    });
+});
+
+function cmp (a, b) { return a.id < b.id ? -1 : 1 }