Adds schema caching capabilities (5s by default) (parse-community#2286)

* Adds schema caching capabilities (off by default)

* Use InMemoryCacheAdapter

* Uses proper adapter to generate a cache

* Fix bugs when running disabled cache

* nits

* nits

* Use options object instead of boolean

* Imrpove concurrency of loadSchema

* Adds testing with SCHEMA_CACHE_ON

* Use CacheController instead of generator

- Makes caching SchemaCache use a generated prefix
- Makes clearing the SchemaCache clear only the cached schema keys
- Enable cache by default (ttl 5s)

Author: flovilmart

spec/ParseInstallation.spec.js

@@ -9,13 +9,19 @@ let Parse = require('parse/node').Parse;
 let rest = require('../src/rest');
 let request = require("request");
 
-let config = new Config('test');
-let database = config.database;
+let config;
+let database;
 let defaultColumns = require('../src/Controllers/SchemaController').defaultColumns;
 
 const installationSchema = { fields: Object.assign({}, defaultColumns._Default, defaultColumns._Installation) };
 
 describe('Installations', () => {
+
+  beforeEach(() => {
+    config = new Config('test');
+    database = config.database;
+  });
+
   it_exclude_dbs(['postgres'])('creates an android installation with ids', (done) => {
     var installId = '12345678-abcd-abcd-abcd-123456789abc';
     var device = 'android';

spec/PointerPermissions.spec.js

@@ -4,6 +4,11 @@ var Schema = require('../src/Controllers/SchemaController');
 var Config = require('../src/Config');
 
 describe('Pointer Permissions', () => {
+
+  beforeEach(() => {
+    new Config(Parse.applicationId).database.schemaCache.clear();
+  });
+
   it_exclude_dbs(['postgres'])('should work with find', (done) => {
     let config = new Config(Parse.applicationId);
     let user = new Parse.User();

spec/RestQuery.spec.js

@@ -8,11 +8,17 @@ var rest = require('../src/rest');
 var querystring = require('querystring');
 var request = require('request');
 
-var config = new Config('test');
-let database = config.database;
+var config;
+let database;
 var nobody = auth.nobody(config);
 
 describe('rest query', () => {
+
+  beforeEach(() => {
+    config = new Config('test');
+    database = config.database;
+  });
+
   it('basic query', (done) => {
     rest.create(config, nobody, 'TestObject', {}).then(() => {
       return rest.find(config, nobody, 'TestObject', {});

spec/Schema.spec.js

@@ -4,7 +4,7 @@ var Config = require('../src/Config');
 var SchemaController = require('../src/Controllers/SchemaController');
 var dd = require('deep-diff');
 
-var config = new Config('test');
+var config;
 
 var hasAllPODobject = () => {
   var obj = new Parse.Object('HasAllPOD');
@@ -20,6 +20,10 @@ var hasAllPODobject = () => {
 };
 
 describe('SchemaController', () => {
+  beforeEach(() => {
+    config = new Config('test');
+  });
+
   it('can validate one object', (done) => {
     config.database.loadSchema().then((schema) => {
       return schema.validateObject('TestObject', {a: 1, b: 'yo', c: false});

spec/helper.js

@@ -59,7 +59,7 @@ var defaultConfiguration = {
     myoauth: {
       module: path.resolve(__dirname, "myoauth") // relative path as it's run from src
     }
-  },
+  }
 };
 
 let openConnections = {};

spec/schemas.spec.js

@@ -116,6 +116,11 @@ var masterKeyHeaders = {
 };
 
 describe('schemas', () => {
+
+  beforeEach(() => {
+     config.database.schemaCache.clear();
+  });
+
   it('requires the master key to get all schemas', (done) => {
     request.get({
       url: 'http://localhost:8378/1/schemas',

src/Config.js

@@ -3,6 +3,8 @@
 // mount is the URL for the root of the API; includes http, domain, etc.
 
 import AppCache from './cache';
+import SchemaCache from './Controllers/SchemaCache';
+import DatabaseController from './Controllers/DatabaseController';
 
 function removeTrailingSlash(str) {
   if (!str) {
@@ -31,7 +33,14 @@ export class Config {
     this.fileKey = cacheInfo.fileKey;
     this.facebookAppIds = cacheInfo.facebookAppIds;
     this.allowClientClassCreation = cacheInfo.allowClientClassCreation;
-    this.database = cacheInfo.databaseController;
+
+    // Create a new DatabaseController per request
+    if (cacheInfo.databaseController) {
+      const schemaCache = new SchemaCache(cacheInfo.cacheController, cacheInfo.schemaCacheTTL);
+      this.database = new DatabaseController(cacheInfo.databaseController.adapter, schemaCache);
+    }
+
+    this.schemaCacheTTL = cacheInfo.schemaCacheTTL;
 
     this.serverURL = cacheInfo.serverURL;
     this.publicServerURL = removeTrailingSlash(cacheInfo.publicServerURL);

src/Controllers/CacheController.js

@@ -13,9 +13,10 @@ function joinKeys(...keys) {
  * eg "Role" or "Session"
  */
 export class SubCache {
-  constructor(prefix, cacheController) {
+  constructor(prefix, cacheController, ttl) {
     this.prefix = prefix;
     this.cache = cacheController;
+    this.ttl = ttl;
   }
 
   get(key) {

src/Controllers/DatabaseController.js

@@ -1,4 +1,4 @@
-// A database adapter that works with data exported from the hosted
+// A database adapter that works with data exported from the hosted
 // Parse database.
 
 import intersect from 'intersect';
@@ -7,7 +7,8 @@ import _         from 'lodash';
 var mongodb = require('mongodb');
 var Parse = require('parse/node').Parse;
 
-var SchemaController = require('../Controllers/SchemaController');
+var SchemaController = require('./SchemaController');
+
 const deepcopy = require('deepcopy');
 
 function addWriteACL(query, acl) {
@@ -44,7 +45,7 @@ const transformObjectACL = ({ ACL, ...result }) => {
   return result;
 }
 
-const specialQuerykeys = ['$and', '$or', '_rperm', '_wperm', '_perishable_token', '_email_verify_token'];
+const specialQuerykeys = ['$and', '$or', '_rperm', '_wperm', '_perishable_token', '_email_verify_token', '_email_verify_token_expires_at'];
 const validateQuery = query => {
   if (query.ACL) {
     throw new Parse.Error(Parse.Error.INVALID_QUERY, 'Cannot query on ACL.');
@@ -80,18 +81,13 @@ const validateQuery = query => {
   });
 }
 
-function DatabaseController(adapter, { skipValidation } = {}) {
+function DatabaseController(adapter, schemaCache) {
   this.adapter = adapter;
-
+  this.schemaCache = schemaCache;
   // We don't want a mutable this.schema, because then you could have
   // one request that uses different schemas for different parts of
   // it. Instead, use loadSchema to get a schema.
   this.schemaPromise = null;
-  this.skipValidation = !!skipValidation;
-}
-
-DatabaseController.prototype.WithoutValidation = function() {
-  return new DatabaseController(this.adapter, { skipValidation: true });
 }
 
 DatabaseController.prototype.collectionExists = function(className) {
@@ -105,20 +101,18 @@ DatabaseController.prototype.purgeCollection = function(className) {
 };
 
 DatabaseController.prototype.validateClassName = function(className) {
-  if (this.skipValidation) {
-    return Promise.resolve();
-  }
   if (!SchemaController.classNameIsValid(className)) {
     return Promise.reject(new Parse.Error(Parse.Error.INVALID_CLASS_NAME, 'invalid className: ' + className));
   }
   return Promise.resolve();
 };
 
 // Returns a promise for a schemaController.
-DatabaseController.prototype.loadSchema = function() {
+DatabaseController.prototype.loadSchema = function(options = {clearCache: false}) {
   if (!this.schemaPromise) {
-    this.schemaPromise = SchemaController.load(this.adapter);
-    this.schemaPromise.then(() => delete this.schemaPromise);
+    this.schemaPromise = SchemaController.load(this.adapter, this.schemaCache, options);
+    this.schemaPromise.then(() => delete this.schemaPromise,
+                             () => delete this.schemaPromise);
   }
   return this.schemaPromise;
 };
@@ -129,7 +123,7 @@ DatabaseController.prototype.loadSchema = function() {
 DatabaseController.prototype.redirectClassNameForKey = function(className, key) {
   return this.loadSchema().then((schema) => {
     var t = schema.getExpectedType(className, key);
-    if (t.type == 'Relation') {
+    if (t && t.type == 'Relation') {
       return t.targetClass;
     } else {
       return className;
@@ -184,13 +178,12 @@ const filterSensitiveData = (isMaster, aclGroup, className, object) => {
 //   acl:  a list of strings. If the object to be updated has an ACL,
 //         one of the provided strings must provide the caller with
 //         write permissions.
-const specialKeysForUpdate = ['_hashed_password', '_perishable_token', '_email_verify_token'];
+const specialKeysForUpdate = ['_hashed_password', '_perishable_token', '_email_verify_token', '_email_verify_token_expires_at'];
 DatabaseController.prototype.update = function(className, query, update, {
   acl,
   many,
   upsert,
-} = {}) {
-
+} = {}, skipSanitization = false) {
   const originalUpdate = update;
   // Make a copy of the object, so we don't mutate the incoming data.
   update = deepcopy(update);
@@ -252,7 +245,7 @@ DatabaseController.prototype.update = function(className, query, update, {
       if (!result) {
         return Promise.reject(new Parse.Error(Parse.Error.OBJECT_NOT_FOUND, 'Object not found.'));
       }
-      if (this.skipValidation) {
+      if (skipSanitization) {
         return Promise.resolve(result);
       }
       return sanitizeDatabaseResult(originalUpdate, result);
@@ -549,8 +542,10 @@ DatabaseController.prototype.reduceInRelation = function(className, query, schem
     return Promise.all(ors.map((aQuery, index) => {
       return this.reduceInRelation(className, aQuery, schema).then((aQuery) => {
         query['$or'][index] = aQuery;
-      })
-    }));
+      });
+    })).then(() => {
+      return Promise.resolve(query);
+    });
   }
 
   let promises = Object.keys(query).map((key) => {
@@ -811,8 +806,8 @@ const untransformObjectACL = ({_rperm, _wperm, ...output}) => {
 }
 
 DatabaseController.prototype.deleteSchema = function(className) {
-  return this.loadSchema()
-  .then(schemaController => schemaController.getOneSchema(className))
+  return this.loadSchema(true)
+  .then(schemaController => schemaController.getOneSchema(className, true))
   .catch(error => {
     if (error === undefined) {
       return { fields: {} };

src/Controllers/SchemaCache.js

@@ -0,0 +1,68 @@
+const MAIN_SCHEMA = "__MAIN_SCHEMA";
+const SCHEMA_CACHE_PREFIX = "__SCHEMA";
+const ALL_KEYS = "__ALL_KEYS";
+
+import { randomString } from '../cryptoUtils';
+
+export default class SchemaCache {
+  cache: Object;
+
+  constructor(cacheController, ttl = 30) {
+    this.ttl = ttl;
+    if (typeof ttl == 'string') {
+      this.ttl = parseInt(ttl);
+    }
+    this.cache = cacheController;
+    this.prefix = SCHEMA_CACHE_PREFIX+randomString(20);
+  }
+
+  put(key, value) {
+    return this.cache.get(this.prefix+ALL_KEYS).then((allKeys) => {
+      allKeys = allKeys || {};
+      allKeys[key] = true;
+      return Promise.all([this.cache.put(this.prefix+ALL_KEYS, allKeys, this.ttl), this.cache.put(key, value, this.ttl)]);
+    });
+  }
+
+  getAllClasses() {
+    if (!this.ttl) {
+      return Promise.resolve(null);
+    }
+    return this.cache.get(this.prefix+MAIN_SCHEMA);
+  }
+
+  setAllClasses(schema) {
+    if (!this.ttl) {
+      return Promise.resolve(null);
+    }
+    return this.put(this.prefix+MAIN_SCHEMA, schema);
+  }
+
+  setOneSchema(className, schema) {
+    if (!this.ttl) {
+      return Promise.resolve(null);
+    }
+    return this.put(this.prefix+className, schema);
+  }
+
+  getOneSchema(className) {
+    if (!this.ttl) {
+      return Promise.resolve(null);
+    }
+    return this.cache.get(this.prefix+className);
+  }
+
+  clear() {
+    // That clears all caches...
+    let promise = Promise.resolve();
+    return this.cache.get(this.prefix+ALL_KEYS).then((allKeys) => {
+      if (!allKeys) {
+        return;
+      }
+      let promises = Object.keys(allKeys).map((key) => {
+        return this.cache.del(key);
+      });
+      return Promise.all(promises);
+    });
+  }
+}