diff --git a/spec/ParseInstallation.spec.js b/spec/ParseInstallation.spec.js index 8681179adb..ea12320a6e 100644 --- a/spec/ParseInstallation.spec.js +++ b/spec/ParseInstallation.spec.js @@ -9,13 +9,19 @@ let Parse = require('parse/node').Parse; let rest = require('../src/rest'); let request = require("request"); -let config = new Config('test'); -let database = config.database; +let config; +let database; let defaultColumns = require('../src/Controllers/SchemaController').defaultColumns; const installationSchema = { fields: Object.assign({}, defaultColumns._Default, defaultColumns._Installation) }; describe('Installations', () => { + + beforeEach(() => { + config = new Config('test'); + database = config.database; + }); + it_exclude_dbs(['postgres'])('creates an android installation with ids', (done) => { var installId = '12345678-abcd-abcd-abcd-123456789abc'; var device = 'android'; diff --git a/spec/PointerPermissions.spec.js b/spec/PointerPermissions.spec.js index ece6d3a17b..4a1b048f64 100644 --- a/spec/PointerPermissions.spec.js +++ b/spec/PointerPermissions.spec.js @@ -4,6 +4,11 @@ var Schema = require('../src/Controllers/SchemaController'); var Config = require('../src/Config'); describe('Pointer Permissions', () => { + + beforeEach(() => { + new Config(Parse.applicationId).database.schemaCache.clear(); + }); + it_exclude_dbs(['postgres'])('should work with find', (done) => { let config = new Config(Parse.applicationId); let user = new Parse.User(); diff --git a/spec/RestQuery.spec.js b/spec/RestQuery.spec.js index 78d18d9a05..a6c8d9e7e1 100644 --- a/spec/RestQuery.spec.js +++ b/spec/RestQuery.spec.js @@ -9,11 +9,17 @@ var querystring = require('querystring'); var request = require('request'); var rp = require('request-promise'); -var config = new Config('test'); -let database = config.database; +var config; +let database; var nobody = auth.nobody(config); describe('rest query', () => { + + beforeEach(() => { + config = new Config('test'); + database = config.database; + }); + it('basic query', (done) => { rest.create(config, nobody, 'TestObject', {}).then(() => { return rest.find(config, nobody, 'TestObject', {}); diff --git a/spec/Schema.spec.js b/spec/Schema.spec.js index 0a6b17b3ea..2c6dc242b8 100644 --- a/spec/Schema.spec.js +++ b/spec/Schema.spec.js @@ -4,7 +4,7 @@ var Config = require('../src/Config'); var SchemaController = require('../src/Controllers/SchemaController'); var dd = require('deep-diff'); -var config = new Config('test'); +var config; var hasAllPODobject = () => { var obj = new Parse.Object('HasAllPOD'); @@ -20,6 +20,10 @@ var hasAllPODobject = () => { }; describe('SchemaController', () => { + beforeEach(() => { + config = new Config('test'); + }); + it('can validate one object', (done) => { config.database.loadSchema().then((schema) => { return schema.validateObject('TestObject', {a: 1, b: 'yo', c: false}); diff --git a/spec/helper.js b/spec/helper.js index 8a1d0c5fd4..c724ea93b7 100644 --- a/spec/helper.js +++ b/spec/helper.js @@ -59,7 +59,7 @@ var defaultConfiguration = { myoauth: { module: path.resolve(__dirname, "myoauth") // relative path as it's run from src } - }, + } }; let openConnections = {}; diff --git a/spec/schemas.spec.js b/spec/schemas.spec.js index 37276b400a..3ad733a227 100644 --- a/spec/schemas.spec.js +++ b/spec/schemas.spec.js @@ -116,6 +116,11 @@ var masterKeyHeaders = { }; describe('schemas', () => { + + beforeEach(() => { + config.database.schemaCache.clear(); + }); + it('requires the master key to get all schemas', (done) => { request.get({ url: 'http://localhost:8378/1/schemas', diff --git a/src/Config.js b/src/Config.js index 963022456c..c6cd30aab6 100644 --- a/src/Config.js +++ b/src/Config.js @@ -3,6 +3,8 @@ // mount is the URL for the root of the API; includes http, domain, etc. import AppCache from './cache'; +import SchemaCache from './Controllers/SchemaCache'; +import DatabaseController from './Controllers/DatabaseController'; function removeTrailingSlash(str) { if (!str) { @@ -32,7 +34,14 @@ export class Config { this.fileKey = cacheInfo.fileKey; this.facebookAppIds = cacheInfo.facebookAppIds; this.allowClientClassCreation = cacheInfo.allowClientClassCreation; - this.database = cacheInfo.databaseController; + + // Create a new DatabaseController per request + if (cacheInfo.databaseController) { + const schemaCache = new SchemaCache(cacheInfo.cacheController, cacheInfo.schemaCacheTTL); + this.database = new DatabaseController(cacheInfo.databaseController.adapter, schemaCache); + } + + this.schemaCacheTTL = cacheInfo.schemaCacheTTL; this.serverURL = cacheInfo.serverURL; this.publicServerURL = removeTrailingSlash(cacheInfo.publicServerURL); diff --git a/src/Controllers/CacheController.js b/src/Controllers/CacheController.js index 27dc4936f7..a6c88e9362 100644 --- a/src/Controllers/CacheController.js +++ b/src/Controllers/CacheController.js @@ -13,9 +13,10 @@ function joinKeys(...keys) { * eg "Role" or "Session" */ export class SubCache { - constructor(prefix, cacheController) { + constructor(prefix, cacheController, ttl) { this.prefix = prefix; this.cache = cacheController; + this.ttl = ttl; } get(key) { diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index cf64caa0bf..f286907f13 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -7,7 +7,8 @@ import _ from 'lodash'; var mongodb = require('mongodb'); var Parse = require('parse/node').Parse; -var SchemaController = require('../Controllers/SchemaController'); +var SchemaController = require('./SchemaController'); + const deepcopy = require('deepcopy'); function addWriteACL(query, acl) { @@ -80,9 +81,9 @@ const validateQuery = query => { }); } -function DatabaseController(adapter) { +function DatabaseController(adapter, schemaCache) { this.adapter = adapter; - + this.schemaCache = schemaCache; // We don't want a mutable this.schema, because then you could have // one request that uses different schemas for different parts of // it. Instead, use loadSchema to get a schema. @@ -107,9 +108,9 @@ DatabaseController.prototype.validateClassName = function(className) { }; // Returns a promise for a schemaController. -DatabaseController.prototype.loadSchema = function() { +DatabaseController.prototype.loadSchema = function(options = {clearCache: false}) { if (!this.schemaPromise) { - this.schemaPromise = SchemaController.load(this.adapter); + this.schemaPromise = SchemaController.load(this.adapter, this.schemaCache, options); this.schemaPromise.then(() => delete this.schemaPromise, () => delete this.schemaPromise); } @@ -805,8 +806,8 @@ const untransformObjectACL = ({_rperm, _wperm, ...output}) => { } DatabaseController.prototype.deleteSchema = function(className) { - return this.loadSchema() - .then(schemaController => schemaController.getOneSchema(className)) + return this.loadSchema(true) + .then(schemaController => schemaController.getOneSchema(className, true)) .catch(error => { if (error === undefined) { return { fields: {} }; diff --git a/src/Controllers/SchemaCache.js b/src/Controllers/SchemaCache.js new file mode 100644 index 0000000000..7a56f10710 --- /dev/null +++ b/src/Controllers/SchemaCache.js @@ -0,0 +1,68 @@ +const MAIN_SCHEMA = "__MAIN_SCHEMA"; +const SCHEMA_CACHE_PREFIX = "__SCHEMA"; +const ALL_KEYS = "__ALL_KEYS"; + +import { randomString } from '../cryptoUtils'; + +export default class SchemaCache { + cache: Object; + + constructor(cacheController, ttl = 30) { + this.ttl = ttl; + if (typeof ttl == 'string') { + this.ttl = parseInt(ttl); + } + this.cache = cacheController; + this.prefix = SCHEMA_CACHE_PREFIX+randomString(20); + } + + put(key, value) { + return this.cache.get(this.prefix+ALL_KEYS).then((allKeys) => { + allKeys = allKeys || {}; + allKeys[key] = true; + return Promise.all([this.cache.put(this.prefix+ALL_KEYS, allKeys, this.ttl), this.cache.put(key, value, this.ttl)]); + }); + } + + getAllClasses() { + if (!this.ttl) { + return Promise.resolve(null); + } + return this.cache.get(this.prefix+MAIN_SCHEMA); + } + + setAllClasses(schema) { + if (!this.ttl) { + return Promise.resolve(null); + } + return this.put(this.prefix+MAIN_SCHEMA, schema); + } + + setOneSchema(className, schema) { + if (!this.ttl) { + return Promise.resolve(null); + } + return this.put(this.prefix+className, schema); + } + + getOneSchema(className) { + if (!this.ttl) { + return Promise.resolve(null); + } + return this.cache.get(this.prefix+className); + } + + clear() { + // That clears all caches... + let promise = Promise.resolve(); + return this.cache.get(this.prefix+ALL_KEYS).then((allKeys) => { + if (!allKeys) { + return; + } + let promises = Object.keys(allKeys).map((key) => { + return this.cache.del(key); + }); + return Promise.all(promises); + }); + } +} diff --git a/src/Controllers/SchemaController.js b/src/Controllers/SchemaController.js index 8481f5344d..d321621f4f 100644 --- a/src/Controllers/SchemaController.js +++ b/src/Controllers/SchemaController.js @@ -273,19 +273,25 @@ class SchemaController { data; perms; - constructor(databaseAdapter) { + constructor(databaseAdapter, schemaCache) { this._dbAdapter = databaseAdapter; - + this._cache = schemaCache; // this.data[className][fieldName] tells you the type of that field, in mongo format this.data = {}; // this.perms[className][operation] tells you the acl-style permissions this.perms = {}; } - reloadData() { + reloadData(options = {clearCache: false}) { + if (options.clearCache) { + this._cache.clear(); + } + if (this.reloadDataPromise && !options.clearCache) { + return this.reloadDataPromise; + } this.data = {}; this.perms = {}; - return this.getAllClasses() + this.reloadDataPromise = this.getAllClasses(options) .then(allSchemas => { allSchemas.forEach(schema => { this.data[schema.className] = injectDefaultSchema(schema).fields; @@ -300,20 +306,51 @@ class SchemaController { classLevelPermissions: {} }); }); + delete this.reloadDataPromise; + }, (err) => { + delete this.reloadDataPromise; + throw err; }); + return this.reloadDataPromise; } - getAllClasses() { - return this._dbAdapter.getAllClasses() - .then(allSchemas => allSchemas.map(injectDefaultSchema)); + getAllClasses(options = {clearCache: false}) { + if (options.clearCache) { + this._cache.clear(); + } + return this._cache.getAllClasses().then((allClasses) => { + if (allClasses && allClasses.length && !options.clearCache) { + return Promise.resolve(allClasses); + } + return this._dbAdapter.getAllClasses() + .then(allSchemas => allSchemas.map(injectDefaultSchema)) + .then(allSchemas => { + return this._cache.setAllClasses(allSchemas).then(() => { + return allSchemas; + }); + }) + }); } - getOneSchema(className, allowVolatileClasses = false) { + getOneSchema(className, allowVolatileClasses = false, options = {clearCache: false}) { + if (options.clearCache) { + this._cache.clear(); + } if (allowVolatileClasses && volatileClasses.indexOf(className) > -1) { - return Promise.resolve(this.data[className]); + return Promise.resolve(this.data[className]); } - return this._dbAdapter.getClass(className) - .then(injectDefaultSchema) + return this._cache.getOneSchema(className).then((cached) => { + if (cached && !options.clearCache) { + return Promise.resolve(cached); + } + return this._dbAdapter.getClass(className) + .then(injectDefaultSchema) + .then((result) => { + return this._cache.setOneSchema(className, result).then(() => { + return result; + }) + }); + }); } // Create a new class that includes the three default fields. @@ -331,6 +368,10 @@ class SchemaController { return this._dbAdapter.createClass(className, convertSchemaToAdapterSchema({ fields, classLevelPermissions, className })) .then(convertAdapterSchemaToParseSchema) + .then((res) => { + this._cache.clear(); + return res; + }) .catch(error => { if (error && error.code === Parse.Error.DUPLICATE_VALUE) { throw new Parse.Error(Parse.Error.INVALID_CLASS_NAME, `Class ${className} already exists.`); @@ -376,7 +417,7 @@ class SchemaController { }); return Promise.all(deletePromises) // Delete Everything - .then(() => this.reloadData()) // Reload our Schema, so we have all the new values + .then(() => this.reloadData({ clearCache: true })) // Reload our Schema, so we have all the new values .then(() => { let promises = insertedFields.map(fieldName => { const type = submittedFields[fieldName]; @@ -410,13 +451,13 @@ class SchemaController { // We don't have this class. Update the schema return this.addClassIfNotExists(className) // The schema update succeeded. Reload the schema - .then(() => this.reloadData()) + .then(() => this.reloadData({ clearCache: true })) .catch(error => { // The schema update failed. This can be okay - it might // have failed because there's a race condition and a different // client is making the exact same schema update that we want. // So just reload the schema. - return this.reloadData(); + return this.reloadData({ clearCache: true }); }) .then(() => { // Ensure that the schema now validates @@ -486,7 +527,7 @@ class SchemaController { } validateCLP(perms, newSchema); return this._dbAdapter.setClassLevelPermissions(className, perms) - .then(() => this.reloadData()); + .then(() => this.reloadData({ clearCache: true })); } // Returns a promise that resolves successfully to the new schema @@ -521,23 +562,26 @@ class SchemaController { `schema mismatch for ${className}.${fieldName}; expected ${expectedType.type || expectedType} but got ${type.type}` ); } + return this; } return this._dbAdapter.addFieldIfNotExists(className, fieldName, type).then(() => { // The update succeeded. Reload the schema - return this.reloadData(); + return this.reloadData({ clearCache: true }); }, error => { //TODO: introspect the error and only reload if the error is one for which is makes sense to reload // The update failed. This can be okay - it might have been a race // condition where another client updated the schema in the same // way that we wanted to. So, just reload the schema - return this.reloadData(); + return this.reloadData({ clearCache: true }); }).then(error => { // Ensure that the schema now validates if (!dbTypeMatchesObjectType(this.getExpectedType(className, fieldName), type)) { throw new Parse.Error(Parse.Error.INVALID_JSON, `Could not add field ${fieldName}`); } + // Remove the cached schema + this._cache.clear(); return this; }); }); @@ -562,7 +606,7 @@ class SchemaController { throw new Parse.Error(136, `field ${fieldName} cannot be changed`); } - return this.getOneSchema(className) + return this.getOneSchema(className, false, {clearCache: true}) .catch(error => { if (error === undefined) { throw new Parse.Error(Parse.Error.INVALID_CLASS_NAME, `Class ${className} does not exist.`); @@ -579,8 +623,9 @@ class SchemaController { return database.adapter.deleteFields(className, schema, [fieldName]) .then(() => database.adapter.deleteClass(`_Join:${fieldName}:${className}`)); } - return database.adapter.deleteFields(className, schema, [fieldName]); + }).then(() => { + this._cache.clear(); }); } @@ -711,9 +756,9 @@ class SchemaController { } // Returns a promise for a new Schema. -const load = dbAdapter => { - let schema = new SchemaController(dbAdapter); - return schema.reloadData().then(() => schema); +const load = (dbAdapter, schemaCache, options) => { + let schema = new SchemaController(dbAdapter, schemaCache); + return schema.reloadData(options).then(() => schema); } // Builds a new schema (in schema API response format) out of an diff --git a/src/ParseServer.js b/src/ParseServer.js index 5eb3f1836a..be7d8d1ecd 100644 --- a/src/ParseServer.js +++ b/src/ParseServer.js @@ -55,6 +55,7 @@ import { UsersRouter } from './Routers/UsersRouter'; import { PurgeRouter } from './Routers/PurgeRouter'; import DatabaseController from './Controllers/DatabaseController'; +import SchemaCache from './Controllers/SchemaCache'; const SchemaController = require('./Controllers/SchemaController'); import ParsePushAdapter from 'parse-server-push-adapter'; import MongoStorageAdapter from './Adapters/Storage/Mongo/MongoStorageAdapter'; @@ -139,6 +140,7 @@ class ParseServer { expireInactiveSessions = true, verbose = false, revokeSessionOnPasswordReset = true, + schemaCacheTTL = 5, // cache for 5s __indexBuildCompletionCallbackForTests = () => {}, }) { // Initialize the node client SDK automatically @@ -197,7 +199,7 @@ class ParseServer { const userController = new UserController(emailControllerAdapter, appId, { verifyUserEmails }); const liveQueryController = new LiveQueryController(liveQuery); const cacheController = new CacheController(cacheControllerAdapter, appId); - const databaseController = new DatabaseController(databaseAdapter); + const databaseController = new DatabaseController(databaseAdapter, new SchemaCache(cacheController, schemaCacheTTL)); const hooksController = new HooksController(appId, databaseController, webhookKey); const analyticsController = new AnalyticsController(analyticsControllerAdapter); @@ -254,6 +256,7 @@ class ParseServer { jsonLogs, revokeSessionOnPasswordReset, databaseController, + schemaCacheTTL }); // To maintain compatibility. TODO: Remove in some version that breaks backwards compatability diff --git a/src/Routers/SchemasRouter.js b/src/Routers/SchemasRouter.js index 4024d0a2db..71c40d55ee 100644 --- a/src/Routers/SchemasRouter.js +++ b/src/Routers/SchemasRouter.js @@ -15,15 +15,15 @@ function classNameMismatchResponse(bodyClass, pathClass) { } function getAllSchemas(req) { - return req.config.database.loadSchema() - .then(schemaController => schemaController.getAllClasses()) + return req.config.database.loadSchema({ clearCache: true}) + .then(schemaController => schemaController.getAllClasses(true)) .then(schemas => ({ response: { results: schemas } })); } function getOneSchema(req) { const className = req.params.className; - return req.config.database.loadSchema() - .then(schemaController => schemaController.getOneSchema(className)) + return req.config.database.loadSchema({ clearCache: true}) + .then(schemaController => schemaController.getOneSchema(className, true)) .then(schema => ({ response: schema })) .catch(error => { if (error === undefined) { @@ -46,7 +46,7 @@ function createSchema(req) { throw new Parse.Error(135, `POST ${req.path} needs a class name.`); } - return req.config.database.loadSchema() + return req.config.database.loadSchema({ clearCache: true}) .then(schema => schema.addClassIfNotExists(className, req.body.fields, req.body.classLevelPermissions)) .then(schema => ({ response: schema })); } @@ -59,7 +59,7 @@ function modifySchema(req) { let submittedFields = req.body.fields || {}; let className = req.params.className; - return req.config.database.loadSchema() + return req.config.database.loadSchema({ clearCache: true}) .then(schema => schema.updateClass(className, submittedFields, req.body.classLevelPermissions, req.config.database)) .then(result => ({response: result})); } diff --git a/src/cli/cli-definitions.js b/src/cli/cli-definitions.js index 3fc1e5dccd..f0b7d0deee 100644 --- a/src/cli/cli-definitions.js +++ b/src/cli/cli-definitions.js @@ -197,5 +197,10 @@ export default { env: "PARSE_SERVER_REVOKE_SESSION_ON_PASSWORD_RESET", help: "When a user changes their password, either through the reset password email or while logged in, all sessions are revoked if this is true. Set to false if you don't want to revoke sessions.", action: booleanParser + }, + "schemaCacheTTL": { + env: "PARSE_SERVER_SCHEMA_CACHE_TTL", + help: "The TTL for caching the schema for optimizing read/write operations. You should put a long TTL when your DB is in production. default to 0; disabled.", + action: numberParser("schemaCacheTTL"), } };