diff --git a/spec/ParseQuery.Aggregate.spec.js b/spec/ParseQuery.Aggregate.spec.js new file mode 100644 index 0000000000..1523ba8f77 --- /dev/null +++ b/spec/ParseQuery.Aggregate.spec.js @@ -0,0 +1,412 @@ +'use strict'; +const Parse = require('parse/node'); +const rp = require('request-promise'); + +const masterKeyHeaders = { + 'X-Parse-Application-Id': 'test', + 'X-Parse-Rest-API-Key': 'test', + 'X-Parse-Master-Key': 'test' +} + +const masterKeyOptions = { + headers: masterKeyHeaders, + json: true +} + +const loadTestData = () => { + const data1 = {score: 10, name: 'foo', sender: {group: 'A'}, size: ['S', 'M']}; + const data2 = {score: 10, name: 'foo', sender: {group: 'A'}, size: ['M', 'L']}; + const data3 = {score: 10, name: 'bar', sender: {group: 'B'}, size: ['S']}; + const data4 = {score: 20, name: 'dpl', sender: {group: 'B'}, size: ['S']}; + const obj1 = new TestObject(data1); + const obj2 = new TestObject(data2); + const obj3 = new TestObject(data3); + const obj4 = new TestObject(data4); + return Parse.Object.saveAll([obj1, obj2, obj3, obj4]); +} + +describe('Parse.Query Aggregate testing', () => { + beforeEach((done) => { + loadTestData().then(done, done); + }); + + it('should only query aggregate with master key', (done) => { + Parse._request('GET', `aggregate/someClass`, {}) + .then(() => {}, (error) => { + expect(error.message).toEqual('unauthorized: master key is required'); + done(); + }); + }); + + it('invalid query invalid key', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + unknown: {}, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .catch((error) => { + expect(error.error.code).toEqual(Parse.Error.INVALID_QUERY); + done(); + }); + }); + + it('invalid query group _id', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + group: { _id: null }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .catch((error) => { + expect(error.error.code).toEqual(Parse.Error.INVALID_QUERY); + done(); + }); + }); + + it('invalid query group objectId required', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + group: {}, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .catch((error) => { + expect(error.error.code).toEqual(Parse.Error.INVALID_QUERY); + done(); + }); + }); + + it('group by field', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + group: { objectId: '$name' }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results.length).toBe(3); + expect(resp.results[0].hasOwnProperty('objectId')).toBe(true); + expect(resp.results[1].hasOwnProperty('objectId')).toBe(true); + expect(resp.results[2].hasOwnProperty('objectId')).toBe(true); + expect(resp.results[0].objectId).not.toBe(undefined); + expect(resp.results[1].objectId).not.toBe(undefined); + expect(resp.results[2].objectId).not.toBe(undefined); + done(); + }).catch(done.fail); + }); + + it('group sum query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + group: { objectId: null, total: { $sum: '$score' } }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results[0].hasOwnProperty('objectId')).toBe(true); + expect(resp.results[0].objectId).toBe(null); + expect(resp.results[0].total).toBe(50); + done(); + }).catch(done.fail); + }); + + it('group count query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + group: { objectId: null, total: { $sum: 1 } }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results[0].hasOwnProperty('objectId')).toBe(true); + expect(resp.results[0].objectId).toBe(null); + expect(resp.results[0].total).toBe(4); + done(); + }).catch(done.fail); + }); + + it('group min query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + group: { objectId: null, minScore: { $min: '$score' } }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results[0].hasOwnProperty('objectId')).toBe(true); + expect(resp.results[0].objectId).toBe(null); + expect(resp.results[0].minScore).toBe(10); + done(); + }).catch(done.fail); + }); + + it('group max query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + group: { objectId: null, maxScore: { $max: '$score' } }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results[0].hasOwnProperty('objectId')).toBe(true); + expect(resp.results[0].objectId).toBe(null); + expect(resp.results[0].maxScore).toBe(20); + done(); + }).catch(done.fail); + }); + + it('group avg query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + group: { objectId: null, avgScore: { $avg: '$score' } }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results[0].hasOwnProperty('objectId')).toBe(true); + expect(resp.results[0].objectId).toBe(null); + expect(resp.results[0].avgScore).toBe(12.5); + done(); + }).catch(done.fail); + }); + + it('limit query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + limit: 2, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results.length).toBe(2); + done(); + }).catch(done.fail); + }); + + it('sort ascending query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + sort: { name: 1 }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results.length).toBe(4); + expect(resp.results[0].name).toBe('bar'); + expect(resp.results[1].name).toBe('dpl'); + expect(resp.results[2].name).toBe('foo'); + expect(resp.results[3].name).toBe('foo'); + done(); + }).catch(done.fail); + }); + + it('sort decending query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + sort: { name: -1 }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results.length).toBe(4); + expect(resp.results[0].name).toBe('foo'); + expect(resp.results[1].name).toBe('foo'); + expect(resp.results[2].name).toBe('dpl'); + expect(resp.results[3].name).toBe('bar'); + done(); + }).catch(done.fail); + }); + + it('skip query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + skip: 2, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results.length).toBe(2); + done(); + }).catch(done.fail); + }); + + it('match query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + match: { score: { $gt: 15 }}, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results.length).toBe(1); + expect(resp.results[0].score).toBe(20); + done(); + }).catch(done.fail); + }); + + it('project query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + project: { name: 1 }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + resp.results.forEach((result) => { + expect(result.name !== undefined).toBe(true); + expect(result.sender).toBe(undefined); + expect(result.size).toBe(undefined); + expect(result.score).toBe(undefined); + }); + done(); + }).catch(done.fail); + }); + + it('project with group query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + project: { score: 1 }, + group: { objectId: '$score', score: { $sum: '$score' } }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results.length).toBe(2); + resp.results.forEach((result) => { + expect(result.hasOwnProperty('objectId')).toBe(true); + expect(result.name).toBe(undefined); + expect(result.sender).toBe(undefined); + expect(result.size).toBe(undefined); + expect(result.score).not.toBe(undefined); + if (result.objectId === 10) { + expect(result.score).toBe(30); + } + if (result.objectId === 20) { + expect(result.score).toBe(20); + } + }); + done(); + }).catch(done.fail); + }); + + it('class does not exist return empty', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + group: { objectId: null, total: { $sum: '$score' } }, + } + }); + rp.get(Parse.serverURL + '/aggregate/UnknownClass', options) + .then((resp) => { + expect(resp.results.length).toBe(0); + done(); + }).catch(done.fail); + }); + + it('field does not exist return empty', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + group: { objectId: null, total: { $sum: '$unknownfield' } }, + } + }); + rp.get(Parse.serverURL + '/aggregate/UnknownClass', options) + .then((resp) => { + expect(resp.results.length).toBe(0); + done(); + }).catch(done.fail); + }); + + it('distinct query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { distinct: 'score' } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results.length).toBe(2); + expect(resp.results.includes(10)).toBe(true); + expect(resp.results.includes(20)).toBe(true); + done(); + }).catch(done.fail); + }); + + it('distinct query with where', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + distinct: 'score', + where: { + name: 'bar' + } + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results[0]).toBe(10); + done(); + }).catch(done.fail); + }); + + it('distinct query with where string', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + distinct: 'score', + where: JSON.stringify({name:'bar'}), + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results[0]).toBe(10); + done(); + }).catch(done.fail); + }); + + it('distinct nested', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { distinct: 'sender.group' } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results.length).toBe(2); + expect(resp.results.includes('A')).toBe(true); + expect(resp.results.includes('B')).toBe(true); + done(); + }).catch(done.fail); + }); + + it('distinct class does not exist return empty', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { distinct: 'unknown' } + }); + rp.get(Parse.serverURL + '/aggregate/UnknownClass', options) + .then((resp) => { + expect(resp.results.length).toBe(0); + done(); + }).catch(done.fail); + }); + + it('distinct field does not exist return empty', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { distinct: 'unknown' } + }); + const obj = new TestObject(); + obj.save().then(() => { + return rp.get(Parse.serverURL + '/aggregate/TestObject', options); + }).then((resp) => { + expect(resp.results.length).toBe(0); + done(); + }).catch(done.fail); + }); + + it('distinct array', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { distinct: 'size' } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results.length).toBe(3); + expect(resp.results.includes('S')).toBe(true); + expect(resp.results.includes('M')).toBe(true); + expect(resp.results.includes('L')).toBe(true); + done(); + }).catch(done.fail); + }); +}); diff --git a/src/Adapters/Storage/Mongo/MongoCollection.js b/src/Adapters/Storage/Mongo/MongoCollection.js index ad1b458d25..128395121b 100644 --- a/src/Adapters/Storage/Mongo/MongoCollection.js +++ b/src/Adapters/Storage/Mongo/MongoCollection.js @@ -60,6 +60,14 @@ export default class MongoCollection { return countOperation; } + distinct(field, query) { + return this._mongoCollection.distinct(field, query); + } + + aggregate(pipeline, { maxTimeMS, readPreference } = {}) { + return this._mongoCollection.aggregate(pipeline, { maxTimeMS, readPreference }).toArray(); + } + insertOne(object) { return this._mongoCollection.insertOne(object); } diff --git a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js index 3142712a3a..78ba9bdd72 100644 --- a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js +++ b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js @@ -405,6 +405,27 @@ export class MongoStorageAdapter { })); } + distinct(className, schema, query, fieldName) { + schema = convertParseSchemaToMongoSchema(schema); + return this._adaptiveCollection(className) + .then(collection => collection.distinct(fieldName, transformWhere(className, query, schema))); + } + + aggregate(className, pipeline, readPreference) { + readPreference = this._parseReadPreference(readPreference); + return this._adaptiveCollection(className) + .then(collection => collection.aggregate(pipeline, { readPreference, maxTimeMS: this._maxTimeMS })) + .then(results => { + results.forEach(result => { + if (result.hasOwnProperty('_id')) { + result.objectId = result._id; + delete result._id; + } + }); + return results; + }); + } + _parseReadPreference(readPreference) { if (readPreference) { switch (readPreference) { diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index f2aec54788..4560ab9b2e 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -165,6 +165,10 @@ const transformDotField = (fieldName) => { return name; } +const transformAggregateField = (fieldName) => { + return fieldName.substr(1); +} + const validateKeys = (object) => { if (typeof object == 'object') { for (const key in object) { @@ -1366,6 +1370,140 @@ export class PostgresStorageAdapter { }); } + distinct(className, schema, query, fieldName) { + debug('distinct', className, query); + let field = fieldName; + let column = fieldName; + if (fieldName.indexOf('.') >= 0) { + field = transformDotFieldToComponents(fieldName).join('->'); + column = fieldName.split('.')[0]; + } + const isArrayField = schema.fields + && schema.fields[fieldName] + && schema.fields[fieldName].type === 'Array'; + const values = [field, column, className]; + const where = buildWhereClause({ schema, query, index: 4 }); + values.push(...where.values); + + const wherePattern = where.pattern.length > 0 ? `WHERE ${where.pattern}` : ''; + let qs = `SELECT DISTINCT ON ($1:raw) $2:raw FROM $3:name ${wherePattern}`; + if (isArrayField) { + qs = `SELECT distinct jsonb_array_elements($1:raw) as $2:raw FROM $3:name ${wherePattern}`; + } + debug(qs, values); + return this._client.any(qs, values) + .catch(() => []) + .then((results) => { + if (fieldName.indexOf('.') === -1) { + return results.map(object => object[field]); + } + const child = fieldName.split('.')[1]; + return results.map(object => object[column][child]); + }); + } + + aggregate(className, pipeline) { + debug('aggregate', className, pipeline); + const values = [className]; + let columns = []; + let countField = null; + let wherePattern = ''; + let limitPattern = ''; + let skipPattern = ''; + let sortPattern = ''; + let groupPattern = ''; + for (let i = 0; i < pipeline.length; i += 1) { + const stage = pipeline[i]; + if (stage.$group) { + for (const field in stage.$group) { + const value = stage.$group[field]; + if (value === null || value === undefined) { + continue; + } + if (field === '_id') { + columns.push(`${transformAggregateField(value)} AS "objectId"`); + groupPattern = `GROUP BY ${transformAggregateField(value)}`; + continue; + } + if (value.$sum) { + if (typeof value.$sum === 'string') { + columns.push(`SUM(${transformAggregateField(value.$sum)}) AS "${field}"`); + } else { + countField = field; + columns.push(`COUNT(*) AS "${field}"`); + } + } + if (value.$max) { + columns.push(`MAX(${transformAggregateField(value.$max)}) AS "${field}"`); + } + if (value.$min) { + columns.push(`MIN(${transformAggregateField(value.$min)}) AS "${field}"`); + } + if (value.$avg) { + columns.push(`AVG(${transformAggregateField(value.$avg)}) AS "${field}"`); + } + } + columns.join(','); + } else { + columns.push('*'); + } + if (stage.$project) { + if (columns.includes('*')) { + columns = []; + } + for (const field in stage.$project) { + const value = stage.$project[field]; + if ((value === 1 || value === true)) { + columns.push(field); + } + } + } + if (stage.$match) { + const patterns = []; + for (const field in stage.$match) { + const value = stage.$match[field]; + Object.keys(ParseToPosgresComparator).forEach(cmp => { + if (value[cmp]) { + const pgComparator = ParseToPosgresComparator[cmp]; + patterns.push(`${field} ${pgComparator} ${value[cmp]}`); + } + }); + } + wherePattern = patterns.length > 0 ? `WHERE ${patterns.join(' ')}` : ''; + } + if (stage.$limit) { + limitPattern = `LIMIT ${stage.$limit}`; + } + if (stage.$skip) { + skipPattern = `OFFSET ${stage.$skip}`; + } + if (stage.$sort) { + const sort = stage.$sort; + const sorting = Object.keys(sort).map((key) => { + if (sort[key] === 1) { + return `"${key}" ASC`; + } + return `"${key}" DESC`; + }).join(','); + sortPattern = sort !== undefined && Object.keys(sort).length > 0 ? `ORDER BY ${sorting}` : ''; + } + } + + const qs = `SELECT ${columns} FROM $1:name ${wherePattern} ${sortPattern} ${limitPattern} ${skipPattern} ${groupPattern}`; + debug(qs, values); + return this._client.any(qs, values).then(results => { + if (countField) { + results[0][countField] = parseInt(results[0][countField], 10); + } + results.forEach(result => { + if (!result.hasOwnProperty('objectId')) { + result.objectId = null; + } + }); + return results; + }); + } + performInitialization({ VolatileClassesSchemas }) { debug('performInitialization'); const promises = VolatileClassesSchemas.map((schema) => { diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index c1cdfdadca..1a120a56a5 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -785,6 +785,8 @@ DatabaseController.prototype.find = function(className, query, { count, keys, op, + distinct, + pipeline, readPreference } = {}) { const isMaster = acl === undefined; @@ -853,6 +855,18 @@ DatabaseController.prototype.find = function(className, query, { } else { return this.adapter.count(className, schema, query, readPreference); } + } else if (distinct) { + if (!classExists) { + return []; + } else { + return this.adapter.distinct(className, schema, query, distinct); + } + } else if (pipeline) { + if (!classExists) { + return []; + } else { + return this.adapter.aggregate(className, pipeline, readPreference); + } } else { if (!classExists) { return []; diff --git a/src/ParseServer.js b/src/ParseServer.js index f24d8ec154..76631ea3d9 100644 --- a/src/ParseServer.js +++ b/src/ParseServer.js @@ -34,6 +34,7 @@ import { SessionsRouter } from './Routers/SessionsRouter'; import { UsersRouter } from './Routers/UsersRouter'; import { PurgeRouter } from './Routers/PurgeRouter'; import { AudiencesRouter } from './Routers/AudiencesRouter'; +import { AggregateRouter } from './Routers/AggregateRouter'; import { ParseServerRESTController } from './ParseServerRESTController'; import * as controllers from './Controllers'; @@ -197,7 +198,8 @@ class ParseServer { new PurgeRouter(), new HooksRouter(), new CloudCodeRouter(), - new AudiencesRouter() + new AudiencesRouter(), + new AggregateRouter() ]; const routes = routers.reduce((memo, router) => { diff --git a/src/RestQuery.js b/src/RestQuery.js index 7126814152..8ddda22cba 100644 --- a/src/RestQuery.js +++ b/src/RestQuery.js @@ -86,6 +86,8 @@ function RestQuery(config, auth, className, restWhere = {}, restOptions = {}, cl case 'count': this.doCount = true; break; + case 'distinct': + case 'pipeline': case 'skip': case 'limit': case 'readPreference': diff --git a/src/Routers/AggregateRouter.js b/src/Routers/AggregateRouter.js new file mode 100644 index 0000000000..925710135d --- /dev/null +++ b/src/Routers/AggregateRouter.js @@ -0,0 +1,77 @@ +import ClassesRouter from './ClassesRouter'; +import rest from '../rest'; +import * as middleware from '../middlewares'; +import Parse from 'parse/node'; + +const ALLOWED_KEYS = [ + 'where', + 'distinct', + 'project', + 'match', + 'redact', + 'limit', + 'skip', + 'unwind', + 'group', + 'sample', + 'sort', + 'geoNear', + 'lookup', + 'out', + 'indexStats', + 'facet', + 'bucket', + 'bucketAuto', + 'sortByCount', + 'addFields', + 'replaceRoot', + 'count', + 'graphLookup', +]; + +export class AggregateRouter extends ClassesRouter { + + handleFind(req) { + const body = Object.assign(req.body, ClassesRouter.JSONFromQuery(req.query)); + const options = {}; + const pipeline = []; + + for (const key in body) { + if (ALLOWED_KEYS.indexOf(key) === -1) { + throw new Parse.Error(Parse.Error.INVALID_QUERY, `Invalid parameter for query: ${key}`); + } + if (key === 'group') { + if (body[key].hasOwnProperty('_id')) { + throw new Parse.Error( + Parse.Error.INVALID_QUERY, + `Invalid parameter for query: group. Please use objectId instead of _id` + ); + } + if (!body[key].hasOwnProperty('objectId')) { + throw new Parse.Error( + Parse.Error.INVALID_QUERY, + `Invalid parameter for query: group. objectId is required` + ); + } + body[key]._id = body[key].objectId; + delete body[key].objectId; + } + pipeline.push({ [`$${key}`]: body[key] }); + } + if (body.distinct) { + options.distinct = String(body.distinct); + } + options.pipeline = pipeline; + if (typeof body.where === 'string') { + body.where = JSON.parse(body.where); + } + return rest.find(req.config, req.auth, this.className(req), body.where, options, req.info.clientSDK) + .then((response) => { return { response }; }); + } + + mountRoutes() { + this.route('GET','/aggregate/:className', middleware.promiseEnforceMasterKeyAccess, req => { return this.handleFind(req); }); + } +} + +export default AggregateRouter;