From 9ef32c7416ca275f99b3e9b7db508005a8083a54 Mon Sep 17 00:00:00 2001 From: Diamond Lewis Date: Sat, 23 Sep 2017 20:51:19 -0500 Subject: [PATCH 1/7] Support for Aggregate Queries --- spec/ParseQuery.Aggregate.spec.js | 213 ++++++++++++++++++ src/Adapters/Storage/Mongo/MongoCollection.js | 8 + .../Storage/Mongo/MongoStorageAdapter.js | 12 + .../Postgres/PostgresStorageAdapter.js | 118 ++++++++++ src/Controllers/DatabaseController.js | 14 ++ src/ParseServer.js | 6 +- src/RestQuery.js | 2 + src/Routers/AggregateRouter.js | 60 +++++ 8 files changed, 431 insertions(+), 2 deletions(-) create mode 100644 spec/ParseQuery.Aggregate.spec.js create mode 100644 src/Routers/AggregateRouter.js diff --git a/spec/ParseQuery.Aggregate.spec.js b/spec/ParseQuery.Aggregate.spec.js new file mode 100644 index 0000000000..eee3d27a6a --- /dev/null +++ b/spec/ParseQuery.Aggregate.spec.js @@ -0,0 +1,213 @@ +'use strict'; +const Parse = require('parse/node'); +const rp = require('request-promise'); + +const masterKeyHeaders = { + 'X-Parse-Application-Id': 'test', + 'X-Parse-Rest-API-Key': 'test', + 'X-Parse-Master-Key': 'test' +} + +const masterKeyOptions = { + headers: masterKeyHeaders, + json: true +} + +const loadTestData = () => { + const data1 = {score: 10, name: 'foo', sender: {group: 'A'}, size: ['S', 'M']}; + const data2 = {score: 10, name: 'foo', sender: {group: 'A'}, size: ['M', 'L']}; + const data3 = {score: 10, name: 'bar', sender: {group: 'B'}, size: ['S']}; + const data4 = {score: 20, name: 'dpl', sender: {group: 'B'}, size: ['S']}; + const obj1 = new TestObject(data1); + const obj2 = new TestObject(data2); + const obj3 = new TestObject(data3); + const obj4 = new TestObject(data4); + return Parse.Object.saveAll([obj1, obj2, obj3, obj4]); +} + +describe('Parse.Query Aggregate testing', () => { + beforeEach((done) => { + loadTestData().then(done, done); + }); + + it('should only query aggregate with master key', (done) => { + Parse._request('GET', `aggregate/someClass`, {}) + .then(() => {}, (error) => { + expect(error.message).toEqual('unauthorized: master key is required'); + done(); + }); + }); + + it('group sum query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + group: { _id: null, total: { $sum: '$score' } }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results[0].total).toBe(50); + done(); + }).catch(done.fail); + }); + + it('group min query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + group: { _id: null, minScore: { $min: '$score' } }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results[0].minScore).toBe(10); + done(); + }).catch(done.fail); + }); + + it('group max query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + group: { _id: null, maxScore: { $max: '$score' } }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results[0].maxScore).toBe(20); + done(); + }).catch(done.fail); + }); + + it('group avg query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + group: { _id: null, avgScore: { $avg: '$score' } }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results[0].avgScore).toBe(12.5); + done(); + }).catch(done.fail); + }); + + it('limit query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + limit: 2, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results.length).toBe(2); + done(); + }).catch(done.fail); + }); + + it('sort query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + sort: { name: 1 }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results.length).toBe(4); + expect(resp.results[0].name).toBe('bar'); + expect(resp.results[1].name).toBe('dpl'); + expect(resp.results[2].name).toBe('foo'); + expect(resp.results[3].name).toBe('foo'); + done(); + }).catch(done.fail); + }); + + it('skip query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + skip: 2, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results.length).toBe(2); + done(); + }).catch(done.fail); + }); + + it('match query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + match: { score: { $gt: 15 }}, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results.length).toBe(1); + expect(resp.results[0].name).toBe('dpl'); + done(); + }).catch(done.fail); + }); + + it('distinct query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { distinct: 'score' } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results.length).toBe(2); + expect(resp.results.indexOf(10) > -1).toBe(true); + expect(resp.results.indexOf(20) > -1).toBe(true); + done(); + }).catch(done.fail); + }); + + it('distinct nested', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { distinct: 'sender.group' } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results.length).toBe(2); + expect(resp.results.indexOf('A') > -1).toBe(true); + expect(resp.results.indexOf('B') > -1).toBe(true); + done(); + }).catch(done.fail); + }); + + it('distinct class does not exist return empty', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { distinct: 'unknown' } + }); + rp.get(Parse.serverURL + '/aggregate/UnknownClass', options) + .then((resp) => { + expect(resp.results.length).toBe(0); + done(); + }).catch(done.fail); + }); + + it('distinct field does not exist return empty', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { distinct: 'unknown' } + }); + const obj = new TestObject(); + obj.save().then(() => { + return rp.get(Parse.serverURL + '/aggregate/TestObject', options); + }).then((resp) => { + expect(resp.results.length).toBe(0); + done(); + }).catch(done.fail); + }); + + it('distinct array', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { distinct: 'size' } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results.length).toBe(3); + expect(resp.results.indexOf('S') > -1).toBe(true); + expect(resp.results.indexOf('M') > -1).toBe(true); + expect(resp.results.indexOf('L') > -1).toBe(true); + done(); + }).catch(done.fail); + }); +}); diff --git a/src/Adapters/Storage/Mongo/MongoCollection.js b/src/Adapters/Storage/Mongo/MongoCollection.js index ad1b458d25..128395121b 100644 --- a/src/Adapters/Storage/Mongo/MongoCollection.js +++ b/src/Adapters/Storage/Mongo/MongoCollection.js @@ -60,6 +60,14 @@ export default class MongoCollection { return countOperation; } + distinct(field, query) { + return this._mongoCollection.distinct(field, query); + } + + aggregate(pipeline, { maxTimeMS, readPreference } = {}) { + return this._mongoCollection.aggregate(pipeline, { maxTimeMS, readPreference }).toArray(); + } + insertOne(object) { return this._mongoCollection.insertOne(object); } diff --git a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js index 3142712a3a..6c9b14c53a 100644 --- a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js +++ b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js @@ -405,6 +405,18 @@ export class MongoStorageAdapter { })); } + distinct(className, schema, query, fieldName) { + schema = convertParseSchemaToMongoSchema(schema); + return this._adaptiveCollection(className) + .then(collection => collection.distinct(fieldName, transformWhere(className, query, schema))); + } + + aggregate(className, pipeline, readPreference) { + readPreference = this._parseReadPreference(readPreference); + return this._adaptiveCollection(className) + .then(collection => collection.aggregate(pipeline, { readPreference, maxTimeMS: this._maxTimeMS })); + } + _parseReadPreference(readPreference) { if (readPreference) { switch (readPreference) { diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index f2aec54788..bb65945e5b 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -165,6 +165,10 @@ const transformDotField = (fieldName) => { return name; } +const transformAggregateField = (fieldName) => { + return fieldName.substr(1); +} + const validateKeys = (object) => { if (typeof object == 'object') { for (const key in object) { @@ -1366,6 +1370,120 @@ export class PostgresStorageAdapter { }); } + distinct(className, schema, query, fieldName) { + debug('distinct', className, query); + let field = fieldName; + let column = fieldName; + if (fieldName.indexOf('.') >= 0) { + field = transformDotFieldToComponents(fieldName).join('->'); + column = fieldName.split('.')[0]; + } + const isArrayField = schema.fields + && schema.fields[fieldName] + && schema.fields[fieldName].type === 'Array'; + const values = [field, column, className]; + const where = buildWhereClause({ schema, query, index: 4 }); + values.push(...where.values); + + const wherePattern = where.pattern.length > 0 ? `WHERE ${where.pattern}` : ''; + let qs = `SELECT DISTINCT ON ($1:raw) $2:raw FROM $3:name ${wherePattern}`; + if (isArrayField) { + qs = `SELECT distinct jsonb_array_elements($1:raw) as $2:raw FROM $3:name ${wherePattern}`; + } + debug(qs, values); + return this._client.any(qs, values) + .catch(() => []) + .then((results) => { + if (fieldName.indexOf('.') === -1) { + return results.map(object => object[field]); + } + const child = fieldName.split('.')[1]; + return results.map(object => object[column][child]); + }); + } + + // TODO: aggregate transform to SQL + aggregate(className, pipeline) { + debug('aggregate', className, pipeline); + const values = [className]; + const columns = []; + let wherePattern = ''; + let limitPattern = ''; + let skipPattern = ''; + let sortPattern = ''; + let groupPattern = ''; + for (let i = 0; i < pipeline.length; i += 1) { + const stage = pipeline[i]; + if (stage.$group) { + for (const field in stage.$group) { + const value = stage.$group[field]; + if (value === null) { + continue; + } + if (field === '_id') { + columns.push('objectId AS _id'); + groupPattern = `GROUP BY ${className}.${transformAggregateField(value)}`; + continue; + } + if (value.$sum) { + if (typeof value.$sum === 'string') { + columns.push(`SUM(${transformAggregateField(value.$sum)}) AS "${field}"`); + } else { + columns.push(`COUNT(*) AS "${field}"`); + } + } + if (value.$max) { + columns.push(`MAX(${transformAggregateField(value.$max)}) AS "${field}"`); + } + if (value.$min) { + columns.push(`MIN(${transformAggregateField(value.$min)}) AS "${field}"`); + } + if (value.$avg) { + columns.push(`AVG(${transformAggregateField(value.$avg)}) AS "${field}"`); + } + } + columns.join(','); + } else { + columns.push('*'); + } + if (stage.$match) { + const patterns = []; + for (const field in stage.$match) { + const value = stage.$match[field]; + Object.keys(ParseToPosgresComparator).forEach(cmp => { + if (value[cmp]) { + const pgComparator = ParseToPosgresComparator[cmp]; + patterns.push(`${field} ${pgComparator} ${value[cmp]}`); + } + }); + } + wherePattern = patterns.length > 0 ? `WHERE ${patterns.join(' ')}` : ''; + } + if (stage.$limit) { + limitPattern = `LIMIT ${stage.$limit}`; + } + if (stage.$skip) { + skipPattern = `OFFSET ${stage.$skip}`; + } + if (stage.$sort) { + const sort = stage.$sort; + const sorting = Object.keys(sort).map((key) => { + if (sort[key] === 1) { + return `"${key}" ASC`; + } + return `"${key}" DESC`; + }).join(','); + sortPattern = sort !== undefined && Object.keys(sort).length > 0 ? `ORDER BY ${sorting}` : ''; + } + } + + const qs = `SELECT ${columns} FROM $1:name ${wherePattern} ${sortPattern} ${limitPattern} ${skipPattern} ${groupPattern}`; + debug(qs, values); + return this._client.any(qs, values) + .catch(() => []) + .then(results => results); + } + performInitialization({ VolatileClassesSchemas }) { debug('performInitialization'); const promises = VolatileClassesSchemas.map((schema) => { diff --git a/src/Controllers/DatabaseController.js b/src/Controllers/DatabaseController.js index c1cdfdadca..1a120a56a5 100644 --- a/src/Controllers/DatabaseController.js +++ b/src/Controllers/DatabaseController.js @@ -785,6 +785,8 @@ DatabaseController.prototype.find = function(className, query, { count, keys, op, + distinct, + pipeline, readPreference } = {}) { const isMaster = acl === undefined; @@ -853,6 +855,18 @@ DatabaseController.prototype.find = function(className, query, { } else { return this.adapter.count(className, schema, query, readPreference); } + } else if (distinct) { + if (!classExists) { + return []; + } else { + return this.adapter.distinct(className, schema, query, distinct); + } + } else if (pipeline) { + if (!classExists) { + return []; + } else { + return this.adapter.aggregate(className, pipeline, readPreference); + } } else { if (!classExists) { return []; diff --git a/src/ParseServer.js b/src/ParseServer.js index be9d81602d..65f6159fa5 100644 --- a/src/ParseServer.js +++ b/src/ParseServer.js @@ -49,7 +49,8 @@ import { SessionsRouter } from './Routers/SessionsRouter'; import { UserController } from './Controllers/UserController'; import { UsersRouter } from './Routers/UsersRouter'; import { PurgeRouter } from './Routers/PurgeRouter'; -import { AudiencesRouter } from './Routers/AudiencesRouter'; +import { AudiencesRouter } from './Routers/AudiencesRouter'; +import { AggregateRouter } from './Routers/AggregateRouter'; import DatabaseController from './Controllers/DatabaseController'; import SchemaCache from './Controllers/SchemaCache'; @@ -394,7 +395,8 @@ class ParseServer { new PurgeRouter(), new HooksRouter(), new CloudCodeRouter(), - new AudiencesRouter() + new AudiencesRouter(), + new AggregateRouter() ]; const routes = routers.reduce((memo, router) => { diff --git a/src/RestQuery.js b/src/RestQuery.js index 832149b145..a9c5daa64b 100644 --- a/src/RestQuery.js +++ b/src/RestQuery.js @@ -86,6 +86,8 @@ function RestQuery(config, auth, className, restWhere = {}, restOptions = {}, cl case 'count': this.doCount = true; break; + case 'distinct': + case 'pipeline': case 'skip': case 'limit': case 'readPreference': diff --git a/src/Routers/AggregateRouter.js b/src/Routers/AggregateRouter.js new file mode 100644 index 0000000000..7d23f97cbf --- /dev/null +++ b/src/Routers/AggregateRouter.js @@ -0,0 +1,60 @@ +import ClassesRouter from './ClassesRouter'; +import rest from '../rest'; +import * as middleware from '../middlewares'; +import Parse from 'parse/node'; + +const ALLOWED_KEYS = [ + 'where', + 'distinct', + 'match', + 'project', + 'redact', + 'limit', + 'skip', + 'unwind', + 'group', + 'sample', + 'sort', + 'geoNear', + 'lookup', + 'indexStats', + 'out', +]; + +export class AggregateRouter extends ClassesRouter { + + handleFind(req) { + const body = Object.assign(req.body, ClassesRouter.JSONFromQuery(req.query)); + const options = {}; + const pipeline = []; + + for (const key of Object.keys(body)) { + if (ALLOWED_KEYS.indexOf(key) === -1) { + throw new Parse.Error(Parse.Error.INVALID_QUERY, `Invalid parameter for query: ${key}`); + } + const specialKey = `$${key}`; + // Handle $out at the last stage of pipeline + if (key !== 'out') { + pipeline.push({ [specialKey]: body[key] }); + } + } + if (body.out) { + pipeline.push({ $out: body.out }); + } + if (body.distinct) { + options.distinct = String(body.distinct); + } + options.pipeline = pipeline; + if (typeof body.where === 'string') { + body.where = JSON.parse(body.where); + } + return rest.find(req.config, req.auth, this.className(req), body.where, options, req.info.clientSDK) + .then((response) => { return { response }; }); + } + + mountRoutes() { + this.route('GET','/aggregate/:className', middleware.promiseEnforceMasterKeyAccess, req => { return this.handleFind(req); }); + } +} + +export default AggregateRouter; From a138d86cc4f8725adbdde512105e8b0b653e8d3e Mon Sep 17 00:00:00 2001 From: Diamond Lewis Date: Sat, 23 Sep 2017 22:21:11 -0500 Subject: [PATCH 2/7] improve pg and coverage --- spec/ParseQuery.Aggregate.spec.js | 121 ++++++++++++++++-- .../Postgres/PostgresStorageAdapter.js | 34 ++++- 2 files changed, 139 insertions(+), 16 deletions(-) diff --git a/spec/ParseQuery.Aggregate.spec.js b/spec/ParseQuery.Aggregate.spec.js index eee3d27a6a..3d6a9c0b83 100644 --- a/spec/ParseQuery.Aggregate.spec.js +++ b/spec/ParseQuery.Aggregate.spec.js @@ -38,6 +38,19 @@ describe('Parse.Query Aggregate testing', () => { }); }); + it('group by field', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + group: { _id: '$name' }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results.length).toBe(3); + done(); + }).catch(done.fail); + }); + it('group sum query', (done) => { const options = Object.assign({}, masterKeyOptions, { body: { @@ -51,6 +64,19 @@ describe('Parse.Query Aggregate testing', () => { }).catch(done.fail); }); + it('group count query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + group: { _id: null, total: { $sum: 1 } }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results[0].total).toBe(4); + done(); + }).catch(done.fail); + }); + it('group min query', (done) => { const options = Object.assign({}, masterKeyOptions, { body: { @@ -103,7 +129,7 @@ describe('Parse.Query Aggregate testing', () => { }).catch(done.fail); }); - it('sort query', (done) => { + it('sort ascending query', (done) => { const options = Object.assign({}, masterKeyOptions, { body: { sort: { name: 1 }, @@ -120,6 +146,23 @@ describe('Parse.Query Aggregate testing', () => { }).catch(done.fail); }); + it('sort decending query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + sort: { name: -1 }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results.length).toBe(4); + expect(resp.results[0].name).toBe('foo'); + expect(resp.results[1].name).toBe('foo'); + expect(resp.results[2].name).toBe('dpl'); + expect(resp.results[3].name).toBe('bar'); + done(); + }).catch(done.fail); + }); + it('skip query', (done) => { const options = Object.assign({}, masterKeyOptions, { body: { @@ -142,7 +185,51 @@ describe('Parse.Query Aggregate testing', () => { rp.get(Parse.serverURL + '/aggregate/TestObject', options) .then((resp) => { expect(resp.results.length).toBe(1); - expect(resp.results[0].name).toBe('dpl'); + expect(resp.results[0].score).toBe(20); + done(); + }).catch(done.fail); + }); + + it('project query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + project: { name: 1 }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + resp.results.forEach((result) => { + expect(result.name !== undefined).toBe(true); + expect(result.sender).toBe(undefined); + expect(result.size).toBe(undefined); + expect(result.score).toBe(undefined); + }); + done(); + }).catch(done.fail); + }); + + it('class does not exist return empty', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + group: { _id: null, total: { $sum: '$score' } }, + } + }); + rp.get(Parse.serverURL + '/aggregate/UnknownClass', options) + .then((resp) => { + expect(resp.results.length).toBe(0); + done(); + }).catch(done.fail); + }); + + it('field does not exist return empty', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + group: { _id: null, total: { $sum: '$unknownfield' } }, + } + }); + rp.get(Parse.serverURL + '/aggregate/UnknownClass', options) + .then((resp) => { + expect(resp.results.length).toBe(0); done(); }).catch(done.fail); }); @@ -154,8 +241,24 @@ describe('Parse.Query Aggregate testing', () => { rp.get(Parse.serverURL + '/aggregate/TestObject', options) .then((resp) => { expect(resp.results.length).toBe(2); - expect(resp.results.indexOf(10) > -1).toBe(true); - expect(resp.results.indexOf(20) > -1).toBe(true); + expect(resp.results.includes(10)).toBe(true); + expect(resp.results.includes(20)).toBe(true); + done(); + }).catch(done.fail); + }); + + it('distint query with where', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + distinct: 'score', + where: { + name: 'bar' + } + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results[0]).toBe(10); done(); }).catch(done.fail); }); @@ -167,8 +270,8 @@ describe('Parse.Query Aggregate testing', () => { rp.get(Parse.serverURL + '/aggregate/TestObject', options) .then((resp) => { expect(resp.results.length).toBe(2); - expect(resp.results.indexOf('A') > -1).toBe(true); - expect(resp.results.indexOf('B') > -1).toBe(true); + expect(resp.results.includes('A')).toBe(true); + expect(resp.results.includes('B')).toBe(true); done(); }).catch(done.fail); }); @@ -204,9 +307,9 @@ describe('Parse.Query Aggregate testing', () => { rp.get(Parse.serverURL + '/aggregate/TestObject', options) .then((resp) => { expect(resp.results.length).toBe(3); - expect(resp.results.indexOf('S') > -1).toBe(true); - expect(resp.results.indexOf('M') > -1).toBe(true); - expect(resp.results.indexOf('L') > -1).toBe(true); + expect(resp.results.includes('S')).toBe(true); + expect(resp.results.includes('M')).toBe(true); + expect(resp.results.includes('L')).toBe(true); done(); }).catch(done.fail); }); diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index bb65945e5b..29be6ab0f7 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -1406,7 +1406,8 @@ export class PostgresStorageAdapter { aggregate(className, pipeline) { debug('aggregate', className, pipeline); const values = [className]; - const columns = []; + let columns = []; + let countField = null; let wherePattern = ''; let limitPattern = ''; let skipPattern = ''; @@ -1417,18 +1418,19 @@ export class PostgresStorageAdapter { if (stage.$group) { for (const field in stage.$group) { const value = stage.$group[field]; - if (value === null) { + if (value === null || value === undefined) { continue; } if (field === '_id') { - columns.push('objectId AS _id'); - groupPattern = `GROUP BY ${className}.${transformAggregateField(value)}`; + columns.push(`${transformAggregateField(value)} AS "_id"`); + groupPattern = `GROUP BY ${transformAggregateField(value)}`; continue; } if (value.$sum) { if (typeof value.$sum === 'string') { columns.push(`SUM(${transformAggregateField(value.$sum)}) AS "${field}"`); } else { + countField = field; columns.push(`COUNT(*) AS "${field}"`); } } @@ -1446,6 +1448,21 @@ export class PostgresStorageAdapter { } else { columns.push('*'); } + if (stage.$project) { + if (columns.includes('*')) { + columns = []; + } + for (const field in stage.$project) { + const value = stage.$project[field]; + const toInclude = (value === 1 || value === true); + if (columns.includes(field) && toInclude) { + continue; + } + if (toInclude) { + columns.push(field); + } + } + } if (stage.$match) { const patterns = []; for (const field in stage.$match) { @@ -1479,9 +1496,12 @@ export class PostgresStorageAdapter { const qs = `SELECT ${columns} FROM $1:name ${wherePattern} ${sortPattern} ${limitPattern} ${skipPattern} ${groupPattern}`; debug(qs, values); - return this._client.any(qs, values) - .catch(() => []) - .then(results => results); + return this._client.any(qs, values).then(results => { + if (countField) { + results[0][countField] = parseInt(results[0][countField], 10); + } + return results; + }); } performInitialization({ VolatileClassesSchemas }) { From a04ede96931995db2a8ca23c98e7c20186ffa0e0 Mon Sep 17 00:00:00 2001 From: Diamond Lewis Date: Sun, 24 Sep 2017 01:07:56 -0500 Subject: [PATCH 3/7] Mongo 3.4 aggregates and tests --- spec/ParseQuery.Aggregate.spec.js | 27 +++++++++++++++++++++++++++ src/Routers/AggregateRouter.js | 23 ++++++++++++----------- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/spec/ParseQuery.Aggregate.spec.js b/spec/ParseQuery.Aggregate.spec.js index 3d6a9c0b83..e2d2ad2187 100644 --- a/spec/ParseQuery.Aggregate.spec.js +++ b/spec/ParseQuery.Aggregate.spec.js @@ -38,6 +38,19 @@ describe('Parse.Query Aggregate testing', () => { }); }); + it('invalid query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + unknown: {}, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .catch((error) => { + expect(error.error.code).toEqual(Parse.Error.INVALID_QUERY); + done(); + }); + }); + it('group by field', (done) => { const options = Object.assign({}, masterKeyOptions, { body: { @@ -263,6 +276,20 @@ describe('Parse.Query Aggregate testing', () => { }).catch(done.fail); }); + it('distint query with where string', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + distinct: 'score', + where: JSON.stringify({name:'bar'}), + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results[0]).toBe(10); + done(); + }).catch(done.fail); + }); + it('distinct nested', (done) => { const options = Object.assign({}, masterKeyOptions, { body: { distinct: 'sender.group' } diff --git a/src/Routers/AggregateRouter.js b/src/Routers/AggregateRouter.js index 7d23f97cbf..1b17f3f881 100644 --- a/src/Routers/AggregateRouter.js +++ b/src/Routers/AggregateRouter.js @@ -6,8 +6,8 @@ import Parse from 'parse/node'; const ALLOWED_KEYS = [ 'where', 'distinct', - 'match', 'project', + 'match', 'redact', 'limit', 'skip', @@ -17,8 +17,16 @@ const ALLOWED_KEYS = [ 'sort', 'geoNear', 'lookup', - 'indexStats', 'out', + 'indexStats', + 'facet', + 'bucket', + 'bucketAuto', + 'sortByCount', + 'addFields', + 'replaceRoot', + 'count', + 'graphLookup', ]; export class AggregateRouter extends ClassesRouter { @@ -28,18 +36,11 @@ export class AggregateRouter extends ClassesRouter { const options = {}; const pipeline = []; - for (const key of Object.keys(body)) { + for (const key in body) { if (ALLOWED_KEYS.indexOf(key) === -1) { throw new Parse.Error(Parse.Error.INVALID_QUERY, `Invalid parameter for query: ${key}`); } - const specialKey = `$${key}`; - // Handle $out at the last stage of pipeline - if (key !== 'out') { - pipeline.push({ [specialKey]: body[key] }); - } - } - if (body.out) { - pipeline.push({ $out: body.out }); + pipeline.push({ [`$${key}`]: body[key] }); } if (body.distinct) { options.distinct = String(body.distinct); From 8b5e858bb0b13d278067ff40ebab34b60593bf94 Mon Sep 17 00:00:00 2001 From: Diamond Lewis Date: Sun, 24 Sep 2017 13:04:00 -0500 Subject: [PATCH 4/7] replace _id with objectId --- spec/ParseQuery.Aggregate.spec.js | 44 +++++++++++++++---- .../Storage/Mongo/MongoStorageAdapter.js | 11 ++++- .../Postgres/PostgresStorageAdapter.js | 1 - src/Routers/AggregateRouter.js | 16 +++++++ 4 files changed, 61 insertions(+), 11 deletions(-) diff --git a/spec/ParseQuery.Aggregate.spec.js b/spec/ParseQuery.Aggregate.spec.js index e2d2ad2187..0ab99d5ac1 100644 --- a/spec/ParseQuery.Aggregate.spec.js +++ b/spec/ParseQuery.Aggregate.spec.js @@ -38,7 +38,7 @@ describe('Parse.Query Aggregate testing', () => { }); }); - it('invalid query', (done) => { + it('invalid query invalid key', (done) => { const options = Object.assign({}, masterKeyOptions, { body: { unknown: {}, @@ -51,10 +51,36 @@ describe('Parse.Query Aggregate testing', () => { }); }); + it('invalid query group _id', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + group: { _id: null }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .catch((error) => { + expect(error.error.code).toEqual(Parse.Error.INVALID_QUERY); + done(); + }); + }); + + it('invalid query group objectId required', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + group: {}, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .catch((error) => { + expect(error.error.code).toEqual(Parse.Error.INVALID_QUERY); + done(); + }); + }); + it('group by field', (done) => { const options = Object.assign({}, masterKeyOptions, { body: { - group: { _id: '$name' }, + group: { objectId: '$name' }, } }); rp.get(Parse.serverURL + '/aggregate/TestObject', options) @@ -67,7 +93,7 @@ describe('Parse.Query Aggregate testing', () => { it('group sum query', (done) => { const options = Object.assign({}, masterKeyOptions, { body: { - group: { _id: null, total: { $sum: '$score' } }, + group: { objectId: null, total: { $sum: '$score' } }, } }); rp.get(Parse.serverURL + '/aggregate/TestObject', options) @@ -80,7 +106,7 @@ describe('Parse.Query Aggregate testing', () => { it('group count query', (done) => { const options = Object.assign({}, masterKeyOptions, { body: { - group: { _id: null, total: { $sum: 1 } }, + group: { objectId: null, total: { $sum: 1 } }, } }); rp.get(Parse.serverURL + '/aggregate/TestObject', options) @@ -93,7 +119,7 @@ describe('Parse.Query Aggregate testing', () => { it('group min query', (done) => { const options = Object.assign({}, masterKeyOptions, { body: { - group: { _id: null, minScore: { $min: '$score' } }, + group: { objectId: null, minScore: { $min: '$score' } }, } }); rp.get(Parse.serverURL + '/aggregate/TestObject', options) @@ -106,7 +132,7 @@ describe('Parse.Query Aggregate testing', () => { it('group max query', (done) => { const options = Object.assign({}, masterKeyOptions, { body: { - group: { _id: null, maxScore: { $max: '$score' } }, + group: { objectId: null, maxScore: { $max: '$score' } }, } }); rp.get(Parse.serverURL + '/aggregate/TestObject', options) @@ -119,7 +145,7 @@ describe('Parse.Query Aggregate testing', () => { it('group avg query', (done) => { const options = Object.assign({}, masterKeyOptions, { body: { - group: { _id: null, avgScore: { $avg: '$score' } }, + group: { objectId: null, avgScore: { $avg: '$score' } }, } }); rp.get(Parse.serverURL + '/aggregate/TestObject', options) @@ -224,7 +250,7 @@ describe('Parse.Query Aggregate testing', () => { it('class does not exist return empty', (done) => { const options = Object.assign({}, masterKeyOptions, { body: { - group: { _id: null, total: { $sum: '$score' } }, + group: { objectId: null, total: { $sum: '$score' } }, } }); rp.get(Parse.serverURL + '/aggregate/UnknownClass', options) @@ -237,7 +263,7 @@ describe('Parse.Query Aggregate testing', () => { it('field does not exist return empty', (done) => { const options = Object.assign({}, masterKeyOptions, { body: { - group: { _id: null, total: { $sum: '$unknownfield' } }, + group: { objectId: null, total: { $sum: '$unknownfield' } }, } }); rp.get(Parse.serverURL + '/aggregate/UnknownClass', options) diff --git a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js index 6c9b14c53a..78ba9bdd72 100644 --- a/src/Adapters/Storage/Mongo/MongoStorageAdapter.js +++ b/src/Adapters/Storage/Mongo/MongoStorageAdapter.js @@ -414,7 +414,16 @@ export class MongoStorageAdapter { aggregate(className, pipeline, readPreference) { readPreference = this._parseReadPreference(readPreference); return this._adaptiveCollection(className) - .then(collection => collection.aggregate(pipeline, { readPreference, maxTimeMS: this._maxTimeMS })); + .then(collection => collection.aggregate(pipeline, { readPreference, maxTimeMS: this._maxTimeMS })) + .then(results => { + results.forEach(result => { + if (result.hasOwnProperty('_id')) { + result.objectId = result._id; + delete result._id; + } + }); + return results; + }); } _parseReadPreference(readPreference) { diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 29be6ab0f7..58af48b47c 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -1402,7 +1402,6 @@ export class PostgresStorageAdapter { }); } - // TODO: aggregate transform to SQL aggregate(className, pipeline) { debug('aggregate', className, pipeline); const values = [className]; diff --git a/src/Routers/AggregateRouter.js b/src/Routers/AggregateRouter.js index 1b17f3f881..925710135d 100644 --- a/src/Routers/AggregateRouter.js +++ b/src/Routers/AggregateRouter.js @@ -40,6 +40,22 @@ export class AggregateRouter extends ClassesRouter { if (ALLOWED_KEYS.indexOf(key) === -1) { throw new Parse.Error(Parse.Error.INVALID_QUERY, `Invalid parameter for query: ${key}`); } + if (key === 'group') { + if (body[key].hasOwnProperty('_id')) { + throw new Parse.Error( + Parse.Error.INVALID_QUERY, + `Invalid parameter for query: group. Please use objectId instead of _id` + ); + } + if (!body[key].hasOwnProperty('objectId')) { + throw new Parse.Error( + Parse.Error.INVALID_QUERY, + `Invalid parameter for query: group. objectId is required` + ); + } + body[key]._id = body[key].objectId; + delete body[key].objectId; + } pipeline.push({ [`$${key}`]: body[key] }); } if (body.distinct) { From ea3ec598ebc901c1784f9139b7802c9fa7ebca8b Mon Sep 17 00:00:00 2001 From: Diamond Lewis Date: Sun, 24 Sep 2017 13:26:13 -0500 Subject: [PATCH 5/7] improve tests for objectId --- spec/ParseQuery.Aggregate.spec.js | 16 ++++++++++++++++ .../Storage/Postgres/PostgresStorageAdapter.js | 8 ++++++++ 2 files changed, 24 insertions(+) diff --git a/spec/ParseQuery.Aggregate.spec.js b/spec/ParseQuery.Aggregate.spec.js index 0ab99d5ac1..9086c85c7e 100644 --- a/spec/ParseQuery.Aggregate.spec.js +++ b/spec/ParseQuery.Aggregate.spec.js @@ -86,6 +86,12 @@ describe('Parse.Query Aggregate testing', () => { rp.get(Parse.serverURL + '/aggregate/TestObject', options) .then((resp) => { expect(resp.results.length).toBe(3); + expect(resp.results[0].hasOwnProperty('objectId')).toBe(true); + expect(resp.results[1].hasOwnProperty('objectId')).toBe(true); + expect(resp.results[2].hasOwnProperty('objectId')).toBe(true); + expect(resp.results[0].objectId).not.toBe(undefined); + expect(resp.results[1].objectId).not.toBe(undefined); + expect(resp.results[2].objectId).not.toBe(undefined); done(); }).catch(done.fail); }); @@ -98,6 +104,8 @@ describe('Parse.Query Aggregate testing', () => { }); rp.get(Parse.serverURL + '/aggregate/TestObject', options) .then((resp) => { + expect(resp.results[0].hasOwnProperty('objectId')).toBe(true); + expect(resp.results[0].objectId).toBe(null); expect(resp.results[0].total).toBe(50); done(); }).catch(done.fail); @@ -111,6 +119,8 @@ describe('Parse.Query Aggregate testing', () => { }); rp.get(Parse.serverURL + '/aggregate/TestObject', options) .then((resp) => { + expect(resp.results[0].hasOwnProperty('objectId')).toBe(true); + expect(resp.results[0].objectId).toBe(null); expect(resp.results[0].total).toBe(4); done(); }).catch(done.fail); @@ -124,6 +134,8 @@ describe('Parse.Query Aggregate testing', () => { }); rp.get(Parse.serverURL + '/aggregate/TestObject', options) .then((resp) => { + expect(resp.results[0].hasOwnProperty('objectId')).toBe(true); + expect(resp.results[0].objectId).toBe(null); expect(resp.results[0].minScore).toBe(10); done(); }).catch(done.fail); @@ -137,6 +149,8 @@ describe('Parse.Query Aggregate testing', () => { }); rp.get(Parse.serverURL + '/aggregate/TestObject', options) .then((resp) => { + expect(resp.results[0].hasOwnProperty('objectId')).toBe(true); + expect(resp.results[0].objectId).toBe(null); expect(resp.results[0].maxScore).toBe(20); done(); }).catch(done.fail); @@ -150,6 +164,8 @@ describe('Parse.Query Aggregate testing', () => { }); rp.get(Parse.serverURL + '/aggregate/TestObject', options) .then((resp) => { + expect(resp.results[0].hasOwnProperty('objectId')).toBe(true); + expect(resp.results[0].objectId).toBe(null); expect(resp.results[0].avgScore).toBe(12.5); done(); }).catch(done.fail); diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index 58af48b47c..ef4a4fcd1d 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -1499,6 +1499,14 @@ export class PostgresStorageAdapter { if (countField) { results[0][countField] = parseInt(results[0][countField], 10); } + results.forEach(result => { + if (result.hasOwnProperty('_id')) { + result.objectId = result._id; + delete result._id; + } else { + result.objectId = null; + } + }); return results; }); } From 6b7b2cabae04561ad2db7dabd1ff64a9b26d9cce Mon Sep 17 00:00:00 2001 From: Diamond Lewis Date: Sun, 24 Sep 2017 14:09:37 -0500 Subject: [PATCH 6/7] project with group query --- spec/ParseQuery.Aggregate.spec.js | 27 +++++++++++++++++++ .../Postgres/PostgresStorageAdapter.js | 13 +++------ 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/spec/ParseQuery.Aggregate.spec.js b/spec/ParseQuery.Aggregate.spec.js index 9086c85c7e..1578062e76 100644 --- a/spec/ParseQuery.Aggregate.spec.js +++ b/spec/ParseQuery.Aggregate.spec.js @@ -263,6 +263,33 @@ describe('Parse.Query Aggregate testing', () => { }).catch(done.fail); }); + it('project with group query', (done) => { + const options = Object.assign({}, masterKeyOptions, { + body: { + project: { score: 1 }, + group: { objectId: '$score', score: { $sum: '$score' } }, + } + }); + rp.get(Parse.serverURL + '/aggregate/TestObject', options) + .then((resp) => { + expect(resp.results.length).toBe(2); + resp.results.forEach((result) => { + expect(result.hasOwnProperty('objectId')).toBe(true); + expect(result.name).toBe(undefined); + expect(result.sender).toBe(undefined); + expect(result.size).toBe(undefined); + expect(result.score).not.toBe(undefined); + if (result.objectId === 10) { + expect(result.score).toBe(30); + } + if (result.objectId === 20) { + expect(result.score).toBe(20); + } + }); + done(); + }).catch(done.fail); + }); + it('class does not exist return empty', (done) => { const options = Object.assign({}, masterKeyOptions, { body: { diff --git a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js index ef4a4fcd1d..4560ab9b2e 100644 --- a/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js +++ b/src/Adapters/Storage/Postgres/PostgresStorageAdapter.js @@ -1421,7 +1421,7 @@ export class PostgresStorageAdapter { continue; } if (field === '_id') { - columns.push(`${transformAggregateField(value)} AS "_id"`); + columns.push(`${transformAggregateField(value)} AS "objectId"`); groupPattern = `GROUP BY ${transformAggregateField(value)}`; continue; } @@ -1453,11 +1453,7 @@ export class PostgresStorageAdapter { } for (const field in stage.$project) { const value = stage.$project[field]; - const toInclude = (value === 1 || value === true); - if (columns.includes(field) && toInclude) { - continue; - } - if (toInclude) { + if ((value === 1 || value === true)) { columns.push(field); } } @@ -1500,10 +1496,7 @@ export class PostgresStorageAdapter { results[0][countField] = parseInt(results[0][countField], 10); } results.forEach(result => { - if (result.hasOwnProperty('_id')) { - result.objectId = result._id; - delete result._id; - } else { + if (!result.hasOwnProperty('objectId')) { result.objectId = null; } }); From 9c866c7d5e22051980c29d8a9883e08640851b02 Mon Sep 17 00:00:00 2001 From: Diamond Lewis Date: Sat, 4 Nov 2017 14:44:18 -0500 Subject: [PATCH 7/7] typo --- spec/ParseQuery.Aggregate.spec.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/ParseQuery.Aggregate.spec.js b/spec/ParseQuery.Aggregate.spec.js index 1578062e76..1523ba8f77 100644 --- a/spec/ParseQuery.Aggregate.spec.js +++ b/spec/ParseQuery.Aggregate.spec.js @@ -329,7 +329,7 @@ describe('Parse.Query Aggregate testing', () => { }).catch(done.fail); }); - it('distint query with where', (done) => { + it('distinct query with where', (done) => { const options = Object.assign({}, masterKeyOptions, { body: { distinct: 'score', @@ -345,7 +345,7 @@ describe('Parse.Query Aggregate testing', () => { }).catch(done.fail); }); - it('distint query with where string', (done) => { + it('distinct query with where string', (done) => { const options = Object.assign({}, masterKeyOptions, { body: { distinct: 'score',