From 878718d84ed45edc9949320d68cb873e0ea1ceb8 Mon Sep 17 00:00:00 2001 From: Brage Staven Date: Tue, 2 Aug 2016 00:40:00 +0200 Subject: [PATCH 1/4] Stream video with GridStoreAdapter --- spec/ParseFile.spec.js | 21 +++++ src/Adapters/Files/GridStoreAdapter.js | 110 +++++++++++++++++++++++++ src/Controllers/FilesController.js | 13 +++ src/Routers/FilesRouter.js | 28 ++++--- 4 files changed, 162 insertions(+), 10 deletions(-) diff --git a/spec/ParseFile.spec.js b/spec/ParseFile.spec.js index 90731c0941..9ab8a06629 100644 --- a/spec/ParseFile.spec.js +++ b/spec/ParseFile.spec.js @@ -491,6 +491,27 @@ describe('Parse.File testing', () => { }); }); + it('supports byte-range requests when requesting a video', done => { + var headers = { + 'Content-Type': 'video/mp4', + 'X-Parse-Application-Id': 'test', + 'X-Parse-REST-API-Key': 'rest' + }; + request.post({ + headers: headers, + url: 'http://localhost:8378/1/files/file', + body: '101010101001010101010101010101010010110101010101010101010' + }, (error, response, body) => { + expect(error).toBe(null); + var b = JSON.parse(body); + request.get(b.url, (error, response, body) =>{ + expect(response.headers['content-type']).toMatch(/^video\/mp4/); + expect(response.headers['accept-ranges']).toMatch(/^bytes/); + done(); + }); + }); + }); + it_exclude_dbs(['postgres'])('creates correct url for old files hosted on files.parsetfss.com', done => { var file = { __type: 'File', diff --git a/src/Adapters/Files/GridStoreAdapter.js b/src/Adapters/Files/GridStoreAdapter.js index d7844a0b5f..2dc46b1528 100644 --- a/src/Adapters/Files/GridStoreAdapter.js +++ b/src/Adapters/Files/GridStoreAdapter.js @@ -67,6 +67,116 @@ export class GridStoreAdapter extends FilesAdapter { getFileLocation(config, filename) { return (config.mount + '/files/' + config.applicationId + '/' + encodeURIComponent(filename)); } + + handleVideoStream(filename, range, res, contentType) { + return this._connect().then(database => { + return GridStore.exist(database, filename) + .then(() => { + let gridStore = new GridStore(database, filename, 'r'); + gridStore.open((err, gridFile) => { + if(!gridFile) { + res.status(404); + res.set('Content-Type', 'text/plain'); + res.end('File not found.'); + return; + } + streamVideo(gridFile,range, res, contentType); + }); + }); + }); + } +} + + /** + * streamVideo is licensed under Creative Commons Attribution 4.0 International License (https://creativecommons.org/licenses/by/4.0/). + * Author: LEROIB at weightingformypizza.(https://weightingformypizza.wordpress.com/2015/06/24/stream-html5-media-content-like-video-audio-from-mongodb-using-express-and-gridstore/) + */ +function streamVideo(gridFile, range, res, contentType) { + var buffer_size = 1024 * 1024;//1024Kb + if (range != null) { + // Range request, partiall stream the file + var parts = range.replace(/bytes=/, "").split("-"); + var partialstart = parts[0]; + var partialend = parts[1]; + var start = partialstart ? parseInt(partialstart, 10) : 0; + var end = partialend ? parseInt(partialend, 10) : gridFile.length - 1; + var chunksize = (end - start) + 1; + + if(chunksize == 1){ + start = 0; + partialend = false; + } + + if(!partialend){ + if(((gridFile.length-1) - start) < (buffer_size)){ + end = gridFile.length - 1; + }else{ + end = start + (buffer_size); + } + chunksize = (end - start) + 1; + } + + if(start == 0 && end == 2){ + chunksize = 1; + } + + res.writeHead(206, { + 'Content-Range': 'bytes ' + start + '-' + end + '/' + gridFile.length, + 'Accept-Ranges': 'bytes', + 'Content-Length': chunksize, + 'Content-Type': contentType, + }); + + gridFile.seek(start, function () { + // get gridFile stream + var stream = gridFile.stream(true); + var ended = false; + var bufferIdx = 0; + var bufferAvail = 0; + var range = (end - start) + 1; + var totalbyteswanted = (end - start) + 1; + var totalbyteswritten = 0; + // write to response + stream.on('data', function (buff) { + bufferAvail += buff.length; + //Ok check if we have enough to cover our range + if(bufferAvail < range) { + //Not enough bytes to satisfy our full range + if(bufferAvail > 0) + { + //Write full buffer + res.write(buff); + totalbyteswritten += buff.length; + range -= buff.length; + bufferIdx += buff.length; + bufferAvail -= buff.length; + } + } + else{ + //Enough bytes to satisfy our full range! + if(bufferAvail > 0) { + var buffer = buff.slice(0,range); + res.write(buffer); + totalbyteswritten += buffer.length; + bufferIdx += range; + bufferAvail -= range; + } + } + if(totalbyteswritten >= totalbyteswanted) { + // totalbytes = 0; + gridFile.close(); + res.end(); + this.destroy(); + } + }); + }); + }else{ + // stream back whole file + res.header("Accept-Ranges", "bytes"); + res.header('Content-Type', contentType); + res.header('Content-Length', gridFile.length); + var stream = gridFile.stream(true).pipe(res); + } } export default GridStoreAdapter; diff --git a/src/Controllers/FilesController.js b/src/Controllers/FilesController.js index a355396e14..df1931444f 100644 --- a/src/Controllers/FilesController.js +++ b/src/Controllers/FilesController.js @@ -82,6 +82,19 @@ export class FilesController extends AdaptableController { expectedAdapterType() { return FilesAdapter; } + + + /** + * Stream video file by serving data in chunks if FilesAdapter is GridStoreAdapter. + * If not; handle the request as usual with "getFileData". + */ + handleVideoStream(filename, range, res, contentType) { + if (this.adapter.constructor.name == 'GridStoreAdapter') { + return this.adapter.handleVideoStream(filename,range,res,contentType); + }else{ + return this.adapter.getFileData(filename); + } + } } export default FilesController; diff --git a/src/Routers/FilesRouter.js b/src/Routers/FilesRouter.js index 160574e1b7..776dd5654d 100644 --- a/src/Routers/FilesRouter.js +++ b/src/Routers/FilesRouter.js @@ -36,16 +36,24 @@ export class FilesRouter { const config = new Config(req.params.appId); const filesController = config.filesController; const filename = req.params.filename; - filesController.getFileData(config, filename).then((data) => { - res.status(200); - var contentType = mime.lookup(filename); - res.set('Content-Type', contentType); - res.end(data); - }).catch((err) => { - res.status(404); - res.set('Content-Type', 'text/plain'); - res.end('File not found.'); - }); + const contentType = mime.lookup(filename); + if (contentType == 'video/mp4' || contentType == 'video/quicktime') { + filesController.handleVideoStream(filename, req.get("Range"), res, contentType).catch((err) => { + res.status(404); + res.set('Content-Type', 'text/plain'); + res.end('File not found.'); + }); + }else{ + filesController.getFileData(config, filename).then((data) => { + res.status(200); + res.set('Content-Type', contentType); + res.end(data); + }).catch((err) => { + res.status(404); + res.set('Content-Type', 'text/plain'); + res.end('File not found.'); + }); + } } createHandler(req, res, next) { From 0a94b9bdb1320f5b568f2281629be5ad7cbb6dd3 Mon Sep 17 00:00:00 2001 From: Brage Staven Date: Tue, 9 Aug 2016 03:47:24 +0200 Subject: [PATCH 2/4] fixing nits. Removing test(Range not accepted as header) --- spec/ParseFile.spec.js | 21 ----- src/Adapters/Files/GridStoreAdapter.js | 111 ++----------------------- src/Controllers/FilesController.js | 13 +-- src/Routers/FilesRouter.js | 98 ++++++++++++++++++++-- 4 files changed, 98 insertions(+), 145 deletions(-) diff --git a/spec/ParseFile.spec.js b/spec/ParseFile.spec.js index 9ab8a06629..90731c0941 100644 --- a/spec/ParseFile.spec.js +++ b/spec/ParseFile.spec.js @@ -491,27 +491,6 @@ describe('Parse.File testing', () => { }); }); - it('supports byte-range requests when requesting a video', done => { - var headers = { - 'Content-Type': 'video/mp4', - 'X-Parse-Application-Id': 'test', - 'X-Parse-REST-API-Key': 'rest' - }; - request.post({ - headers: headers, - url: 'http://localhost:8378/1/files/file', - body: '101010101001010101010101010101010010110101010101010101010' - }, (error, response, body) => { - expect(error).toBe(null); - var b = JSON.parse(body); - request.get(b.url, (error, response, body) =>{ - expect(response.headers['content-type']).toMatch(/^video\/mp4/); - expect(response.headers['accept-ranges']).toMatch(/^bytes/); - done(); - }); - }); - }); - it_exclude_dbs(['postgres'])('creates correct url for old files hosted on files.parsetfss.com', done => { var file = { __type: 'File', diff --git a/src/Adapters/Files/GridStoreAdapter.js b/src/Adapters/Files/GridStoreAdapter.js index 2dc46b1528..8586c7a8e0 100644 --- a/src/Adapters/Files/GridStoreAdapter.js +++ b/src/Adapters/Files/GridStoreAdapter.js @@ -68,114 +68,13 @@ export class GridStoreAdapter extends FilesAdapter { return (config.mount + '/files/' + config.applicationId + '/' + encodeURIComponent(filename)); } - handleVideoStream(filename, range, res, contentType) { + getFileRange(filename: string) { return this._connect().then(database => { - return GridStore.exist(database, filename) - .then(() => { - let gridStore = new GridStore(database, filename, 'r'); - gridStore.open((err, gridFile) => { - if(!gridFile) { - res.status(404); - res.set('Content-Type', 'text/plain'); - res.end('File not found.'); - return; - } - streamVideo(gridFile,range, res, contentType); - }); - }); - }); - } -} - - /** - * streamVideo is licensed under Creative Commons Attribution 4.0 International License (https://creativecommons.org/licenses/by/4.0/). - * Author: LEROIB at weightingformypizza.(https://weightingformypizza.wordpress.com/2015/06/24/stream-html5-media-content-like-video-audio-from-mongodb-using-express-and-gridstore/) - */ -function streamVideo(gridFile, range, res, contentType) { - var buffer_size = 1024 * 1024;//1024Kb - if (range != null) { - // Range request, partiall stream the file - var parts = range.replace(/bytes=/, "").split("-"); - var partialstart = parts[0]; - var partialend = parts[1]; - var start = partialstart ? parseInt(partialstart, 10) : 0; - var end = partialend ? parseInt(partialend, 10) : gridFile.length - 1; - var chunksize = (end - start) + 1; - - if(chunksize == 1){ - start = 0; - partialend = false; - } - - if(!partialend){ - if(((gridFile.length-1) - start) < (buffer_size)){ - end = gridFile.length - 1; - }else{ - end = start + (buffer_size); - } - chunksize = (end - start) + 1; - } - - if(start == 0 && end == 2){ - chunksize = 1; - } - - res.writeHead(206, { - 'Content-Range': 'bytes ' + start + '-' + end + '/' + gridFile.length, - 'Accept-Ranges': 'bytes', - 'Content-Length': chunksize, - 'Content-Type': contentType, + return GridStore.exist(database, filename).then(() => { + let gridStore = new GridStore(database, filename, 'r'); + return gridStore.open(); }); - - gridFile.seek(start, function () { - // get gridFile stream - var stream = gridFile.stream(true); - var ended = false; - var bufferIdx = 0; - var bufferAvail = 0; - var range = (end - start) + 1; - var totalbyteswanted = (end - start) + 1; - var totalbyteswritten = 0; - // write to response - stream.on('data', function (buff) { - bufferAvail += buff.length; - //Ok check if we have enough to cover our range - if(bufferAvail < range) { - //Not enough bytes to satisfy our full range - if(bufferAvail > 0) - { - //Write full buffer - res.write(buff); - totalbyteswritten += buff.length; - range -= buff.length; - bufferIdx += buff.length; - bufferAvail -= buff.length; - } - } - else{ - //Enough bytes to satisfy our full range! - if(bufferAvail > 0) { - var buffer = buff.slice(0,range); - res.write(buffer); - totalbyteswritten += buffer.length; - bufferIdx += range; - bufferAvail -= range; - } - } - if(totalbyteswritten >= totalbyteswanted) { - // totalbytes = 0; - gridFile.close(); - res.end(); - this.destroy(); - } - }); - }); - }else{ - // stream back whole file - res.header("Accept-Ranges", "bytes"); - res.header('Content-Type', contentType); - res.header('Content-Length', gridFile.length); - var stream = gridFile.stream(true).pipe(res); + }); } } diff --git a/src/Controllers/FilesController.js b/src/Controllers/FilesController.js index df1931444f..4eafde3987 100644 --- a/src/Controllers/FilesController.js +++ b/src/Controllers/FilesController.js @@ -83,17 +83,8 @@ export class FilesController extends AdaptableController { return FilesAdapter; } - - /** - * Stream video file by serving data in chunks if FilesAdapter is GridStoreAdapter. - * If not; handle the request as usual with "getFileData". - */ - handleVideoStream(filename, range, res, contentType) { - if (this.adapter.constructor.name == 'GridStoreAdapter') { - return this.adapter.handleVideoStream(filename,range,res,contentType); - }else{ - return this.adapter.getFileData(filename); - } + getFileRange(config, filename) { + return this.adapter.getFileRange(filename); } } diff --git a/src/Routers/FilesRouter.js b/src/Routers/FilesRouter.js index 776dd5654d..41f680ec50 100644 --- a/src/Routers/FilesRouter.js +++ b/src/Routers/FilesRouter.js @@ -37,13 +37,19 @@ export class FilesRouter { const filesController = config.filesController; const filename = req.params.filename; const contentType = mime.lookup(filename); - if (contentType == 'video/mp4' || contentType == 'video/quicktime') { - filesController.handleVideoStream(filename, req.get("Range"), res, contentType).catch((err) => { - res.status(404); - res.set('Content-Type', 'text/plain'); - res.end('File not found.'); - }); - }else{ + if (req.get['Range']) { + if (typeof filesController.adapter.constructor.name !== 'undefined') { + if (filesController.adapter.constructor.name == 'GridStoreAdapter') { + filesController.getFileRange(config, filename).then((gridFile) => { + handleRangeRequest(gridFile, req, res, contentType); + }).catch((err) => { + res.status(404); + res.set('Content-Type', 'text/plain'); + res.end('File not found.'); + }); + } + } + } else { filesController.getFileData(config, filename).then((data) => { res.status(200); res.set('Content-Type', contentType); @@ -101,3 +107,81 @@ export class FilesRouter { }); } } + +function handleRangeRequest(gridFile, req, res, contentType) { + var buffer_size = 1024 * 1024;//1024Kb + // Range request, partiall stream the file + var parts = req.get["Range"].replace(/bytes=/, "").split("-"); + var partialstart = parts[0]; + var partialend = parts[1]; + var start = partialstart ? parseInt(partialstart, 10) : 0; + var end = partialend ? parseInt(partialend, 10) : gridFile.length - 1; + var chunksize = (end - start) + 1; + + if (chunksize == 1) { + start = 0; + partialend = false; + } + + if (!partialend) { + if (((gridFile.length-1) - start) < (buffer_size)) { + end = gridFile.length - 1; + }else{ + end = start + (buffer_size); + } + chunksize = (end - start) + 1; + } + + if (start == 0 && end == 2) { + chunksize = 1; + } + + res.writeHead(206, { + 'Content-Range': 'bytes ' + start + '-' + end + '/' + gridFile.length, + 'Accept-Ranges': 'bytes', + 'Content-Length': chunksize, + 'Content-Type': contentType, + }); + + gridFile.seek(start, function () { + // get gridFile stream + var stream = gridFile.stream(true); + var ended = false; + var bufferIdx = 0; + var bufferAvail = 0; + var range = (end - start) + 1; + var totalbyteswanted = (end - start) + 1; + var totalbyteswritten = 0; + // write to response + stream.on('data', function (buff) { + bufferAvail += buff.length; + //Ok check if we have enough to cover our range + if (bufferAvail < range) { + //Not enough bytes to satisfy our full range + if (bufferAvail > 0) { + //Write full buffer + res.write(buff); + totalbyteswritten += buff.length; + range -= buff.length; + bufferIdx += buff.length; + bufferAvail -= buff.length; + } + }else{ + //Enough bytes to satisfy our full range! + if (bufferAvail > 0) { + const buffer = buff.slice(0,range); + res.write(buffer); + totalbyteswritten += buffer.length; + bufferIdx += range; + bufferAvail -= range; + } + } + if (totalbyteswritten >= totalbyteswanted) { + //totalbytes = 0; + gridFile.close(); + res.end(); + this.destroy(); + } + }); + }); +} From b63938311f0aa2c6ea4d283c0dfd900a94fcf1fe Mon Sep 17 00:00:00 2001 From: Brage Staven Date: Tue, 9 Aug 2016 04:08:10 +0200 Subject: [PATCH 3/4] nit --- src/Routers/FilesRouter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Routers/FilesRouter.js b/src/Routers/FilesRouter.js index 41f680ec50..5d4ff74859 100644 --- a/src/Routers/FilesRouter.js +++ b/src/Routers/FilesRouter.js @@ -129,7 +129,7 @@ function handleRangeRequest(gridFile, req, res, contentType) { }else{ end = start + (buffer_size); } - chunksize = (end - start) + 1; + chunksize = (end - start) + 1; } if (start == 0 && end == 2) { From a77fa217f75b814c7aec7c752846db503411b89c Mon Sep 17 00:00:00 2001 From: Brage Staven Date: Tue, 9 Aug 2016 23:36:19 +0200 Subject: [PATCH 4/4] Changed names. Added function to check if stream-requirements is fulfilled. --- src/Adapters/Files/GridStoreAdapter.js | 2 +- src/Controllers/FilesController.js | 4 +- src/Routers/FilesRouter.js | 56 ++++++++++++++++---------- 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/src/Adapters/Files/GridStoreAdapter.js b/src/Adapters/Files/GridStoreAdapter.js index 8586c7a8e0..da7754a8fc 100644 --- a/src/Adapters/Files/GridStoreAdapter.js +++ b/src/Adapters/Files/GridStoreAdapter.js @@ -68,7 +68,7 @@ export class GridStoreAdapter extends FilesAdapter { return (config.mount + '/files/' + config.applicationId + '/' + encodeURIComponent(filename)); } - getFileRange(filename: string) { + getFileStream(filename: string) { return this._connect().then(database => { return GridStore.exist(database, filename).then(() => { let gridStore = new GridStore(database, filename, 'r'); diff --git a/src/Controllers/FilesController.js b/src/Controllers/FilesController.js index 4eafde3987..6bbeb4377d 100644 --- a/src/Controllers/FilesController.js +++ b/src/Controllers/FilesController.js @@ -83,8 +83,8 @@ export class FilesController extends AdaptableController { return FilesAdapter; } - getFileRange(config, filename) { - return this.adapter.getFileRange(filename); + getFileStream(config, filename) { + return this.adapter.getFileStream(filename); } } diff --git a/src/Routers/FilesRouter.js b/src/Routers/FilesRouter.js index 92fc9b93b6..d116776dfc 100644 --- a/src/Routers/FilesRouter.js +++ b/src/Routers/FilesRouter.js @@ -37,18 +37,14 @@ export class FilesRouter { const filesController = config.filesController; const filename = req.params.filename; const contentType = mime.lookup(filename); - if (req.get['Range']) { - if (typeof filesController.adapter.constructor.name !== 'undefined') { - if (filesController.adapter.constructor.name == 'GridStoreAdapter') { - filesController.getFileRange(config, filename).then((gridFile) => { - handleRangeRequest(gridFile, req, res, contentType); - }).catch((err) => { - res.status(404); - res.set('Content-Type', 'text/plain'); - res.end('File not found.'); - }); - } - } + if (isFileStreamable(req, filesController)) { + filesController.getFileStream(config, filename).then((stream) => { + handleFileStream(stream, req, res, contentType); + }).catch((err) => { + res.status(404); + res.set('Content-Type', 'text/plain'); + res.end('File not found.'); + }); } else { filesController.getFileData(config, filename).then((data) => { res.status(200); @@ -109,14 +105,30 @@ export class FilesRouter { } } -function handleRangeRequest(gridFile, req, res, contentType) { +function isFileStreamable(req, filesController){ + if (req.get['Range']) { + if (!(typeof filesController.adapter.getFileStream === 'function')) { + return false; + } + if (typeof filesController.adapter.constructor.name !== 'undefined') { + if (filesController.adapter.constructor.name == 'GridStoreAdapter') { + return true; + } + } + } + return false; +} + +// handleFileStream is licenced under Creative Commons Attribution 4.0 International License (https://creativecommons.org/licenses/by/4.0/). +// Author: LEROIB at weightingformypizza (https://weightingformypizza.wordpress.com/2015/06/24/stream-html5-media-content-like-video-audio-from-mongodb-using-express-and-gridstore/). +function handleFileStream(stream, req, res, contentType) { var buffer_size = 1024 * 1024;//1024Kb // Range request, partiall stream the file var parts = req.get["Range"].replace(/bytes=/, "").split("-"); var partialstart = parts[0]; var partialend = parts[1]; var start = partialstart ? parseInt(partialstart, 10) : 0; - var end = partialend ? parseInt(partialend, 10) : gridFile.length - 1; + var end = partialend ? parseInt(partialend, 10) : stream.length - 1; var chunksize = (end - start) + 1; if (chunksize == 1) { @@ -125,8 +137,8 @@ function handleRangeRequest(gridFile, req, res, contentType) { } if (!partialend) { - if (((gridFile.length-1) - start) < (buffer_size)) { - end = gridFile.length - 1; + if (((stream.length-1) - start) < (buffer_size)) { + end = stream.length - 1; }else{ end = start + (buffer_size); } @@ -138,15 +150,15 @@ function handleRangeRequest(gridFile, req, res, contentType) { } res.writeHead(206, { - 'Content-Range': 'bytes ' + start + '-' + end + '/' + gridFile.length, + 'Content-Range': 'bytes ' + start + '-' + end + '/' + stream.length, 'Accept-Ranges': 'bytes', 'Content-Length': chunksize, 'Content-Type': contentType, }); - gridFile.seek(start, function () { + stream.seek(start, function () { // get gridFile stream - var stream = gridFile.stream(true); + var gridFileStream = stream.stream(true); var ended = false; var bufferIdx = 0; var bufferAvail = 0; @@ -154,7 +166,7 @@ function handleRangeRequest(gridFile, req, res, contentType) { var totalbyteswanted = (end - start) + 1; var totalbyteswritten = 0; // write to response - stream.on('data', function (buff) { + gridFileStream.on('data', function (buff) { bufferAvail += buff.length; //Ok check if we have enough to cover our range if (bufferAvail < range) { @@ -167,7 +179,7 @@ function handleRangeRequest(gridFile, req, res, contentType) { bufferIdx += buff.length; bufferAvail -= buff.length; } - }else{ + } else { //Enough bytes to satisfy our full range! if (bufferAvail > 0) { const buffer = buff.slice(0,range); @@ -179,7 +191,7 @@ function handleRangeRequest(gridFile, req, res, contentType) { } if (totalbyteswritten >= totalbyteswanted) { //totalbytes = 0; - gridFile.close(); + stream.close(); res.end(); this.destroy(); }