Comparing version 0.0.2-6 to 0.2.0-11
@@ -13,26 +13,74 @@ var mongodb = require("mongodb"); | ||
var MAX_MONGO_QUEUE_LENGTH = 3000; | ||
var LOGLEVEL_ERROR = 0; | ||
var LOGLEVEL_WARNING = 1; | ||
var LOGLEVEL_INFO = 2; | ||
var LOGLEVEL_DEBUG = 3; | ||
var constants = {"FH_LOWEST_VERSION": 1, "FH_THUMBNAIL_APPEND":"thumbnail_"}; | ||
var constants = {"LOG_LEVEL": LOGLEVEL_ERROR, "LOWEST_VERSION": 1, "THUMBNAIL_PREPEND":"thumbnail_", "DEFAULT_THUMBNAIL_WIDTH": 200, "DEFAULT_THUMBNAIL_HEIGHT":200, "MAX_MONGO_QUEUE_LENGTH": 1000, "ROOT_COLLECTION": "fileStorage"}; | ||
var MongoFileHandler = function(){ | ||
var defaultLogger = undefined; | ||
var MongoFileHandler = function(config, logger){ | ||
if(logger){ | ||
defaultLogger = logger; | ||
}else { | ||
defaultLogger = { | ||
"info": (constants.LOG_LEVEL >= LOGLEVEL_INFO)?function (msg) { | ||
console.log("INFO ", msg); | ||
}:function(){}, | ||
"debug": (constants.LOG_LEVEL >= LOGLEVEL_DEBUG)?function (msg) { | ||
console.log("DEBUG ", msg); | ||
}:function(){}, | ||
"warning": (constants.LOG_LEVEL >= LOGLEVEL_WARNING)?function (msg) { | ||
console.log("WARNING", msg); | ||
}:function(){}, | ||
"error": (constants.LOG_LEVEL >= LOGLEVEL_ERROR)?function (msg) { | ||
console.log("ERROR ", msg); | ||
}:function(){} | ||
}; | ||
} | ||
if(config && config.fileStorage){ | ||
var fileStorageConfig = config.fileStorage; | ||
for (var configKey in fileStorageConfig){ | ||
constants[configKey] = fileStorageConfig[configKey]; | ||
} | ||
} | ||
} | ||
MongoFileHandler.prototype.saveFile = function(db, fileName, fileReadStream, options, cb){ | ||
//Need to pause the stream on first tick or it will stop when passed into other functions | ||
if(!cb || typeof(cb) != "function"){ | ||
return new Error("Callback function is required for saveFile request."); | ||
} | ||
if(!db || !fileName || !fileReadStream || !options){ | ||
return cb(new Error("Incorrect parameters for saveFile request")); | ||
} | ||
fileReadStream.pause(); | ||
saveFile(db, fileName, fileReadStream, options, cb); | ||
} | ||
MongoFileHandler.prototype.deleteFile = function(db, fileName, version, cb){ | ||
console.log("In deleteFile"); | ||
MongoFileHandler.prototype.deleteFile = function(db, searchOptions, cb){ | ||
defaultLogger.debug("In deleteFile"); | ||
fileExists(db, fileName, function(err, exists){ | ||
if(err) return cb(err); | ||
if(!cb || typeof(cb) != "function"){ | ||
return new Error("Callback function is required for deleteFile request."); | ||
} | ||
if(!db || !searchOptions || !searchOptions.groupId){ | ||
return cb(new Error("Incorrect parameters for deleteFile request")); | ||
} | ||
fileExists(db, searchOptions, function(err, exists){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
if(exists){ | ||
deleteFile(db, fileName, version, cb); | ||
deleteFile(db, searchOptions, cb); | ||
}else{ | ||
cb(new Error("No file exists with the name " + fileName)); | ||
cb(new Error("No file exists with the groupId " + groupId)); | ||
} | ||
@@ -42,14 +90,22 @@ }); | ||
MongoFileHandler.prototype.deleteAllFileVersions = function(db, fileName, cb){ | ||
console.log("In deleteAllFileVersions"); | ||
MongoFileHandler.prototype.deleteAllFileVersions = function(db, groupId, cb){ | ||
defaultLogger.debug("In deleteAllFileVersions"); | ||
fileExists(db, fileName, function(err, exists){ | ||
if(err) return cb(err); | ||
if(!cb || typeof(cb) != "function"){ | ||
return new Error("Callback function is required for deleteAllFileVersions request."); | ||
} | ||
if(!db || !groupId){ | ||
return cb(new Error("Incorrect parameters for deleteAllFileVersions request")); | ||
} | ||
fileExists(db, {"groupId": groupId}, function(err, exists){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
if(exists){ | ||
getAllFileVersions(db, fileName, function(err, fileVersions){ | ||
if(err) return cb(err); | ||
getAllFileVersions(db, groupId, function(err, fileVersions){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
async.eachSeries(fileVersions, function(fileVersion, cb){ | ||
deleteFile(db, fileName, fileVersion, cb); | ||
deleteFile(db, {"groupId": groupId, "version": fileVersion}, cb); | ||
}, function(err){ | ||
@@ -65,72 +121,47 @@ cb(err); | ||
MongoFileHandler.prototype.migrateFiles = function(dbFrom, dbTo, listOfFileIds, cb){ | ||
console.log("In migrateFiles"); | ||
console.log(listOfFileIds); | ||
MongoFileHandler.prototype.migrateFiles = function(dbFrom, dbTo, migrationEntries, cb){ | ||
defaultLogger.debug("In migrateFiles"); | ||
var migratedFiles = {}; | ||
//Check that all of the files listed exist | ||
//Guarding against a file not existing. | ||
async.eachSeries(listOfFileIds, function(fileEntry, cb){ | ||
var fileId = fileEntry; | ||
if(!cb || typeof(cb) != "function"){ | ||
return new Error("Callback function is required for migrateFiles request."); | ||
} | ||
fileExists(dbFrom, fileId, function(err, exists){ | ||
if(err) return cb(err); | ||
if(!dbFrom || !dbTo || !migrationEntries){ | ||
return cb(new Error("Incorrect parameters for migrateFiles request ")) | ||
} | ||
if(!exists){ | ||
cb(new Error("File " + fileId + " does not exist")); | ||
}else{ | ||
cb(); | ||
} | ||
}); | ||
}, function(err){ | ||
if(err){ | ||
return cb(err); | ||
} | ||
sanityCheckMigrateFilesList(dbFrom, migrationEntries, function(err){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
async.eachSeries(listOfFileIds, function(oldFileId, cb){ | ||
//getFileStream | ||
//saveFile | ||
async.eachSeries(migrationEntries, function(oldFileEntry, cb){ | ||
getFileDetails(dbFrom, oldFileEntry, {}, function(err, oldFileDetails){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
getFileStream(dbFrom, oldFileEntry, {}, function(err, fileStream){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
getFileDetails(dbFrom, {"fileId": oldFileId}, {}, function(err, oldFileDetails){ | ||
if(err) return cb(err); | ||
getFileStream(dbFrom, oldFileId, {}, function(err, fileStream){ | ||
if(err) return cb(err); | ||
fileStream.pause(); | ||
var fileFromName = oldFileDetails.filename; | ||
saveFile(dbTo, fileFromName, fileStream, {}, function(err, newFileId){ | ||
if(err) return cb(err); | ||
saveFile(dbTo, oldFileDetails.filename, fileStream, {"groupId": oldFileEntry.groupId, "migratingFile": true}, function(err, migratedFile){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
migratedFiles[oldFileId] = newFileId; | ||
if(oldFileDetails.metadata.thumbnail){ | ||
generateThumbnailFileName(oldFileDetails.filename, function(thumbFileName){ | ||
getFileStream(dbFrom, {"groupId": oldFileDetails.metadata.groupId, "version": oldFileDetails.metadata.version}, {"thumbnail": true}, function(err, thumbFileStream){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
//If there is a thumbnail associated with a file it must be migrated also. | ||
var fileThumbnailId = oldFileDetails.metadata.thumbnail; | ||
if(fileThumbnailId){ | ||
getFileStream(dbFrom, fileThumbnailId, {}, function(err, thumbnailFileStream){ | ||
if(err) return cb(err); | ||
saveFile(dbTo, thumbFileName, thumbFileStream, {"migratingThumbnail": true, "migratedThumbnailVersion": migratedFile.version}, function(err, thumbFileDetails){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
//Pausing the stream before sending it to saveFile function | ||
thumbnailFileStream.pause(); | ||
var gridStoreThumb = new GridStore(dbFrom, fileThumbnailId, "r"); | ||
gridStoreThumb.open(function(err, thumbnailFile){ | ||
if(err) return cb(err); | ||
var gridStore = new GridStore(dbTo, migratedFile._id, "w+", {"root": constants.ROOT_COLLECTION}); | ||
gridStore.open(function(err, file){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
var thumbnailName = thumbnailFile.filename; | ||
gridStoreThumb.close(function(err, result){ | ||
if(err) return cb(err); | ||
file.metadata.thumbnail = thumbFileDetails._id; | ||
migratedFile.thumbnailHash = thumbFileDetails.hash; | ||
migratedFile.thumbnailFileName = thumbFileDetails.fileName; | ||
saveFile(dbTo, thumbnailName, thumbnailFileStream, {}, function(err, newThumbnailFileId){ | ||
if(err) return cb(err); | ||
//Save the new thumbnail | ||
var newFileGridStore = new GridStore(dbTo, newFileId, "w+"); | ||
newFileGridStore.open(function(err, newFile){ | ||
if(err) return cb(err); | ||
newFile.metadata.thumbnail = newThumbnailFileId; | ||
newFileGridStore.close(function(err, result){ | ||
cb(err); | ||
}); | ||
gridStore.close(function(err, result){ | ||
migratedFiles[oldFileDetails.metadata.groupId] = migratedFile; | ||
cb(err); | ||
}); | ||
@@ -141,3 +172,4 @@ }); | ||
}); | ||
}else{ | ||
} else { // No thumbnail -- finished with this file. | ||
migratedFiles[oldFileDetails.metadata.groupId] = migratedFile; | ||
cb(err); | ||
@@ -154,72 +186,76 @@ } | ||
MongoFileHandler.prototype.getFileVersions = function(db, fileName, cb){ | ||
console.log("In getFileVersions"); | ||
MongoFileHandler.prototype.getFileHistory = function(db, groupId, cb){ | ||
defaultLogger.debug("In getFileHistory"); | ||
var fileDetails = []; | ||
fileExists(db, fileName, function(err, exists){ | ||
if(err) return cb(err); | ||
if(!db || !groupId || !cb){ | ||
return cb(new Error("Incorrect parameters for getFileHistory request ")) | ||
} | ||
if(exists){ | ||
getAllFileVersions(db, fileName, cb); | ||
}else{ | ||
cb(new Error("File " + fileName + " does not exist")); | ||
} | ||
getFileDetails(db, {"groupId": groupId}, {"allMatchingFiles": true}, function(err, allGroupFileDetails){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
async.eachSeries(allGroupFileDetails, function(fileEntry, cb){ | ||
var returnEntry = {"fileName": fileEntry.filename, "version": fileEntry.metadata.version, "contentType": fileEntry.contentType, "groupId": fileEntry.metadata.groupId, "hash": fileEntry.md5}; | ||
fileDetails.push(returnEntry); | ||
cb(undefined); | ||
}, function(err){ | ||
cb(err, fileDetails); | ||
}); | ||
}); | ||
} | ||
MongoFileHandler.prototype.getFileIds = function(db, fileEntries, cb){ | ||
console.log("In getFileIds"); | ||
if(!(fileEntries && Array.isArray(fileEntries))){ | ||
return cb(new Error("No file details array specified")); | ||
MongoFileHandler.prototype.streamFile = function(db, searchOptions, fileOptions, cb){ | ||
defaultLogger.debug("In streamFile"); | ||
if(!cb || typeof(cb) != "function"){ | ||
return new Error("Callback function is required for streamFile request."); | ||
} | ||
//Holding the necessary fileIds | ||
var fileIds = {}; | ||
if(!db || !searchOptions || !fileOptions){ | ||
return cb(new Error("Incorrect parameters for streamFile request ")) | ||
} | ||
//Guarding against a file not existing. | ||
async.eachSeries(fileEntries, function(fileEntry, cb){ | ||
var fileName = fileEntry.name; | ||
fileExists(db, searchOptions, function(err, exists){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
fileExists(db, fileName, function(err, exists){ | ||
if(err) return cb(err); | ||
if(exists){ | ||
getFileStream(db, searchOptions, fileOptions, cb); | ||
}else { | ||
cb(new Error("The file with params " + JSON.stringify(searchOptions) + " does not exist.")); | ||
} | ||
}); | ||
} | ||
if(!exists){ | ||
cb(new Error("File " + fileName + " does not exist")); | ||
}else{ | ||
cb(); | ||
} | ||
}); | ||
}, function(err){ | ||
if(err){ | ||
return cb(err); | ||
} | ||
MongoFileHandler.prototype.exportFilesStream = function(db, exportEntries, zipStream, cb){ | ||
defaultLogger.debug("In exportFilesStream"); | ||
if(!cb || typeof(cb) != "function"){ | ||
return new Error("Callback function is required for exportFilesStream request."); | ||
} | ||
async.eachSeries(fileEntries, function(fileEntry, cb){ | ||
var fileName = fileEntry.name; | ||
var fileVersion = fileEntry.version; | ||
var fileThumbnail = fileEntry.thumbnail; | ||
if(!db || !exportEntries || !zipStream){ | ||
return cb(new Error("Incorrect parameters for exportFilesStream request ")) | ||
} | ||
var fileOptions = {}; | ||
fileOptions.thumbnail = fileThumbnail; | ||
if(fileVersion){ | ||
fileOptions.version = fileVersion; | ||
} | ||
var gridFileStream = new GridFsStream(db, mongodb); | ||
getFileDetails(db, {"fileName": fileName}, fileOptions, function(err, fileDetails){ | ||
if(err) return cb(err); | ||
sanityCheckExportParameters(db, exportEntries, function(err){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
async.eachSeries(exportEntries, function(fileEntry, cb){ | ||
if(fileIds[fileName]){ | ||
fileIds[fileName][fileDetails.metadata.version] = fileDetails._id; | ||
}else{ | ||
fileIds[fileName] = {}; | ||
fileIds[fileName][fileDetails.metadata.version] = fileDetails._id; | ||
} | ||
getFileDetails(db, fileEntry, {}, function(err, fileDetails){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
var fileReadStream = gridFileStream.createReadStream({"_id": fileDetails._id, "root": constants.ROOT_COLLECTION}); | ||
cb(err); | ||
//adding this file into the zipStream | ||
zipStream.append(fileReadStream, {"name": fileDetails.filename}, cb); | ||
}); | ||
}, function(err){ | ||
cb(err, fileIds); | ||
cb(err); | ||
}); | ||
@@ -229,92 +265,112 @@ }); | ||
MongoFileHandler.prototype.streamFile = function(db, fileId, options, cb){ | ||
console.log("In streamFile"); | ||
function sanityCheckExportParameters(db, exportEntries, cb){ | ||
var groupIdEntries = {}; | ||
async.eachSeries(exportEntries, function(fileEntry, cb){ | ||
//GroupId is required to export a file | ||
if(!(fileEntry.groupId)){ | ||
return cb(new Error("Export must contain a groupId. Aborting. " + JSON.stringify(fileEntry))); | ||
} | ||
//Can't migrate two files with the same groupId | ||
if(groupIdEntries[fileEntry.groupId]){ | ||
return cb(new Error("Duplicate groupId entries for groupId " + fileEntry.groupId)); | ||
} else { | ||
groupIdEntries[fileEntry.groupId] = true; | ||
} | ||
fileExists(db, fileId, function(err, exists){ | ||
if(err) return cb(err); | ||
fileExists(db, fileEntry, function(err, exists){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
if(exists){ | ||
getFileStream(db, fileId, options, cb); | ||
}else { | ||
cb(new Error("The file with id " + fileId + " does not exist.")); | ||
} | ||
if(!exists){ | ||
return cb(new Error("File " + JSON.stringify(fileEntry) + " does not exist")); | ||
}else{ | ||
return cb(); | ||
} | ||
}); | ||
}, function(err){ | ||
return cb(err); | ||
}); | ||
} | ||
MongoFileHandler.prototype.exportFilesStream = function(db, listOfFileIds, zipStream, cb){ | ||
console.log("In exportFilesStream"); | ||
var gridFileStream = new GridFsStream(db, mongodb); | ||
function sanityCheckMigrateFilesList(db, migrationEntries, cb){ | ||
async.eachSeries(listOfFileIds, function(fileEntry, cb){ | ||
var groupIdEntries = {}; | ||
async.eachSeries(migrationEntries, function(fileEntry, cb){ | ||
fileExists(db, fileEntry.fileId, function(err, exists){ | ||
if(err) return cb(err); | ||
//Should not be able to migrate files by hash value. | ||
if(fileEntry.hash){ | ||
return cb(new Error("Cannot migrate files by hash value. Aborting. " + fileEntry.hash)); | ||
} | ||
//GroupId is required to migrate a file | ||
if(!(fileEntry.groupId)){ | ||
return cb(new Error("Migration must contain a groupId. Aborting. " + JSON.stringify(fileEntry))); | ||
} | ||
//Can't migrate two files with the same groupId | ||
if(groupIdEntries[fileEntry.groupId]){ | ||
return cb(new Error("Duplicate groupId entries for groupId " + fileEntry.groupId)); | ||
} else { | ||
groupIdEntries[fileEntry.groupId] = true; | ||
} | ||
fileExists(db, fileEntry, function(err, exists){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
if(!exists){ | ||
return cb(new Error("No File with id " + fileEntry.fileId + " exists.")); | ||
return cb(new Error("File " + JSON.stringify(fileEntry) + " does not exist")); | ||
}else{ | ||
return cb(); | ||
} | ||
}); | ||
}, function(err){ | ||
return cb(err); | ||
}); | ||
} | ||
getFileDetails(db, {"fileId": fileEntry.fileId}, {}, function(err, fileDetails){ | ||
if(err) return cb(err); | ||
function saveFile(db, fileName, fileReadStream, options, cb){ | ||
defaultLogger.debug("In saveFile"); | ||
var fileReadStream = gridFileStream.createReadStream({"_id": fileEntry.fileId}); | ||
var thumbnailReadStream; | ||
if(!options.groupId){// If there is no groupId specified, then need to create a new file | ||
createNewFile(db, fileName, fileReadStream, options, function(err, result){ | ||
return cb(err, result); | ||
}); | ||
}else{ | ||
fileExists(db, {"groupId": options.groupId}, function(err, exists){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
//If there is a thumbnail file, then this should be added also. | ||
if(fileDetails.metadata.thumbnail){ | ||
zipStream.append(fileReadStream, {"name": fileEntry.fileName}, function(err){ | ||
if(err) return cb(err); | ||
generateThumbnailFileName(fileEntry.fileName, function(thumbnailFileName){ | ||
thumbnailReadStream = gridFileStream.createReadStream({"_id": fileDetails.metadata.thumbnail}); | ||
zipStream.append(thumbnailReadStream, {"name": thumbnailFileName}, function(err){ | ||
if(err) return cb(err); | ||
cb(err); | ||
}); | ||
}); | ||
if(exists){ | ||
updateExistingFile(db, fileName, fileReadStream, options, function(err, result){ | ||
return cb(err, result); | ||
}); | ||
}else{ | ||
if(options.migratingFile){//Exception to the rule that the groupId is required is when a file is being migrated. If that is the case and the group does not exist, just create the file | ||
createNewFile(db, fileName, fileReadStream, options, function(err, result){ | ||
return cb(err, result); | ||
}); | ||
}else{ | ||
//adding this file into the zipStream | ||
zipStream.append(fileReadStream, {"name": fileEntry.fileName}, cb); | ||
return cb(new Error("A groupId parameter " + options.groupId + " was specified. No file group exists for this groupId. Aborting.")); | ||
} | ||
}) | ||
} | ||
}); | ||
} | ||
}, function(err){ | ||
cb(err); | ||
}); | ||
} | ||
function saveFile(db, fileName, fileReadStream, options, cb){ | ||
console.log("In saveFile"); | ||
fileExists(db, fileName, function(err, exists){ | ||
if(err) return cb(err); | ||
if(exists){ | ||
updateExistingFile(db, fileName, fileReadStream, options, function(err, result){ | ||
cb(err, result); | ||
}) | ||
}else{ | ||
createNewFile(db, fileName, fileReadStream, options, function(err, result){ | ||
cb(err, result); | ||
}); | ||
} | ||
}); | ||
} | ||
function getAllFileVersions(db, fileName, cb){ | ||
function getAllFileVersions(db, groupId, cb){ | ||
console.log("In getAllFileVersions"); | ||
var gridStore = new GridStore(db, fileName, "r"); | ||
defaultLogger.debug("In getAllFileVersions"); | ||
var gridStore = new GridStore(db, null, "r", {"root": constants.ROOT_COLLECTION}); | ||
gridStore.collection(function(err, collection){ | ||
if(err) return cb(err); | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
collection.find({"filename":fileName},{sort:{"metadata.version":-1},"fields": {"metadata.version":1}}, function(err, files){ | ||
if(err) return cb(err); | ||
collection.find({"metadata.groupId": groupId},{sort:{"metadata.version":-1},"fields": {"metadata.version":1}}, function(err, files){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
files.toArray(function(err, filesArray){ | ||
if(err) return cb(err); | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
if(!(filesArray && Array.isArray(filesArray) && filesArray.length > 0)) return cb(new Error("Collection query is empty for file " + fileName)); | ||
@@ -330,3 +386,3 @@ | ||
}, function(err){ | ||
if(err) return cb(err); | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
@@ -342,15 +398,15 @@ gridStore.close(function(err, result){ | ||
function deleteFile(db, searchOptions, cb){ | ||
defaultLogger.debug("In deleteFile"); | ||
getFileDetails(db, searchOptions, {}, function(err, fileDetails){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
function deleteFile(db, fileName, version, cb){ | ||
console.log("In deleteFile"); | ||
getFileDetails(db, {"fileName": fileName}, {"version":version}, function(err, fileDetails){ | ||
if(err) return cb(err); | ||
var gridStore = new GridStore(db, fileDetails._id,"r"); | ||
var gridStore = new GridStore(db, fileDetails._id, "r", {"root": constants.ROOT_COLLECTION}); | ||
gridStore.open(function(err, openGridStore){ | ||
if(err) return cb(err); | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
openGridStore.unlink(function(err, result){ | ||
if(err) return cb(err); | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
@@ -366,9 +422,9 @@ gridStore.close(function(err, result){ | ||
function generateThumbnailFileName(fileName, cb){ | ||
console.log("In generateThumbnailFileName"); | ||
return cb(constants.FH_THUMBNAIL_APPEND + fileName); | ||
defaultLogger.debug("In generateThumbnailFileName"); | ||
return cb(constants.THUMBNAIL_PREPEND + fileName); | ||
} | ||
function createImageThumbnail(db, fileName, fileId, version, options, cb){ | ||
function createImageThumbnail(db, fileName, fullImageDetails, version, options, cb){ | ||
//Need to pipe the readStream to graphics magic, that will then pipe it to storage | ||
console.log("In createImageThumbnail"); | ||
defaultLogger.debug("In createImageThumbnail"); | ||
var gridFileStream = new GridFsStream(db, mongodb); | ||
@@ -379,22 +435,27 @@ | ||
generateThumbnailFileName(fileName, function(thumbFileName){ | ||
var thumbnailFileReadStream = gridFileStream.createReadStream({"_id": fileId}); | ||
var thumbnailFileWriteStream = gridFileStream.createWriteStream({"mode": "w", "_id": new ObjectId(), "filename":thumbFileName, "content_type":fileContentType, "metadata":{"version": version}, "limit": MAX_MONGO_QUEUE_LENGTH}); | ||
console.log("After generateThumbnailFileName", thumbFileName, options.thumbnail.width, options.thumbnail.height); | ||
//Default thumbnail size if not specified | ||
if(!options.thumbnail.width) options.thumbnail.width = constants.DEFAULT_THUMBNAIL_WIDTH; | ||
if(!options.thumbnail.height) options.thumbnail.height = constants.DEFAULT_THUMBNAIL_HEIGHT; | ||
var thumbnailFileReadStream = gridFileStream.createReadStream({"_id": fullImageDetails._id, "root": constants.ROOT_COLLECTION}); | ||
var thumbnailFileWriteStream = gridFileStream.createWriteStream({"mode": "w", "_id": new ObjectId(), "filename":thumbFileName, "content_type":fileContentType, "metadata":{"version": version, "width": options.thumbnail.width, "height": options.thumbnail.height, "isThumbnail": true}, "limit": constants.MAX_MONGO_QUEUE_LENGTH, "root": constants.ROOT_COLLECTION}); | ||
defaultLogger.debug("After generateThumbnailFileName", thumbFileName, options.thumbnail.width, options.thumbnail.height); | ||
//Creating a thumbnail from the input stream, process and then storing. | ||
if(!options.thumbnail.width) options.thumbnail.width = 200; | ||
if(!options.thumbnail.height) options.thumbnail.height = 200; | ||
//When it's finished, can call the callback | ||
thumbnailFileWriteStream.on("close", function(thumbnailFile){ | ||
//Setting the thumbnail Id of the file | ||
console.log("thumbnail fileWriteStream Closed"); | ||
var gridStore = new GridStore(db, fileId, "w+"); | ||
defaultLogger.debug("thumbnail fileWriteStream Closed"); | ||
var gridStore = new GridStore(db, fullImageDetails._id, "w+", {"root": constants.ROOT_COLLECTION}); | ||
gridStore.open(function(err, parentFile){ | ||
if(err) return cb(err); | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
parentFile.metadata.thumbnail = thumbnailFile._id; | ||
gridStore.close(function(err, fileObject){ | ||
if(err) return cb(err); | ||
return cb(undefined, fileId); | ||
gridStore.close(function(err){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
fullImageDetails["thumbnailHash"] = thumbnailFile.md5; | ||
fullImageDetails["thumbnailFileName"] = thumbnailFile.filename; | ||
return cb(undefined, fullImageDetails); | ||
}); | ||
@@ -410,5 +471,4 @@ }); | ||
function createFileWithVersion(db, fileName, fileStream, version, options, cb){ | ||
console.log("In createFileWithVersion"); | ||
defaultLogger.debug("In createFileWithVersion"); | ||
var gridFileStream = new GridFsStream(db, mongodb); | ||
@@ -419,14 +479,19 @@ | ||
var fileObjectId = new ObjectId(); | ||
var groupId = fileObjectId; | ||
if(options.groupId){ | ||
groupId = options.groupId; | ||
} | ||
//If it is an image and options.thumbnail is true | ||
if(fileContentType.indexOf("image/") != -1 && options.thumbnail){ | ||
console.log("File " + fileName + " IS AN IMAGE"); | ||
defaultLogger.debug("File " + fileName + " IS AN IMAGE"); | ||
console.log("AFTER createImageThumbnail"); | ||
defaultLogger.debug("AFTER createImageThumbnail"); | ||
var fileWriteStream = gridFileStream.createWriteStream({"mode": "w", "_id": new ObjectId(), "filename":fileName, "content_type": fileContentType.toString(), "metadata":{"version": version}, "limit":MAX_MONGO_QUEUE_LENGTH}); | ||
var fileWriteStream = gridFileStream.createWriteStream({"mode": "w", "_id": fileObjectId, "filename":fileName, "content_type": fileContentType.toString(), "metadata":{"version": version, "groupId": groupId}, "limit":constants.MAX_MONGO_QUEUE_LENGTH, "root": constants.ROOT_COLLECTION}); | ||
fileStream.on("error", function(err){console.log("READING ERROR ", err);}); | ||
fileStream.on("error", function(err){defaultLogger.error(err); return cb(err);}); | ||
fileWriteStream.on("close", function(file){ | ||
createImageThumbnail(db, fileName, file._id, version, options, cb); | ||
createImageThumbnail(db, fileName, {"fileName": file.filename, "contentType": file.contentType, "hash": file.md5, "version": file.metadata.version, "groupId": file.metadata.groupId, "_id": file._id}, version, options, cb); | ||
}); | ||
@@ -438,13 +503,26 @@ | ||
}else{ | ||
var fileWriteStream = gridFileStream.createWriteStream({"mode": "w", "_id": new ObjectId(), "filename":fileName, "content_type": fileContentType.toString(), metadata:{"version": version}, "limit":MAX_MONGO_QUEUE_LENGTH}); | ||
var fileMetadata = {}; | ||
//If moving a thumbnail, ensure the version of the thumbnail matches the version of the image it came from. | ||
if(options.migratingThumbnail){ | ||
fileMetadata.version = options.migratedThumbnailVersion; | ||
} else { | ||
fileMetadata.version = version; | ||
fileMetadata.groupId = groupId; | ||
} | ||
console.log("WriteStream Created"); | ||
if(options.migratingFile){ //If migrating a file, then the groupId will migrate also. | ||
fileMetadata.groupId = options.groupId; | ||
} | ||
fileStream.on("error", function(err){console.log("READING ERROR ", err);}); | ||
var fileWriteStream = gridFileStream.createWriteStream({"mode": "w", "_id": fileObjectId, "filename":fileName, "content_type": fileContentType.toString(), "metadata": fileMetadata , "limit":constants.MAX_MONGO_QUEUE_LENGTH, "root": constants.ROOT_COLLECTION}); | ||
defaultLogger.debug("WriteStream Created"); | ||
fileStream.on("error", function(err){defaultLogger.error(err); return cb(err);}); | ||
fileWriteStream.on("close", function(file){ | ||
console.log("Write Stream Closed"); | ||
return cb(undefined, file._id); | ||
}).on("error", function(err){console.log("WRITING ERROR ", err);}); | ||
defaultLogger.debug("Write Stream Closed"); | ||
return cb(undefined, {"fileName": file.filename, "contentType": file.contentType, "hash": file.md5, "version": file.metadata.version, "groupId": file.metadata.groupId, "_id": file._id}); | ||
}).on("error", function(err){defaultLogger.error(err); return cb(err);}); | ||
@@ -458,53 +536,124 @@ fileStream.resume(); | ||
function fileExists(db, fileName, cb){ | ||
console.log("In fileExists"); | ||
GridStore.exist(db, fileName, function(err, exists){ | ||
return cb(err, exists); | ||
function fileExists(db, searchOptions, cb){ | ||
defaultLogger.debug("In fileExists "); | ||
var searchQuery = {}; | ||
if(searchOptions.groupId){ | ||
searchQuery["metadata.groupId"] = searchOptions.groupId; | ||
} else if(searchOptions.hash){ | ||
searchQuery.md5 = searchOptions.hash; | ||
} else { | ||
return cb(new Error("No search options specified for fileExists. Aborting.")); | ||
} | ||
//Checking for specific version | ||
if(searchOptions.version){ | ||
searchQuery["metadata.version"] = searchOptions.version; | ||
} | ||
db.collection(constants.ROOT_COLLECTION + ".files", function(err, collection){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
collection.find(searchQuery, {}, function(err, foundFiles){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
foundFiles.count(function(err, numFiles){ | ||
var groupExists = numFiles > 0; | ||
return cb(err, groupExists); | ||
}); | ||
}); | ||
}); | ||
} | ||
//Utility function to search for files in the database. | ||
function getFileDetails(db, fileSelectionCriteria, fileOptions, cb){ | ||
console.log("In getFileDetails"); | ||
var selectionQuery = {}; | ||
var gridStore = undefined; | ||
if(fileSelectionCriteria.fileName){ | ||
selectionQuery = {"filename": fileSelectionCriteria.fileName}; | ||
gridStore = new GridStore(db, fileSelectionCriteria.fileName, "r"); | ||
}else if(fileSelectionCriteria.fileId){ | ||
selectionQuery = {"_id": fileSelectionCriteria.fileId}; | ||
gridStore = new GridStore(db, fileSelectionCriteria.fileId, "r"); | ||
} else{ | ||
return cb(new Error("getFileDetails: No search criteria selected")); | ||
defaultLogger.debug("In getFileDetails "); | ||
var selectionQuery = undefined; | ||
if(fileSelectionCriteria.groupId){ | ||
selectionQuery= {"metadata.groupId": fileSelectionCriteria.groupId}; | ||
} else if(fileSelectionCriteria.hash){ | ||
selectionQuery= {"md5": fileSelectionCriteria.hash}; | ||
} | ||
var gridStore = new GridStore(db, null, "r", {"root": constants.ROOT_COLLECTION}); | ||
var fileOfInterest = undefined; | ||
gridStore.collection(function(err, collection){ | ||
collection.find(selectionQuery, { "sort": {"metadata.version":-1}}, function(err, files){ | ||
if(err) return cb(err); | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
files.toArray(function(err, filesArray){ | ||
if(err) return cb(err); | ||
if(!(filesArray && Array.isArray(filesArray) && filesArray.length > 0)) return cb(new Error("No files exist for fileName " + fileName)); | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
if(!(filesArray && Array.isArray(filesArray) && filesArray.length > 0)) return cb(new Error("No files exist for groupId " + fileSelectionCriteria.groupId)); | ||
//If there is no version, get the latest version | ||
var fileOfInterest = undefined; | ||
if(!fileOptions.version){ | ||
fileOfInterest = filesArray[0]; | ||
}else{ | ||
fileOfInterest = filesArray.filter(function(file){return file.metadata.version == fileOptions.version;}); | ||
//If the file details are needed for all files in the groupId, then an array containing all of the file details is returned. | ||
if(fileOptions.allMatchingFiles == true){ | ||
fileOfInterest = filesArray; | ||
}else{// Just want the details of a single file. | ||
//If there is no version, get the latest version | ||
if(fileOfInterest.length == 1){ | ||
fileOfInterest = fileOfInterest[0]; | ||
//If no version is found or we have a hash query, just want the first entry of the array. | ||
if(!fileSelectionCriteria.version || fileSelectionCriteria.hash){ | ||
fileOfInterest = filesArray[0]; | ||
}else{ | ||
fileOfInterest = filesArray.filter(function(file){return file.metadata.version == fileSelectionCriteria.version;}); | ||
if(fileOfInterest.length == 1){ | ||
fileOfInterest = fileOfInterest[0]; | ||
} | ||
else{ | ||
return cb(new Error("Unexpected number of files returned for groupId " + fileSelectionCriteria.groupId, fileOfInterest.length)); | ||
} | ||
} | ||
else{ | ||
return cb(new Error("Unexpected number of files returned for fileName " + fileSelectionCriteria.fileName, fileOfInterest.length)); | ||
} | ||
if(!Array.isArray(fileOfInterest)){ | ||
//Actually want the thumbnail for the file, return the details for that file instead. | ||
if(fileOptions.thumbnail){ | ||
if(fileOfInterest.metadata.thumbnail){ | ||
getThumbnailFileDetails(db, fileOfInterest.metadata.thumbnail, function(err, thumbFileDetails){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
fileOfInterest = thumbFileDetails; | ||
gridStore.close(function(err, result){ | ||
return cb(err, fileOfInterest); | ||
}); | ||
}); | ||
}else { | ||
return cb(new Error("Thumbnail for file " + JSON.stringify(fileSelectionCriteria) + " does not exist.")); | ||
} | ||
}else { | ||
gridStore.close(function(err, result){ | ||
return cb(err, fileOfInterest); | ||
}); | ||
} | ||
}else{ | ||
gridStore.close(function(err, result){ | ||
return cb(err, fileOfInterest); | ||
}); | ||
} | ||
}); | ||
}); | ||
}); | ||
} | ||
function getThumbnailFileDetails(db, thumbId, cb){ | ||
var gridStore = new GridStore(db, null, "r", {"root": constants.ROOT_COLLECTION}); | ||
gridStore.collection(function(err, collection){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
collection.find({"_id": thumbId}, {}, function(err, files){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
files.toArray(function(err, filesArray){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
if(filesArray.length == 0){ | ||
return cb(new Error("Search For Unique Thumbnail with Id " + thumbId + " found no files. Aborting")); | ||
} else if(filesArray.length > 1){ | ||
return cb(new Error("Search For Unique Thumbnail with Id " + thumbId + " found more than one file. Aborting")); | ||
} else { | ||
return cb(undefined, filesArray[0]); | ||
} | ||
gridStore.close(function(err, result){ | ||
return cb(err, fileOfInterest); | ||
}); | ||
}); | ||
@@ -517,3 +666,3 @@ }); | ||
function incrementFileVersion(oldFileVersion, cb){ | ||
console.log("In incrementFileVersion"); | ||
defaultLogger.debug("In incrementFileVersion"); | ||
return cb(oldFileVersion + 1); | ||
@@ -524,10 +673,10 @@ } | ||
function updateExistingFile(db, fileName, fileReadStream, options, cb){ | ||
console.log("In updateExistingFile"); | ||
defaultLogger.debug("In updateExistingFile"); | ||
getFileDetails(db, {"fileName": fileName}, options, function(err, fileInfo){ | ||
if(err) return cb(err); | ||
getFileDetails(db, {"groupId": options.groupId}, {}, function(err, fileInfo){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
var latestFileVersion = fileInfo.metadata.version; | ||
incrementFileVersion(latestFileVersion, function(newFileVersion){ | ||
if(!newFileVersion) return cb(new Error("File version was not incremented for file " + fileName)); | ||
console.log("New File Version ", newFileVersion); | ||
defaultLogger.debug("New File Version ", newFileVersion); | ||
createFileWithVersion(db, fileName, fileReadStream, newFileVersion, options, cb); | ||
@@ -540,40 +689,25 @@ }); | ||
function createNewFile(db, fileName, fileReadStream, options, cb){ | ||
console.log("In createNewFile"); | ||
createFileWithVersion(db, fileName, fileReadStream, constants.FH_LOWEST_VERSION, options, cb); | ||
defaultLogger.debug("In createNewFile"); | ||
createFileWithVersion(db, fileName, fileReadStream, constants.LOWEST_VERSION, options, cb); | ||
} | ||
function getFileStream(db, fileId, options, cb){ | ||
console.log("In getFileStream"); | ||
var gridStore = new GridStore(db, fileId, "r"); | ||
if(options.thumbnail){ | ||
//Need the thumbnail of the image | ||
gridStore.open(function(err, file){ | ||
if(err) return cb(err); | ||
function getFileStream(db, searchOptions, fileOptions, cb){ | ||
defaultLogger.debug("In getFileStream"); | ||
if(!file.metadata.thumbnail){ | ||
return cb(new Error("Thumbnail does not exist for file " + file.filename)); | ||
} | ||
var thumbFileId = file.metadata.thumbnail; | ||
var gridFileStream = new GridFsStream(db, mongodb); | ||
var fileReadStream = gridFileStream.createReadStream({"_id": thumbFileId}); | ||
gridStore.close(function(err, result){ | ||
if(err) return cb(err); | ||
return cb(undefined, fileReadStream); | ||
}); | ||
}); | ||
}else{ | ||
getFileDetails(db, searchOptions, fileOptions, function(err, fileDetails){ | ||
if(err) { defaultLogger.error(err); return cb(err);} | ||
var gridFileStream = new GridFsStream(db, mongodb); | ||
var fileReadStream = gridFileStream.createReadStream({"_id": fileId}); | ||
gridStore.close(function(err, result){ | ||
if(err) return cb(err); | ||
return cb(undefined, fileReadStream); | ||
}); | ||
} | ||
var fileReadStream = gridFileStream.createReadStream({"_id": fileDetails._id, "root": constants.ROOT_COLLECTION}); | ||
return cb(err, fileReadStream); | ||
}); | ||
} | ||
function getFileContentType(fileName, cb){ | ||
console.log("In getFileContentType"); | ||
defaultLogger.debug("In getFileContentType"); | ||
var mimeType = mime.lookup(fileName); | ||
console.log(mimeType); | ||
defaultLogger.debug(mimeType); | ||
@@ -583,18 +717,3 @@ return cb(mimeType); | ||
function constructFileOptions(fileName, cb){ | ||
console.log("In constructFileOptions"); | ||
//Need to get the mime type of the file based on the extension | ||
getFileContentType(fileName, function(fileMimeType){ | ||
//If it is an image, need to set thumbnail option | ||
if(fileMimeType.indexOf("image/") != -1){ | ||
var options = {"image":true, "thumbnail": true}; | ||
return cb(options); | ||
}else{ | ||
return cb({}); | ||
} | ||
}); | ||
} | ||
module.exports.MongoFileHandler = MongoFileHandler; |
{ | ||
"name": "fh-gridfs", | ||
"version": "0.0.2-6", | ||
"version": "0.2.0-11", | ||
"description": "Wrapper for file storage using gridfs for mongo databases.", | ||
@@ -8,5 +8,3 @@ "scripts": { | ||
}, | ||
"repository": "https://github.com/fheng/fh-gridfs.git", | ||
"author": "Niall Donnelly <niall.donnelly@feedhenry.com>", | ||
"license": "BSD", | ||
"engines": { | ||
@@ -27,4 +25,5 @@ "node": "*" | ||
"assert": "0.4.9", | ||
"util": "0.4.9" | ||
"util": "0.4.9", | ||
"usage": "0.3.8" | ||
} | ||
} |
@@ -1,1 +0,1 @@ | ||
0.0.2-6 | ||
0.2.0-11 |
Sorry, the diff of this file is not supported yet
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Major refactor
Supply chain riskPackage has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.
Found 1 instance in 1 package
No repository
Supply chain riskPackage does not have a linked source code repository. Without this field, a package will have no reference to the location of the source code use to generate the package.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Misc. License Issues
License(Experimental) A package's licensing information has fine-grained problems.
Found 1 instance in 1 package
35395
0
551
3
1