Improved files to distinguish subtypes (allows multiple different files tabs at a entity)

Attachments via the improved files
Block thumbnails at mosaico templates as a separate files tab
Some fixes

All not tested yet
This commit is contained in:
Tomas Bures 2018-08-02 15:49:27 +05:30
parent ade0fc87f2
commit 32cad03f4f
32 changed files with 683 additions and 346 deletions

View file

@ -9,6 +9,7 @@ const { enforce, filterObject } = require('../lib/helpers');
const shares = require('./shares');
const namespaceHelpers = require('../lib/namespace-helpers');
const files = require('./files');
const templates = require('./templates');
const { CampaignSource, CampaignType} = require('../shared/campaigns');
const segments = require('./segments');
@ -33,16 +34,22 @@ async function listDTAjax(context, params) {
);
}
async function getById(context, id) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'campaign', id, 'view');
const entity = await tx('campaigns').where('id', id).first();
async function getByIdTx(tx, context, id, withPermissions = true) {
await shares.enforceEntityPermissionTx(tx, context, 'campaign', id, 'view');
const entity = await tx('campaigns').where('id', id).first();
if (withPermissions) {
entity.permissions = await shares.getPermissionsTx(tx, context, 'campaign', id);
}
entity.data = JSON.parse(entity.data);
entity.data = JSON.parse(entity.data);
return entity;
return entity;
}
async function getById(context, id, withPermissions = true) {
return await knex.transaction(async tx => {
return await getByIdTx(tx, context, id, withPermissions);
});
}
@ -50,7 +57,7 @@ async function _validateAndPreprocess(tx, context, entity, isCreate) {
await namespaceHelpers.validateEntity(tx, entity);
if (isCreate) {
enforce(entity.type === CampaignType.REGULAR && entity.type === CampaignType.RSS && entity.type === CampaignType.TRIGGERED, 'Unknown campaign type');
enforce(entity.type === CampaignType.REGULAR || entity.type === CampaignType.RSS || entity.type === CampaignType.TRIGGERED, 'Unknown campaign type');
if (entity.source === CampaignSource.TEMPLATE || entity.source === CampaignSource.CUSTOM_FROM_TEMPLATE) {
await shares.enforceEntityPermissionTx(tx, context, 'template', entity.data.sourceTemplate, 'view');
@ -66,6 +73,8 @@ async function _validateAndPreprocess(tx, context, entity, isCreate) {
await segments.getByIdTx(tx, context, entity.list, entity.segment);
}
await shares.enforceEntityPermissionTx(tx, context, 'send_configuration', entity.send_configuration, 'viewPublic');
entity.data = JSON.stringify(entity.data);
}
@ -73,9 +82,30 @@ async function create(context, entity) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'namespace', entity.namespace, 'createCampaign');
let copyFilesFromTemplateId;
let copyFilesFrom = null;
if (entity.source === CampaignSource.CUSTOM_FROM_TEMPLATE) {
copyFilesFromTemplateId = entity.data.sourceTemplate;
copyFilesFrom = {
entityType: 'template',
entityId: entity.data.sourceTemplate
};
const template = await templates.getByIdTx(tx, context, entity.data.sourceTemplate, false);
entity.data.sourceCustom = {
type: template.type,
data: template.data,
html: template.html,
text: template.text
};
} else if (entity.source === CampaignSource.CUSTOM_FROM_CAMPAIGN) {
copyFilesFrom = {
entityType: 'campaign',
entityId: entity.data.sourceCampaign
};
const sourceCampaign = await getByIdTx(tx, context, entity.data.sourceCampaign, false);
entity.data.sourceCustom = sourceCampaign.data.sourceCustom;
}
await _validateAndPreprocess(tx, context, entity, true);
@ -119,8 +149,8 @@ async function create(context, entity) {
await shares.rebuildPermissionsTx(tx, { entityTypeId: 'campaign', entityId: id });
if (copyFilesFromTemplateId) {
files.copyAllTx(tx, context, 'template', copyFilesFromTemplateId, 'campaign', id);
if (copyFilesFrom) {
await files.copyAllTx(tx, context, copyFilesFrom.entityType, copyFilesFrom.entityId, 'campaign', id);
}
return id;
@ -168,6 +198,7 @@ async function remove(context, id) {
module.exports = {
hash,
listDTAjax,
getByIdTx,
getById,
create,
updateWithConsistencyCheck,

View file

@ -10,49 +10,63 @@ const interoperableErrors = require('../shared/interoperable-errors');
const permissions = require('../lib/permissions');
const {getTrustedUrl} = require('../lib/urls');
const crypto = require('crypto');
const bluebird = require('bluebird');
const cryptoPseudoRandomBytes = bluebird.promisify(crypto.pseudoRandomBytes);
const entityTypes = permissions.getEntityTypes();
const filesDir = path.join(__dirname, '..', 'files');
function enforceTypePermitted(type) {
enforce(type in entityTypes && entityTypes[type].filesTable);
const ReplacementBehavior = {
NONE: 0,
REPLACE: 1,
RENAME: 2
};
function enforceTypePermitted(type, subType) {
enforce(type in entityTypes && entityTypes[type].files && entityTypes[type].files[subType]);
}
function getFilePath(type, entityId, filename) {
return path.join(path.join(filesDir, type, entityId.toString()), filename);
function getFilePath(type, subType, entityId, filename) {
return path.join(path.join(filesDir, type, subType, entityId.toString()), filename);
}
function getFileUrl(context, type, entityId, filename) {
return getTrustedUrl(`files/${type}/${entityId}/${filename}`, context)
function getFileUrl(context, type, subType, entityId, filename) {
return getTrustedUrl(`files/${type}/${subType}/${entityId}/${filename}`, context)
}
function getFilesTable(type) {
return entityTypes[type].filesTable;
function getFilesTable(type, subType) {
return entityTypes[type].files[subType].table;
}
async function listDTAjax(context, type, entityId, params) {
enforceTypePermitted(type);
await shares.enforceEntityPermission(context, type, entityId, 'viewFiles');
function getFilesPermission(type, subType, operation) {
return entityTypes[type].files[subType].permissions[operation];
}
async function listDTAjax(context, type, subType, entityId, params) {
enforceTypePermitted(type, subType);
await shares.enforceEntityPermission(context, type, entityId, getFilesPermission(type, subType, 'view'));
return await dtHelpers.ajaxList(
params,
builder => builder.from(getFilesTable(type)).where({entity: entityId}),
builder => builder.from(getFilesTable(type, subType)).where({entity: entityId}),
['id', 'originalname', 'filename', 'size', 'created']
);
}
async function list(context, type, entityId) {
enforceTypePermitted(type);
async function list(context, type, subType, entityId) {
enforceTypePermitted(type, subType);
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, type, entityId, 'viewFiles');
return await tx(getFilesTable(type)).where({entity: entityId}).select(['id', 'originalname', 'filename', 'size', 'created']).orderBy('originalname', 'asc');
await shares.enforceEntityPermissionTx(tx, context, type, entityId, getFilesPermission(type, subType, 'view'));
return await tx(getFilesTable(type, subType)).where({entity: entityId}).select(['id', 'originalname', 'filename', 'size', 'created']).orderBy('originalname', 'asc');
});
}
async function getFileById(context, type, id) {
enforceTypePermitted(type);
async function getFileById(context, type, subType, id) {
enforceTypePermitted(type, subType);
const file = await knex.transaction(async tx => {
const file = await tx(getFilesTable(type)).where('id', id).first();
await shares.enforceEntityPermissionTx(tx, context, type, file.entity, 'viewFiles');
const file = await tx(getFilesTable(type, subType)).where('id', id).first();
await shares.enforceEntityPermissionTx(tx, context, type, file.entity, getFilesPermission(type, subType, 'view'));
return file;
});
@ -63,15 +77,15 @@ async function getFileById(context, type, id) {
return {
mimetype: file.mimetype,
name: file.originalname,
path: getFilePath(type, file.entity, file.filename)
path: getFilePath(type, subType, file.entity, file.filename)
};
}
async function _getFileBy(context, type, entityId, key, value) {
enforceTypePermitted(type);
async function _getFileBy(context, type, subType, entityId, key, value) {
enforceTypePermitted(type, subType);
const file = await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, type, entityId, 'viewFiles');
const file = await tx(getFilesTable(type)).where({entity: entityId, [key]: value}).first();
await shares.enforceEntityPermissionTx(tx, context, type, entityId, getFilesPermission(type, subType, 'view'));
const file = await tx(getFilesTable(type, subType)).where({entity: entityId, [key]: value}).first();
return file;
});
@ -82,30 +96,31 @@ async function _getFileBy(context, type, entityId, key, value) {
return {
mimetype: file.mimetype,
name: file.originalname,
path: getFilePath(type, file.entity, file.filename)
path: getFilePath(type, subType, file.entity, file.filename)
};
}
async function getFileByOriginalName(context, type, entityId, name) {
return await _getFileBy(context, type, entityId, 'originalname', name)
async function getFileByOriginalName(context, type, subType, entityId, name) {
return await _getFileBy(context, type, subType, entityId, 'originalname', name)
}
async function getFileByFilename(context, type, entityId, name) {
return await _getFileBy(context, type, entityId, 'filename', name)
async function getFileByFilename(context, type, subType, entityId, name) {
return await _getFileBy(context, type, subType, entityId, 'filename', name)
}
async function getFileByUrl(context, type, entityId, url) {
const urlPrefix = getTrustedUrl(`files/${type}/${entityId}/`, context);
async function getFileByUrl(context, type, subType, entityId, url) {
const urlPrefix = getTrustedUrl(`files/${type}/${subType}/${entityId}/`, context);
if (url.startsWith(urlPrefix)) {
const name = url.substring(urlPrefix.length);
return await getFileByFilename(context, type, entityId, name);
return await getFileByFilename(context, type, subType, entityId, name);
} else {
throw new interoperableErrors.NotFoundError();
}
}
async function createFiles(context, type, entityId, files, getUrl = null, dontReplace = false) {
enforceTypePermitted(type);
// Adds files to an entity. The source data can be either a file (then it's path is contained in file.path) or in-memory data (then it's content is in file.data).
async function createFiles(context, type, subType, entityId, files, replacementBehavior) {
enforceTypePermitted(type, subType);
if (files.length == 0) {
// No files uploaded
return {uploaded: 0};
@ -118,31 +133,39 @@ async function createFiles(context, type, entityId, files, getUrl = null, dontRe
const filesRet = [];
await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, type, entityId, 'manageFiles');
await shares.enforceEntityPermissionTx(tx, context, type, entityId, getFilesPermission(type, subType, 'manage'));
const existingNamesRows = await tx(getFilesTable(type)).where('entity', entityId).select(['filename', 'originalname']);
const existingNameMap = new Map();
const existingNamesRows = await tx(getFilesTable(type, subType)).where('entity', entityId).select(['id', 'filename', 'originalname']);
const existingNameSet = new Set();
for (const row of existingNamesRows) {
existingNameMap.set(row.originalname, row);
existingNameSet.add(row.originalname);
}
const originalNameSet = new Set();
// The processedNameSet holds originalnames of entries which have been already processed in the upload batch. It prevents uploading two files with the same originalname
const processedNameSet = new Set();
// Create entities for files
for (const file of files) {
const parsedOriginalName = path.parse(file.originalname);
let originalName = parsedOriginalName.base;
if (dontReplace) {
if (!file.filename) {
// This is taken from multer/storage/disk.js and adapted for async/await
file.filename = (await cryptoPseudoRandomBytes(16)).toString('hex');
}
if (replacementBehavior === ReplacementBehavior.RENAME) {
let suffix = 1;
while (existingNameMap.has(originalName) || originalNameSet.has(originalName)) {
while (existingNameSet.has(originalName) || processedNameSet.has(originalName)) {
originalName = parsedOriginalName.name + '-' + suffix + parsedOriginalName.ext;
suffix++;
}
}
if (originalNameSet.has(originalName)) {
// The file has an original name same as another file
if (replacementBehavior === ReplacementBehavior.NONE && (existingNameSet.has(originalName) || processedNameSet.has(originalName))) {
// The file has an original name same as another file in the same upload batch or it has an original name same as another already existing file
ignoredFiles.push(file);
} else {
@ -161,46 +184,61 @@ async function createFiles(context, type, entityId, files, getUrl = null, dontRe
name: file.filename,
originalName: originalName,
size: file.size,
type: file.mimetype,
type: file.mimetype
};
filesRetEntry.url = getFileUrl(context, type, entityId, file.filename);
filesRetEntry.thumbnailUrl = getFileUrl(context, type, entityId, file.filename); // TODO - use smaller thumbnails
filesRetEntry.url = getFileUrl(context, type, subType, entityId, file.filename);
if (file.mimetype.startsWith('image/')) {
filesRetEntry.thumbnailUrl = getFileUrl(context, type, subType, entityId, file.filename); // TODO - use smaller thumbnails,
}
filesRet.push(filesRetEntry);
}
if (existingNameMap.has(originalName)) {
removedFiles.push(existingNameMap.get(originalName));
processedNameSet.add(originalName);
}
if (replacementBehavior === ReplacementBehavior.REPLACE) {
for (const row of existingNamesRows) {
const idsToRemove = [];
if (processedNameSet.has(row.originalname)) {
removedFiles.push(row);
idsToRemove.push(row.id);
}
}
originalNameSet.add(originalName);
await tx(getFilesTable(type, subType)).where('entity', entityId).whereIn('id', idsToRemove).del();
}
const originalNameArray = Array.from(originalNameSet);
await tx(getFilesTable(type)).where('entity', entityId).whereIn('originalname', originalNameArray).del();
if (fileEntities) {
await tx(getFilesTable(type)).insert(fileEntities);
await tx(getFilesTable(type, subType)).insert(fileEntities);
}
});
// Move new files from upload directory to files directory
for (const file of filesToMove) {
const filePath = getFilePath(type, entityId, file.filename);
// The names should be unique, so overwrite is disabled
// The directory is created if it does not exist
// Empty options argument is passed, otherwise fails
await fs.moveAsync(file.path, filePath, {});
const filePath = getFilePath(type, subType, entityId, file.filename);
if (file.path) {
// The names should be unique, so overwrite is disabled
// The directory is created if it does not exist
// Empty options argument is passed, otherwise fails
await fs.moveAsync(file.path, filePath, {});
} else if (file.data) {
await fs.outputFile(filePath, file.data);
}
}
// Remove replaced files from files directory
for (const file of removedFiles) {
const filePath = getFilePath(type, entityId, file.filename);
const filePath = getFilePath(type, subType, entityId, file.filename);
await fs.removeAsync(filePath);
}
// Remove ignored files from upload directory
for (const file of ignoredFiles) {
await fs.removeAsync(file.path);
if (file.path) {
await fs.removeAsync(file.path);
}
}
return {
@ -212,38 +250,38 @@ async function createFiles(context, type, entityId, files, getUrl = null, dontRe
};
}
async function removeFile(context, type, id) {
enforceTypePermitted(type);
async function removeFile(context, type, subType, id) {
enforceTypePermitted(type, subType);
const file = await knex.transaction(async tx => {
const file = await tx(getFilesTable(type)).where('id', id).select('entity', 'filename').first();
await shares.enforceEntityPermissionTx(tx, context, type, file.entity, 'manageFiles');
await tx(getFilesTable(type)).where('id', id).del();
const file = await tx(getFilesTable(type, subType)).where('id', id).select('entity', 'filename').first();
await shares.enforceEntityPermissionTx(tx, context, type, file.entity, getFilesPermission(type, subType, 'manage'));
await tx(getFilesTable(type, subType)).where('id', id).del();
return {filename: file.filename, entity: file.entity};
});
const filePath = getFilePath(type, file.entity, file.filename);
const filePath = getFilePath(type, subType, file.entity, file.filename);
await fs.removeAsync(filePath);
}
async function copyAllTx(tx, context, fromType, fromEntityId, toType, toEntityId) {
enforceTypePermitted(fromType);
await shares.enforceEntityPermissionTx(tx, context, fromType, fromEntityId, 'viewFiles');
async function copyAllTx(tx, context, fromType, fromSubType, fromEntityId, toType, toSubType, toEntityId) {
enforceTypePermitted(fromType, fromSubType);
await shares.enforceEntityPermissionTx(tx, context, fromType, fromEntityId, getFilesPermission(fromType, fromSubType, 'view'));
enforceTypePermitted(toType);
await shares.enforceEntityPermissionTx(tx, context, toType, toEntityId, 'manageFiles');
enforceTypePermitted(toType, toSubType);
await shares.enforceEntityPermissionTx(tx, context, toType, toEntityId, getFilesPermission(toType, toSubType, 'manage'));
const rows = await tx(getFilesTable(fromType)).where({entity: fromEntityId});
const rows = await tx(getFilesTable(fromType, fromSubType)).where({entity: fromEntityId});
for (const row of rows) {
const fromFilePath = getFilePath(fromType, fromEntityId, row.filename);
const toFilePath = getFilePath(toType, toEntityId, row.filename);
const fromFilePath = getFilePath(fromType, fromSubType, fromEntityId, row.filename);
const toFilePath = getFilePath(toType, toSubType, toEntityId, row.filename);
await fs.copyAsync(fromFilePath, toFilePath, {});
delete row.id;
row.entity = toEntityId;
}
await tx(getFilesTable(toType)).insert(rows);
await tx(getFilesTable(toType, toSubType)).insert(rows);
}
@ -259,5 +297,6 @@ module.exports = {
removeFile,
getFileUrl,
getFilePath,
copyAllTx
copyAllTx,
ReplacementBehavior
};

View file

@ -15,17 +15,21 @@ function hash(entity) {
return hasher.hash(filterObject(entity, allowedKeys));
}
async function getByIdTx(tx, context, id, withPermissions = true) {
await shares.enforceEntityPermissionTx(tx, context, 'template', id, 'view');
const entity = await tx('templates').where('id', id).first();
entity.data = JSON.parse(entity.data);
if (withPermissions) {
entity.permissions = await shares.getPermissionsTx(tx, context, 'template', id);
}
return entity;
}
async function getById(context, id, withPermissions = true) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'template', id, 'view');
const entity = await tx('templates').where('id', id).first();
entity.data = JSON.parse(entity.data);
if (withPermissions) {
entity.permissions = await shares.getPermissionsTx(tx, context, 'template', id);
}
return entity;
return await getByIdTx(tx, context, id, withPermissions);
});
}
@ -100,6 +104,7 @@ async function remove(context, id) {
module.exports = {
hash,
getByIdTx,
getById,
listDTAjax,
create,