public/mailtrain
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Basic import seems to work

Browse source
This commit is contained in:
Tomas Bures 2018-09-01 21:29:10 +02:00
parent 16519c5353
commit d74806dde3
21 changed files with 555 additions and 749 deletions
Download patch file Download diff file Expand all files Collapse all files

50
client/src/lists/imports/CUD.js
View file

@ -175,7 +175,7 @@ export default class CUD extends Component {
} else {
const mappingType = Number.parseInt(state.getIn(['mapping_type', 'value']));
if (mappingType === MappingType.BASIC_SUBSCRIBE) {
if (mappingType === MappingType.BASIC_SUBSCRIBE || mappingType === MappingType.BASIC_UNSUBSCRIBE) {
if (!state.getIn(['mapping_fields_email_column', 'value'])) {
state.setIn(['mapping_fields_email_column', 'error'], t('Email mapping has to be provided'));
}
@ -225,24 +225,33 @@ export default class CUD extends Component {
for (const field of this.props.fieldsGrouped) {
if (field.column) {
mapping.fields[field.column] = {
column: data['mapping_fields_' + field.column + '_column']
};
const colMapping = data['mapping_fields_' + field.column + '_column'];
if (colMapping) {
mapping.fields[field.column] = {
column: colMapping
};
}
} else {
for (const option of field.settings.options) {
const col = field.groupedOptions[option.key].column;
mapping.fields[col] = {
column: data['mapping_fields_' + col + '_column']
};
const colMapping = data['mapping_fields_' + col + '_column'];
if (colMapping) {
mapping.fields[col] = {
column: colMapping
};
}
}
}
}
}
if (data.mapping_type === MappingType.BASIC_SUBSCRIBE || data.mapping_type === MappingType.BASIC_UNSUBSCRIBE) {
mapping.fields.email = {
column: data.mapping_fields_email_column
};
}
data.mapping = mapping;
}
@ -317,7 +326,7 @@ export default class CUD extends Component {
let mappingSettings = null;
const mappingType = Number.parseInt(this.getFormValue('mapping_type'));
if (mappingType === MappingType.BASIC_SUBSCRIBE) {
if (mappingType === MappingType.BASIC_SUBSCRIBE || mappingType === MappingType.BASIC_UNSUBSCRIBE) {
const sampleRow = this.getFormValue('sampleRow');
const sourceOpts = [];
sourceOpts.push({key: '', label: t('–– Select ––')});
@ -332,28 +341,33 @@ export default class CUD extends Component {
}
}
const settingsRows = [];
const mappingRows = [
<Dropdown key="email" id="mapping_fields_email_column" label={t('Email')} options={sourceOpts}/>
];
for (const field of this.props.fieldsGrouped) {
if (field.column) {
mappingRows.push(
<Dropdown key={field.column} id={'mapping_fields_' + field.column + '_column'} label={field.name} options={sourceOpts}/>
);
} else {
for (const option of field.settings.options) {
const col = field.groupedOptions[option.key].column;
if (mappingType === MappingType.BASIC_SUBSCRIBE) {
settingsRows.push(<CheckBox key="checkEmails" id="mapping_settings_checkEmails" text={t('Check imported emails')}/>)
for (const field of this.props.fieldsGrouped) {
if (field.column) {
mappingRows.push(
<Dropdown key={col} id={'mapping_fields_' + col + '_column'} label={field.groupedOptions[option.key].name} options={sourceOpts}/>
<Dropdown key={field.column} id={'mapping_fields_' + field.column + '_column'} label={field.name} options={sourceOpts}/>
);
} else {
for (const option of field.settings.options) {
const col = field.groupedOptions[option.key].column;
mappingRows.push(
<Dropdown key={col} id={'mapping_fields_' + col + '_column'} label={field.groupedOptions[option.key].name} options={sourceOpts}/>
);
}
}
}
}
mappingSettings = (
<div>
<CheckBox id="mapping_settings_checkEmails" text={t('Check imported emails')}/>
{settingsRows}
<Fieldset label={t('Mapping')} className={styles.mapping}>
{mappingRows}
</Fieldset>

18
client/src/lists/imports/RunStatus.js
View file

@ -18,6 +18,7 @@ import axios from "../../lib/axios";
import {getUrl} from "../../lib/urls";
import moment from "moment";
import {runStatusInProgress} from "../../../../shared/imports";
import {Table} from "../../lib/table";
@translate()
@withPageHelpers
@ -52,6 +53,10 @@ export default class Status extends Component {
this.setState({
entity: resp.data
});
if (this.failedTableNode) {
this.failedTableNode.refresh();
}
}
async periodicRefreshTask() {
@ -77,7 +82,13 @@ export default class Status extends Component {
const entity = this.state.entity;
const imprt = this.props.imprt;
return (
const columns = [
{ data: 1, title: t('Row') },
{ data: 2, title: t('Email') },
{ data: 3, title: t('Reason') }
];
return (
<div>
<Title>{t('Import Run Status')}</Title>
@ -90,6 +101,11 @@ export default class Status extends Component {
<AlignedRow label={t('New entries')}>{entity.new}</AlignedRow>
<AlignedRow label={t('Failed entries')}>{entity.failed}</AlignedRow>
{entity.error && <AlignedRow label={t('Error')}><pre>{entity.error}</pre></AlignedRow>}
<hr/>
<h3>{t('Failed Rows')}</h3>
<Table ref={node => this.failedTableNode = node} withHeader dataUrl={`rest/import-run-failed-table/${this.props.list.id}/${this.props.imprt.id}/${this.props.entity.id}`} columns={columns} />
</div>
);
}

10
client/src/lists/imports/Status.js
View file

@ -97,7 +97,10 @@ export default class Status extends Component {
}
await this.refreshEntity();
this.runsTableNode.refresh();
if (this.runsTableNode) {
this.runsTableNode.refresh();
}
}
async stopRunAsync() {
@ -112,7 +115,10 @@ export default class Status extends Component {
}
await this.refreshEntity();
this.runsTableNode.refresh();
if (this.runsTableNode) {
this.runsTableNode.refresh();
}
}
render() {

3
client/src/lists/imports/helpers.js
View file

@ -27,7 +27,8 @@ export function getImportTypes(t) {
[RunStatus.SCHEDULED]: t('Starting'),
[RunStatus.RUNNING]: t('Running'),
[RunStatus.STOPPING]: t('Stopping'),
[RunStatus.FINISHED]: t('Finished')
[RunStatus.FINISHED]: t('Finished'),
[RunStatus.FAILED]: t('Failed')
};
const mappingTypeLabels = {

1
client/src/lists/subscriptions/List.js
View file

@ -86,6 +86,7 @@ export default class List extends Component {
const segments = this.props.segments;
const columns = [
{ data: 1, title: t('CID') },
{ data: 2, title: t('Email') },
{ data: 3, title: t('Status'), render: (data, display, rowData) => this.subscriptionStatusLabels[data] + (rowData[5] ? ', ' + t('Blacklisted') : '') },
{ data: 4, title: t('Created'), render: data => data ? moment(data).fromNow() : '' }

94
client/src/lists/subscriptions/helpers.js
View file

@ -4,7 +4,7 @@ import React from "react";
import {SubscriptionStatus} from "../../../../shared/lists";
import {ACEEditor, CheckBoxGroup, DatePicker, Dropdown, InputField, RadioGroup, TextArea} from "../../lib/form";
import {formatBirthday, formatDate, parseBirthday, parseDate} from "../../../../shared/date";
import {getFieldKey} from '../../../../shared/lists';
import {getFieldColumn} from '../../../../shared/lists';
import 'brace/mode/json';
export function getSubscriptionStatusLabels(t) {
@ -24,13 +24,13 @@ export function getFieldTypes(t) {
const groupedFieldTypes = {};
const stringFieldType = long => ({
form: groupedField => long ? <TextArea key={getFieldKey(groupedField)} id={getFieldKey(groupedField)} label={groupedField.name}/> : <InputField key={getFieldKey(groupedField)} id={getFieldKey(groupedField)} label={groupedField.name}/>,
form: groupedField => long ? <TextArea key={getFieldColumn(groupedField)} id={getFieldColumn(groupedField)} label={groupedField.name}/> : <InputField key={getFieldColumn(groupedField)} id={getFieldColumn(groupedField)} label={groupedField.name}/>,
assignFormData: (groupedField, data) => {
const value = data[getFieldKey(groupedField)];
data[getFieldKey(groupedField)] = value || '';
const value = data[getFieldColumn(groupedField)];
data[getFieldColumn(groupedField)] = value || '';
},
initFormData: (groupedField, data) => {
data[getFieldKey(groupedField)] = '';
data[getFieldColumn(groupedField)] = '';
},
assignEntity: (groupedField, data) => {},
validate: (groupedField, state) => {},
@ -38,86 +38,86 @@ export function getFieldTypes(t) {
});
const numberFieldType = {
form: groupedField => <InputField key={getFieldKey(groupedField)} id={getFieldKey(groupedField)} label={groupedField.name}/>,
form: groupedField => <InputField key={getFieldColumn(groupedField)} id={getFieldColumn(groupedField)} label={groupedField.name}/>,
assignFormData: (groupedField, data) => {
const value = data[getFieldKey(groupedField)];
data[getFieldKey(groupedField)] = value ? value.toString() : '';
const value = data[getFieldColumn(groupedField)];
data[getFieldColumn(groupedField)] = value ? value.toString() : '';
},
initFormData: (groupedField, data) => {
data[getFieldKey(groupedField)] = '';
data[getFieldColumn(groupedField)] = '';
},
assignEntity: (groupedField, data) => {
data[getFieldKey(groupedField)] = parseInt(data[getFieldKey(groupedField)]);
data[getFieldColumn(groupedField)] = parseInt(data[getFieldColumn(groupedField)]);
},
validate: (groupedField, state) => {
const value = state.getIn([getFieldKey(groupedField), 'value']).trim();
const value = state.getIn([getFieldColumn(groupedField), 'value']).trim();
if (value !== '' && isNaN(value)) {
state.setIn([getFieldKey(groupedField), 'error'], t('Value must be a number'));
state.setIn([getFieldColumn(groupedField), 'error'], t('Value must be a number'));
} else {
state.setIn([getFieldKey(groupedField), 'error'], null);
state.setIn([getFieldColumn(groupedField), 'error'], null);
}
},
indexable: true
};
const dateFieldType = {
form: groupedField => <DatePicker key={getFieldKey(groupedField)} id={getFieldKey(groupedField)} label={groupedField.name} dateFormat={groupedField.settings.dateFormat} />,
form: groupedField => <DatePicker key={getFieldColumn(groupedField)} id={getFieldColumn(groupedField)} label={groupedField.name} dateFormat={groupedField.settings.dateFormat} />,
assignFormData: (groupedField, data) => {
const value = data[getFieldKey(groupedField)];
data[getFieldKey(groupedField)] = value ? formatDate(groupedField.settings.dateFormat, value) : '';
const value = data[getFieldColumn(groupedField)];
data[getFieldColumn(groupedField)] = value ? formatDate(groupedField.settings.dateFormat, value) : '';
},
initFormData: (groupedField, data) => {
data[getFieldKey(groupedField)] = '';
data[getFieldColumn(groupedField)] = '';
},
assignEntity: (groupedField, data) => {
const date = parseDate(groupedField.settings.dateFormat, data[getFieldKey(groupedField)]);
data[getFieldKey(groupedField)] = date;
const date = parseDate(groupedField.settings.dateFormat, data[getFieldColumn(groupedField)]);
data[getFieldColumn(groupedField)] = date;
},
validate: (groupedField, state) => {
const value = state.getIn([getFieldKey(groupedField), 'value']);
const value = state.getIn([getFieldColumn(groupedField), 'value']);
const date = parseDate(groupedField.settings.dateFormat, value);
if (value !== '' && !date) {
state.setIn([getFieldKey(groupedField), 'error'], t('Date is invalid'));
state.setIn([getFieldColumn(groupedField), 'error'], t('Date is invalid'));
} else {
state.setIn([getFieldKey(groupedField), 'error'], null);
state.setIn([getFieldColumn(groupedField), 'error'], null);
}
},
indexable: true
};
const birthdayFieldType = {
form: groupedField => <DatePicker key={getFieldKey(groupedField)} id={getFieldKey(groupedField)} label={groupedField.name} dateFormat={groupedField.settings.dateFormat} birthday />,
form: groupedField => <DatePicker key={getFieldColumn(groupedField)} id={getFieldColumn(groupedField)} label={groupedField.name} dateFormat={groupedField.settings.dateFormat} birthday />,
assignFormData: (groupedField, data) => {
const value = data[getFieldKey(groupedField)];
data[getFieldKey(groupedField)] = value ? formatBirthday(groupedField.settings.dateFormat, value) : '';
const value = data[getFieldColumn(groupedField)];
data[getFieldColumn(groupedField)] = value ? formatBirthday(groupedField.settings.dateFormat, value) : '';
},
initFormData: (groupedField, data) => {
data[getFieldKey(groupedField)] = '';
data[getFieldColumn(groupedField)] = '';
},
assignEntity: (groupedField, data) => {
const date = parseBirthday(groupedField.settings.dateFormat, data[getFieldKey(groupedField)]);
data[getFieldKey(groupedField)] = date;
const date = parseBirthday(groupedField.settings.dateFormat, data[getFieldColumn(groupedField)]);
data[getFieldColumn(groupedField)] = date;
},
validate: (groupedField, state) => {
const value = state.getIn([getFieldKey(groupedField), 'value']);
const value = state.getIn([getFieldColumn(groupedField), 'value']);
const date = parseBirthday(groupedField.settings.dateFormat, value);
if (value !== '' && !date) {
state.setIn([getFieldKey(groupedField), 'error'], t('Date is invalid'));
state.setIn([getFieldColumn(groupedField), 'error'], t('Date is invalid'));
} else {
state.setIn([getFieldKey(groupedField), 'error'], null);
state.setIn([getFieldColumn(groupedField), 'error'], null);
}
},
indexable: true
};
const jsonFieldType = {
form: groupedField => <ACEEditor key={getFieldKey(groupedField)} id={getFieldKey(groupedField)} label={groupedField.name} mode="json" height="300px"/>,
form: groupedField => <ACEEditor key={getFieldColumn(groupedField)} id={getFieldColumn(groupedField)} label={groupedField.name} mode="json" height="300px"/>,
assignFormData: (groupedField, data) => {
const value = data[getFieldKey(groupedField)];
data[getFieldKey(groupedField)] = value || '';
const value = data[getFieldColumn(groupedField)];
data[getFieldColumn(groupedField)] = value || '';
},
initFormData: (groupedField, data) => {
data[getFieldKey(groupedField)] = '';
data[getFieldColumn(groupedField)] = '';
},
assignEntity: (groupedField, data) => {},
validate: (groupedField, state) => {},
@ -125,25 +125,25 @@ export function getFieldTypes(t) {
};
const enumSingleFieldType = componentType => ({
form: groupedField => React.createElement(componentType, { key: getFieldKey(groupedField), id: getFieldKey(groupedField), label: groupedField.name, options: groupedField.settings.options }, null),
form: groupedField => React.createElement(componentType, { key: getFieldColumn(groupedField), id: getFieldColumn(groupedField), label: groupedField.name, options: groupedField.settings.options }, null),
assignFormData: (groupedField, data) => {
if (data[getFieldKey(groupedField)] === null) {
if (data[getFieldColumn(groupedField)] === null) {
if (groupedField.default_value) {
data[getFieldKey(groupedField)] = groupedField.default_value;
data[getFieldColumn(groupedField)] = groupedField.default_value;
} else if (groupedField.settings.options.length > 0) {
data[getFieldKey(groupedField)] = groupedField.settings.options[0].key;
data[getFieldColumn(groupedField)] = groupedField.settings.options[0].key;
} else {
data[getFieldKey(groupedField)] = '';
data[getFieldColumn(groupedField)] = '';
}
}
},
initFormData: (groupedField, data) => {
if (groupedField.default_value) {
data[getFieldKey(groupedField)] = groupedField.default_value;
data[getFieldColumn(groupedField)] = groupedField.default_value;
} else if (groupedField.settings.options.length > 0) {
data[getFieldKey(groupedField)] = groupedField.settings.options[0].key;
data[getFieldColumn(groupedField)] = groupedField.settings.options[0].key;
} else {
data[getFieldKey(groupedField)] = '';
data[getFieldColumn(groupedField)] = '';
}
},
assignEntity: (groupedField, data) => {
@ -153,14 +153,14 @@ export function getFieldTypes(t) {
});
const enumMultipleFieldType = componentType => ({
form: groupedField => React.createElement(componentType, { key: getFieldKey(groupedField), id: getFieldKey(groupedField), label: groupedField.name, options: groupedField.settings.options }, null),
form: groupedField => React.createElement(componentType, { key: getFieldColumn(groupedField), id: getFieldColumn(groupedField), label: groupedField.name, options: groupedField.settings.options }, null),
assignFormData: (groupedField, data) => {
if (data[getFieldKey(groupedField)] === null) {
data[getFieldKey(groupedField)] = [];
if (data[getFieldColumn(groupedField)] === null) {
data[getFieldColumn(groupedField)] = [];
}
},
initFormData: (groupedField, data) => {
data[getFieldKey(groupedField)] = [];
data[getFieldColumn(groupedField)] = [];
},
assignEntity: (groupedField, data) => {},
validate: (groupedField, state) => {},

4
lib/importer.js
View file

@ -22,10 +22,10 @@ function spawn(callback) {
await tx('imports').where('status', ImportStatus.PREP_STOPPING).update({status: ImportStatus.PREP_FAILED});
await tx('imports').where('status', ImportStatus.RUN_RUNNING).update({status: ImportStatus.RUN_SCHEDULED});
await tx('imports').where('status', ImportStatus.RUN_STOPPING).update({status: ImportStatus.RUN_FINISHED});
await tx('imports').where('status', ImportStatus.RUN_STOPPING).update({status: ImportStatus.RUN_FAILED});
await tx('import_runs').where('status', RunStatus.RUNNING).update({status: RunStatus.SCHEDULED});
await tx('import_runs').where('status', RunStatus.STOPPING).update({status: RunStatus.FINISHED});
await tx('import_runs').where('status', RunStatus.STOPPING).update({status: RunStatus.FAILED});
}).then(() => {
importerProcess = fork(path.join(__dirname, '..', 'services', 'importer.js'), [], {

4
lib/subscription-mail-helpers.js
View file

@ -7,7 +7,7 @@ const {getTrustedUrl} = require('./urls');
const _ = require('./translate')._;
const util = require('util');
const contextHelpers = require('./context-helpers');
const {getFieldKey} = require('../shared/lists');
const {getFieldColumn} = require('../shared/lists');
const forms = require('../models/forms');
const mailers = require('./mailers');
@ -101,7 +101,7 @@ async function _sendMail(list, email, template, subject, relativeUrls, subscript
const encryptionKeys = [];
for (const fld of flds) {
if (fld.type === 'gpg' && fld.value) {
encryptionKeys.push(subscription[getFieldKey(fld)].value.trim());
encryptionKeys.push(subscription[getFieldColumn(fld)].value.trim());
}
}

10
lib/tools.js
View file

@ -16,7 +16,6 @@ const fsReadFile = bluebird.promisify(require('fs').readFile);
const jsdomEnv = bluebird.promisify(require('jsdom').env);
const templates = new Map();
async function getTemplate(template) {
@ -74,14 +73,7 @@ async function mergeTemplateIntoLayout(template, layout) {
return source;
}
async function validateEmail(address, checkBlocked) {
let user = (address || '').toString().split('@').shift().toLowerCase().replace(/[^a-z0-9]/g, '');
if (checkBlocked && blockedUsers.indexOf(user) >= 0) {
throw new new Error(util.format(_('Blocked email address "%s"'), address));
}
async function validateEmail(address) {
const result = await new Promise(resolve => {
const result = isemail.validate(address, {
checkDNS: true,

128
models/fields.js
View file

@ -11,7 +11,7 @@ const validators = require('../shared/validators');
const shortid = require('shortid');
const segments = require('./segments');
const { formatDate, formatBirthday, parseDate, parseBirthday } = require('../shared/date');
const { getFieldKey } = require('../shared/lists');
const { getFieldColumn } = require('../shared/lists');
const { cleanupFromPost } = require('../lib/helpers');
@ -157,7 +157,8 @@ fieldTypes.option = {
indexed: true,
grouped: false,
enumerated: false,
cardinality: Cardinality.SINGLE
cardinality: Cardinality.SINGLE,
parsePostValue: (field, value) => !(['false', 'no', '0', ''].indexOf((value || '').toString().trim().toLowerCase()) >= 0)
};
fieldTypes['date'] = {
@ -572,7 +573,7 @@ async function forHbs(context, listId, subscription) { // assumes grouped subscr
for (const fld of flds) {
const type = fieldTypes[fld.type];
const fldKey = getFieldKey(fld);
const fldCol = getFieldColumn(fld);
const entry = {
name: fld.name,
@ -583,12 +584,12 @@ async function forHbs(context, listId, subscription) { // assumes grouped subscr
};
if (!type.grouped && !type.enumerated) {
// subscription[fldKey] may not exists because we are getting the data from "fromPost"
entry.value = (subscription ? type.forHbs(fld, subscription[fldKey]) : null) || '';
// subscription[fldCol] may not exists because we are getting the data from "fromPost"
entry.value = (subscription ? type.forHbs(fld, subscription[fldCol]) : null) || '';
} else if (type.grouped) {
const options = [];
const value = (subscription ? subscription[fldKey] : null) || (type.cardinality === Cardinality.SINGLE ? null : []);
const value = (subscription ? subscription[fldCol] : null) || (type.cardinality === Cardinality.SINGLE ? null : []);
for (const optCol in fld.groupedOptions) {
const opt = fld.groupedOptions[optCol];
@ -611,7 +612,7 @@ async function forHbs(context, listId, subscription) { // assumes grouped subscr
} else if (type.enumerated) {
const options = [];
const value = (subscription ? subscription[fldKey] : null) || null;
const value = (subscription ? subscription[fldCol] : null) || null;
for (const opt of fld.settings.options) {
options.push({
@ -631,64 +632,97 @@ async function forHbs(context, listId, subscription) { // assumes grouped subscr
return customFields;
}
// Converts subscription data received via POST request from subscription form or via subscribe request to API v1 to subscription structure supported by subscriptions model.
// If a field is not specified in the POST data, it omits it also in the returned subscription
async function fromPost(context, listId, data, partial) { // assumes grouped subscription
// This is to handle option values from API v1
function isSelected(value) {
return ['false', 'no', '0', ''].indexOf((value || '').toString().trim().toLowerCase()) >= 0 ? false : true;
}
const flds = await listGrouped(context, listId);
// Converts subscription data received via (1) POST request from subscription form, (2) via subscribe request to API v1 to subscription structure supported by subscriptions model,
// or (3) from import.
// If a field is not specified in the POST data, it is also omitted in the returned subscription
function _fromText(listId, data, flds, isGrouped, keyName, singleCardUsesKeyName) {
const subscription = {};
for (const fld of flds) {
if (fld.key in data) {
const type = fieldTypes[fld.type];
const fldKey = getFieldKey(fld);
if (isGrouped) {
for (const fld of flds) {
const fldKey = fld[keyName];
if (fldKey && fldKey in data) {
const type = fieldTypes[fld.type];
const fldCol = getFieldColumn(fld);
let value = null;
let value = null;
if (!type.grouped && !type.enumerated) {
value = type.parsePostValue(fld, cleanupFromPost(data[fld.key]));
if (!type.grouped && !type.enumerated) {
value = type.parsePostValue(fld, cleanupFromPost(data[fldKey]));
} else if (type.grouped) {
if (type.cardinality === Cardinality.SINGLE) {
for (const optCol in fld.groupedOptions) {
const opt = fld.groupedOptions[optCol];
} else if (type.grouped) {
if (type.cardinality === Cardinality.SINGLE) {
for (const optCol in fld.groupedOptions) {
const opt = fld.groupedOptions[optCol];
const optKey = opt[keyName];
// This handles two different formats for grouped dropdowns and radios.
// The first part of the condition handles the POST requests from the subscription form, while the
// second part handles the subscribe request to API v1
if (data[fld.key] === opt.key || isSelected(data[opt.key])) {
value = opt.column
// This handles two different formats for grouped dropdowns and radios.
// The first part of the condition handles the POST requests from the subscription form, while the
// second part handles the subscribe request to API v1
if (singleCardUsesKeyName) {
if (data[fldKey] === optKey) {
value = opt.column
}
} else {
const optType = fieldTypes[opt.type];
const optValue = optType.parsePostValue(fld, cleanupFromPost(data[optKey]));
if (optValue) {
value = opt.column
}
}
}
} else {
value = [];
for (const optCol in fld.groupedOptions) {
const opt = fld.groupedOptions[optCol];
const optKey = opt[keyName];
const optType = fieldTypes[opt.type];
const optValue = optType.parsePostValue(fld, cleanupFromPost(data[optKey]));
if (optValue) {
value.push(opt.column);
}
}
}
} else {
value = [];
for (const optCol in fld.groupedOptions) {
const opt = fld.groupedOptions[optCol];
if (isSelected(data[opt.key])) {
value.push(opt.column);
}
}
} else if (type.enumerated) {
value = data[fldKey];
}
} else if (type.enumerated) {
value = data[fld.key];
subscription[fldCol] = value;
}
}
subscription[fldKey] = value;
} else {
for (const fld of flds) {
const fldKey = fld[keyName];
if (fldKey && fldKey in data) {
const type = fieldTypes[fld.type];
const fldCol = getFieldColumn(fld);
subscription[fldCol] = type.parsePostValue(fld, cleanupFromPost(data[fldKey]));
}
}
}
return subscription;
}
async function fromPost(context, listId, data) { // assumes grouped subscription and indexation by merge key
const flds = await listGrouped(context, listId);
return _fromText(listId, data, flds, true, 'key', true);
}
async function fromAPI(context, listId, data) { // assumes grouped subscription and indexation by merge key
const flds = await listGrouped(context, listId);
return _fromText(listId, data, flds, true, 'key', false);
}
function fromImport(listId, flds, data) { // assumes ungrouped subscription and indexation by column
return _fromText(listId, data, flds, true, 'column', false);
}
// This is to handle circular dependency with segments.js
Object.assign(module.exports, {
@ -709,5 +743,7 @@ Object.assign(module.exports, {
removeAllByListIdTx,
serverValidate,
forHbs,
fromPost
fromPost,
fromAPI,
fromImport
});

24
models/import-runs.js
View file

@ -35,15 +35,35 @@ async function listDTAjax(context, listId, importId, params) {
builder => builder
.from('import_runs')
.innerJoin('imports', 'import_runs.import', 'imports.id')
.where({'imports.list': listId, 'imports.id': importId}),
.where({'imports.list': listId, 'imports.id': importId})
.orderBy('import_runs.id', 'desc'),
[ 'import_runs.id', 'import_runs.created', 'import_runs.finished', 'import_runs.status', 'import_runs.processed', 'import_runs.new', 'import_runs.failed']
);
});
}
async function listFailedDTAjax(context, listId, importId, importRunId, params) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewImports');
return await dtHelpers.ajaxListTx(
tx,
params,
builder => builder
.from('import_failed')
.innerJoin('import_runs', 'import_failed.run', 'import_runs.id')
.innerJoin('imports', 'import_runs.import', 'imports.id')
.where({'imports.list': listId, 'imports.id': importId, 'import_runs.id': importRunId})
.orderBy('import_failed.source_id', 'asc'),
[ 'import_failed.id', 'import_failed.source_id', 'import_failed.email', 'import_failed.reason']
);
});
}
module.exports = {
getById,
listDTAjax
listDTAjax,
listFailedDTAjax
};

5
models/shares.js
View file

@ -8,6 +8,7 @@ const entitySettings = require('../lib/entity-settings');
const interoperableErrors = require('../shared/interoperable-errors');
const log = require('npmlog');
const {getGlobalNamespaceId} = require('../shared/namespaces');
const {getAdminId} = require('../shared/users');
// TODO: This would really benefit from some permission cache connected to rebuildPermissions
// A bit of the problem is that the cache would have to expunged as the result of other processes modifying entites/permissions
@ -146,7 +147,7 @@ async function rebuildPermissionsTx(tx, restriction) {
// To prevent users locking out themselves, we consider user with id 1 to be the admin and always assign it
// the admin role. The admin role is a global role that has admin===true
// If this behavior is not desired, it is enough to delete the user with id 1.
const adminUser = await tx('users').where('id', 1 /* Admin user id */).first();
const adminUser = await tx('users').where('id', getAdminId()).first();
if (adminUser) {
let adminRole;
for (const role in config.roles.global) {
@ -157,7 +158,7 @@ async function rebuildPermissionsTx(tx, restriction) {
}
if (adminRole) {
await tx('users').update('role', adminRole).where('id', 1 /* Admin user id */);
await tx('users').update('role', adminRole).where('id', getAdminId());
}
}

157
models/subscriptions.js
View file

@ -7,11 +7,12 @@ const dtHelpers = require('../lib/dt-helpers');
const interoperableErrors = require('../shared/interoperable-errors');
const shares = require('./shares');
const fields = require('./fields');
const { SubscriptionStatus, getFieldKey } = require('../shared/lists');
const { SubscriptionStatus, getFieldColumn } = require('../shared/lists');
const segments = require('./segments');
const { enforce, filterObject } = require('../lib/helpers');
const moment = require('moment');
const { formatDate, formatBirthday } = require('../shared/date');
const crypto = require('crypto');
const allowedKeysBase = new Set(['email', 'tz', 'is_test', 'status']);
@ -54,7 +55,7 @@ fieldTypes['radio-enum'] = fieldTypes['dropdown-enum'] = fieldTypes['radio-group
fieldTypes.date = {
afterJSON: (groupedField, entity) => {
const key = getFieldKey(groupedField);
const key = getFieldColumn(groupedField);
if (key in entity) {
entity[key] = entity[key] ? moment(entity[key]).toDate() : null;
}
@ -64,7 +65,7 @@ fieldTypes.date = {
fieldTypes.birthday = {
afterJSON: (groupedField, entity) => {
const key = getFieldKey(groupedField);
const key = getFieldColumn(groupedField);
if (key in entity) {
entity[key] = entity[key] ? moment(entity[key]).toDate() : null;
}
@ -86,14 +87,14 @@ async function getGroupedFieldsMap(tx, listId) {
const groupedFields = await fields.listGroupedTx(tx, listId);
const result = {};
for (const fld of groupedFields) {
result[getFieldKey(fld)] = fld;
result[getFieldColumn(fld)] = fld;
}
return result;
}
function groupSubscription(groupedFieldsMap, entity) {
for (const fldKey in groupedFieldsMap) {
const fld = groupedFieldsMap[fldKey];
for (const fldCol in groupedFieldsMap) {
const fld = groupedFieldsMap[fldCol];
const fieldType = fields.getFieldType(fld.type);
if (fieldType.grouped) {
@ -124,7 +125,7 @@ function groupSubscription(groupedFieldsMap, entity) {
}
}
entity[fldKey] = value;
entity[fldCol] = value;
} else if (fieldType.enumerated) {
// This is enum-xxx type. We just make sure that the options we give out match the field settings.
@ -132,36 +133,36 @@ function groupSubscription(groupedFieldsMap, entity) {
const allowedKeys = new Set(fld.settings.options.map(x => x.key));
if (!allowedKeys.has(entity[fldKey])) {
entity[fldKey] = null;
if (!allowedKeys.has(entity[fldCol])) {
entity[fldCol] = null;
}
}
}
}
function ungroupSubscription(groupedFieldsMap, entity) {
for (const fldKey in groupedFieldsMap) {
const fld = groupedFieldsMap[fldKey];
for (const fldCol in groupedFieldsMap) {
const fld = groupedFieldsMap[fldCol];
const fieldType = fields.getFieldType(fld.type);
if (fieldType.grouped) {
if (fieldType.cardinality === fields.Cardinality.SINGLE) {
const value = entity[fldKey];
const value = entity[fldCol];
for (const optionKey in fld.groupedOptions) {
const option = fld.groupedOptions[optionKey];
entity[option.column] = option.column === value;
}
} else {
const values = entity[fldKey] || []; // The default (empty array) is here because create may be called with an entity that has some fields not filled in
const values = entity[fldCol] || []; // The default (empty array) is here because create may be called with an entity that has some fields not filled in
for (const optionKey in fld.groupedOptions) {
const option = fld.groupedOptions[optionKey];
entity[option.column] = values.includes(option.column);
}
}
delete entity[fldKey];
delete entity[fldCol];
} else if (fieldType.enumerated) {
// This is enum-xxx type. We just make sure that the options we give out match the field settings.
@ -169,8 +170,8 @@ function ungroupSubscription(groupedFieldsMap, entity) {
const allowedKeys = new Set(fld.settings.options.map(x => x.key));
if (!allowedKeys.has(entity[fldKey])) {
entity[fldKey] = null;
if (!allowedKeys.has(entity[fldCol])) {
entity[fldCol] = null;
}
}
}
@ -259,29 +260,29 @@ async function listDTAjax(context, listId, segmentId, params) {
const idxMap = {};
for (const listFld of listFlds) {
const fldKey = getFieldKey(listFld);
const fld = groupedFieldsMap[fldKey];
const fldCol = getFieldColumn(listFld);
const fld = groupedFieldsMap[fldCol];
if (fld.column) {
columns.push(listTable + '.' + fld.column);
} else {
columns.push({
name: listTable + '.' + fldKey,
name: listTable + '.' + fldCol,
raw: 0
})
}
idxMap[fldKey] = listFldIdx;
idxMap[fldCol] = listFldIdx;
listFldIdx += 1;
}
for (const fldKey in groupedFieldsMap) {
const fld = groupedFieldsMap[fldKey];
for (const fldCol in groupedFieldsMap) {
const fld = groupedFieldsMap[fldCol];
if (fld.column) {
if (!(fldKey in idxMap)) {
if (!(fldCol in idxMap)) {
extraColumns.push(listTable + '.' + fld.column);
idxMap[fldKey] = listFldIdx;
idxMap[fldCol] = listFldIdx;
listFldIdx += 1;
}
@ -313,19 +314,19 @@ async function listDTAjax(context, listId, segmentId, params) {
{
mapFun: data => {
const entity = {};
for (const fldKey in idxMap) {
for (const fldCol in idxMap) {
// This is a bit of hacking. We rely on the fact that if a field has a column, then the column is the field key.
// Then it has the group id with value 0. groupSubscription will be able to process the fields that have a column
// and it will assign values to the fields that don't have a value (i.e. those that currently have the group id and value 0).
entity[fldKey] = data[idxMap[fldKey]];
entity[fldCol] = data[idxMap[fldCol]];
}
groupSubscription(groupedFieldsMap, entity);
for (const listFld of listFlds) {
const fldKey = getFieldKey(listFld);
const fld = groupedFieldsMap[fldKey];
data[idxMap[fldKey]] = fieldTypes[fld.type].listRender(fld, entity[fldKey]);
const fldCol = getFieldColumn(listFld);
const fld = groupedFieldsMap[fldCol];
data[idxMap[fldCol]] = fieldTypes[fld.type].listRender(fld, entity[fldCol]);
}
},
@ -393,7 +394,7 @@ async function serverValidate(context, listId, data) {
async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta, isCreate) {
enforce(entity.email, 'Email must be set');
const existingWithKeyQuery = tx(getSubscriptionTableName(listId)).where('email', entity.email);
const existingWithKeyQuery = tx(getSubscriptionTableName(listId)).where('hash_email', hashEmail(entity.email));
if (!isCreate) {
existingWithKeyQuery.whereNot('id', entity.id);
@ -407,9 +408,11 @@ async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta
throw new interoperableErrors.DuplicitEmailError();
}
} else {
// This is here because of the API, which allows one to send subscriptions without caring about whether they already exist, what their status is, etc.
// In the case, the subscription is existing, we should not change the status. If it does not exist, we are fine with changing the status
if (meta.subscribeIfNoExisting && !entity.status) {
// This is here because of the API endpoint, which allows one to submit subscriptions without caring about whether they already exist, what their status is, etc.
// The same for import where we need to subscribed only those (existing and new) that have not been unsubscribed already.
// In the case, the subscription is existing, we should not change the status. If it does not exist, we are fine with changing the status to SUBSCRIBED
if (meta && meta.subscribeIfNoExisting && !entity.status) {
entity.status = SubscriptionStatus.SUBSCRIBED;
}
}
@ -425,14 +428,18 @@ async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta
}
}
function updateSourcesAndHash(subscription, source, groupedFieldsMap) {
function hashEmail(email) {
return crypto.createHash('sha512').update(email).digest("base64");
}
function updateSourcesAndHashEmail(subscription, source, groupedFieldsMap) {
if ('email' in subscription) {
subscription.hash_email = crypto.createHash('sha512').update(subscription.email).digest("base64");
subscription.hash_email = hashEmail(subscription.email);
subscription.source_email = source;
}
for (const fldKey in groupedFieldsMap) {
const fld = groupedFieldsMap[fldKey];
for (const fldCol in groupedFieldsMap) {
const fld = groupedFieldsMap[fldCol];
const fieldType = fields.getFieldType(fld.type);
if (fieldType.grouped) {
@ -444,8 +451,8 @@ function updateSourcesAndHash(subscription, source, groupedFieldsMap) {
}
}
} else {
if (fldKey in subscription) {
subscription['source_' + fldKey] = source;
if (fldCol in subscription) {
subscription['source_' + fldCol] = source;
}
}
}
@ -490,39 +497,42 @@ async function _create(tx, listId, filteredEntity) {
If it is unsubscribed and meta.updateOfUnsubscribedAllowed, the existing subscription is changed based on the provided data.
If meta.updateAllowed is true, it updates even an active subscription.
*/
async function create(context, listId, entity, source, meta /* meta is provided when called from /confirm/subscribe/:cid */) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
async function createTxWithGroupedFieldsMap(tx, context, listId, groupedFieldsMap, entity, source, meta) {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
const groupedFieldsMap = await getGroupedFieldsMap(tx, listId);
const allowedKeys = getAllowedKeys(groupedFieldsMap);
const allowedKeys = getAllowedKeys(groupedFieldsMap);
await _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta, true);
await _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta, true);
const filteredEntity = filterObject(entity, allowedKeys);
filteredEntity.status_change = new Date();
const filteredEntity = filterObject(entity, allowedKeys);
filteredEntity.status_change = new Date();
ungroupSubscription(groupedFieldsMap, filteredEntity);
ungroupSubscription(groupedFieldsMap, filteredEntity);
updateSourcesAndHash(filteredEntity, source, groupedFieldsMap);
updateSourcesAndHashEmail(filteredEntity, source, groupedFieldsMap);
filteredEntity.opt_in_ip = meta && meta.ip;
filteredEntity.opt_in_country = meta && meta.country;
filteredEntity.imported = meta && !!meta.imported;
filteredEntity.opt_in_ip = meta && meta.ip;
filteredEntity.opt_in_country = meta && meta.country;
if (meta && meta.update) { // meta.update is set by _validateAndPreprocess
await _update(tx, listId, meta.existing, filteredEntity);
meta.cid = meta.existing.cid; // The cid is needed by /confirm/subscribe/:cid
return meta.existing.id;
} else {
filteredEntity.cid = shortid.generate();
if (meta && meta.update) { // meta.update is set by _validateAndPreprocess
await _update(tx, listId, meta.existing, filteredEntity);
meta.cid = meta.existing.cid; // The cid is needed by /confirm/subscribe/:cid
return meta.existing.id;
} else {
filteredEntity.cid = shortid.generate();
if (meta) {
meta.cid = filteredEntity.cid; // The cid is needed by /confirm/subscribe/:cid
}
return await _create(tx, listId, filteredEntity);
if (meta) {
meta.cid = filteredEntity.cid; // The cid is needed by /confirm/subscribe/:cid
}
return await _create(tx, listId, filteredEntity);
}
}
async function create(context, listId, entity, source, meta) {
return await knex.transaction(async tx => {
const groupedFieldsMap = await getGroupedFieldsMap(tx, listId);
return await createTxWithGroupedFieldsMap(tx, context, listId, groupedFieldsMap, entity, source, meta);
});
}
@ -551,7 +561,7 @@ async function updateWithConsistencyCheck(context, listId, entity, source) {
ungroupSubscription(groupedFieldsMap, filteredEntity);
updateSourcesAndHash(filteredEntity, source, groupedFieldsMap);
updateSourcesAndHashEmail(filteredEntity, source, groupedFieldsMap);
await _update(tx, listId, existing, filteredEntity);
});
@ -580,7 +590,7 @@ async function remove(context, listId, id) {
async function removeByEmailAndGet(context, listId, email) {
return await knex.transaction(async tx => {
const existing = await tx(getSubscriptionTableName(listId)).where('email', email).first();
const existing = await tx(getSubscriptionTableName(listId)).where('hash_email', hashEmail(email)).first();
return await _removeAndGetTx(tx, context, listId, existing);
});
}
@ -624,21 +634,25 @@ async function _unsubscribeAndGetTx(tx, context, listId, existingSubscription, c
async function unsubscribeByIdAndGet(context, listId, subscriptionId) {
return await knex.transaction(async tx => {
const existing = await tx(getSubscriptionTableName(listId)).where('id', subscriptionId).first();
return _unsubscribeAndGetTx(tx, context, listId, existing);
return await _unsubscribeAndGetTx(tx, context, listId, existing);
});
}
async function unsubscribeByCidAndGet(context, listId, subscriptionCid, campaignCid) {
return await knex.transaction(async tx => {
const existing = await tx(getSubscriptionTableName(listId)).where('cid', subscriptionCid).first();
return _unsubscribeAndGetTx(tx, context, listId, existing, campaignCid);
return await _unsubscribeAndGetTx(tx, context, listId, existing, campaignCid);
});
}
async function unsubscribeByEmailAndGetTx(tx, context, listId, email) {
const existing = await tx(getSubscriptionTableName(listId)).where('hash_email', hashEmail(email)).first();
return await _unsubscribeAndGetTx(tx, context, listId, existing);
}
async function unsubscribeByEmailAndGet(context, listId, email) {
return await knex.transaction(async tx => {
const existing = await tx(getSubscriptionTableName(listId)).where('email', email).first();
return _unsubscribeAndGetTx(tx, context, listId, existing);
return await unsubscribeByEmailAndGetTx(tx, context, listId, email);
});
}
@ -652,7 +666,7 @@ async function updateAddressAndGet(context, listId, subscriptionId, emailNew) {
}
if (existing.email !== emailNew) {
await tx(getSubscriptionTableName(listId)).where('email', emailNew).del();
await tx(getSubscriptionTableName(listId)).where('hash_email', hashEmail(emailNew)).del();
await tx(getSubscriptionTableName(listId)).where('id', subscriptionId).update({
email: emailNew
@ -717,12 +731,15 @@ module.exports = {
listDTAjax,
serverValidate,
create,
getGroupedFieldsMap,
createTxWithGroupedFieldsMap,
updateWithConsistencyCheck,
remove,
removeByEmailAndGet,
unsubscribeByCidAndGet,
unsubscribeByIdAndGet,
unsubscribeByEmailAndGet,
unsubscribeByEmailAndGetTx,
updateAddressAndGet,
updateManaged,
getListsWithEmail

4
routes/api.js
View file

@ -37,14 +37,14 @@ router.postAsync('/subscribe/:listCid', passport.loggedIn, async (req, res) => {
throw new APIError('Missing EMAIL', 400);
}
const emailErr = await tools.validateEmail(input.EMAIL, false);
const emailErr = await tools.validateEmail(input.EMAIL);
if (emailErr) {
const errMsg = tools.validateEmailGetMessage(emailErr, input.email);
log.error('API', errMsg);
throw new APIError(errMsg, 400);
}
const subscription = await fields.fromPost(req.context, list.id, input, true);
const subscription = await fields.fromAPI(req.context, list.id, input);
if (input.TIMEZONE) {
subscription.tz = (input.TIMEZONE || '').toString().trim();

519
routes/rest/editors.js
View file

@ -8,525 +8,6 @@ const premailerPrepareAsync = bluebird.promisify(premailerApi.prepare);
const router = require('../../lib/router-async').create();
/*
FIXME
const { nodeifyFunction } = require('../lib/nodeify');
const getSettings = nodeifyFunction(require('../models/settings').get);
const htmlToText = require('html-to-text');
const log = require('npmlog');
const config = require('config');
const express = require('express');
const router = new express.Router();
const passport = require('../lib/passport');
const os = require('os');
const fs = require('fs');
const path = require('path');
const mkdirp = require('mkdirp');
const crypto = require('crypto');
const events = require('events');
const httpMocks = require('node-mocks-http');
const multiparty = require('multiparty');
const escapeStringRegexp = require('escape-string-regexp');
const jqueryFileUpload = require('jquery-file-upload-middleware');
const gm = require('gm').subClass({
imageMagick: true
});
const url = require('url');
const _ = require('../lib/translate')._;
const mailer = require('../lib/mailer');
const templates = require('../lib/models/templates');
const campaigns = require('../lib/models/campaigns');
router.all('/*', (req, res, next) => {
if (!req.user) {
return res.status(403).send(_('Need to be logged in to access restricted content'));
}
if (req.originalUrl.startsWith('/editorapi/img?')) {
return next();
}
if (!config.editors.map(e => e[0]).includes(req.query.editor)) {
return res.status(500).send(_('Invalid editor name'));
}
next();
});
jqueryFileUpload.on('begin', fileInfo => {
fileInfo.name = fileInfo.name
.toLowerCase()
.replace(/ /g, '-')
.replace(/[^a-z0-9+-.]+/g, '');
});
const listImages = (dir, dirURL, callback) => {
fs.readdir(dir, (err, files = []) => {
if (err && err.code !== 'ENOENT') {
return callback(err.message || err);
}
files = files.filter(name => /\.(jpe?g|png|gif)$/i.test(name));
files = files.map(name => ({
// mosaico
name,
url: dirURL + '/' + name,
thumbnailUrl: dirURL + '/thumbnail/' + name,
// grapejs
src: dirURL + '/' + name
}));
callback(null, files);
});
};
const placeholderImage = (width, height, callback) => {
const magick = gm(width, height, '#707070');
const size = 40;
let x = 0;
let y = 0;
// stripes
while (y < height) {
magick
.fill('#808080')
.drawPolygon([x, y], [x + size, y], [x + size * 2, y + size], [x + size * 2, y + size * 2])
.drawPolygon([x, y + size], [x + size, y + size * 2], [x, y + size * 2]);
x = x + size * 2;
if (x > width) {
x = 0;
y = y + size * 2;
}
}
// text
magick
.fill('#B0B0B0')
.fontSize(20)
.drawText(0, 0, width + ' x ' + height, 'center');
magick.stream('png', (err, stream) => {
if (err) {
return callback(err);
}
const image = {
format: 'PNG',
stream
};
callback(null, image);
});
};
const resizedImage = (src, method, width, height, callback) => {
const pathname = path.join('/', url.parse(src).pathname);
const filePath = path.join(__dirname, '..', 'public', pathname);
const magick = gm(filePath);
magick.format((err, format) => {
if (err) {
return callback(err);
}
const streamHandler = (err, stream) => {
if (err) {
return callback(err);
}
const image = {
format,
stream
};
callback(null, image);
};
switch (method) {
case 'resize':
return magick
.autoOrient()
.resize(width, height)
.stream(streamHandler);
case 'cover':
return magick
.autoOrient()
.resize(width, height + '^')
.gravity('Center')
.extent(width, height + '>')
.stream(streamHandler);
default:
return callback(new Error(_('Method not supported')));
}
});
};
const getProcessedImage = (dynamicUrl, callback) => {
if (!dynamicUrl.includes('/editorapi/img?')) {
return callback(new Error('Invalid dynamicUrl'));
}
const {
src,
method,
params = '600,null'
} = url.parse(dynamicUrl, true).query;
let width = params.split(',')[0];
let height = params.split(',')[1];
const sanitizeSize = (val, min, max, defaultVal, allowNull) => {
if (val === 'null' && allowNull) {
return null;
}
val = Number(val) || defaultVal;
val = Math.max(min, val);
val = Math.min(max, val);
return val;
};
if (method === 'placeholder') {
width = sanitizeSize(width, 1, 2048, 600, false);
height = sanitizeSize(height, 1, 2048, 300, false);
placeholderImage(width, height, callback);
} else {
width = sanitizeSize(width, 1, 2048, 600, false);
height = sanitizeSize(height, 1, 2048, 300, true);
resizedImage(src, method, width, height, callback);
}
};
const getStaticImageUrl = (dynamicUrl, staticDir, staticDirUrl, callback) => {
if (!dynamicUrl.includes('/editorapi/img?')) {
return callback(null, dynamicUrl);
}
mkdirp(staticDir, err => {
if (err) {
return callback(err);
}
fs.readdir(staticDir, (err, files) => {
if (err) {
return callback(err);
}
const hash = crypto.createHash('md5').update(dynamicUrl).digest('hex');
const match = files.find(el => el.startsWith(hash));
if (match) {
return callback(null, staticDirUrl + '/' + match);
}
getProcessedImage(dynamicUrl, (err, image) => {
if (err) {
return callback(err);
}
const fileName = hash + '.' + image.format.toLowerCase();
const filePath = path.join(staticDir, fileName);
const fileUrl = staticDirUrl + '/' + fileName;
const writeStream = fs.createWriteStream(filePath);
writeStream.on('error', err => callback(err));
writeStream.on('finish', () => callback(null, fileUrl));
image.stream.pipe(writeStream);
});
});
});
};
const prepareHtml = (html, editorName, callback) => {
getSettings('serviceUrl', (err, serviceUrl) => {
if (err) {
return callback(err.message || err);
}
const srcs = new Map();
const re = /<img[^>]+src="([^"]*\/editorapi\/img\?[^"]+)"/ig;
let jobs = 0;
let result;
while ((result = re.exec(html)) !== null) {
srcs.set(result[1], result[1]);
}
const done = () => {
if (jobs === 0) {
for (const [key, value] of srcs) {
// console.log(`replace dynamicUrl: ${key} - with staticUrl: ${value}`);
html = html.replace(new RegExp(escapeStringRegexp(key), 'g'), value);
}
return callback(null, html);
}
};
const staticDir = path.join(__dirname, '..', 'public', editorName, 'uploads', 'static');
const staticDirUrl = url.resolve(serviceUrl, editorName + '/uploads/static');
for (const key of srcs.keys()) {
jobs++;
const dynamicUrl = key.replace(/&amp;/g, '&');
getStaticImageUrl(dynamicUrl, staticDir, staticDirUrl, (err, staticUrl) => {
if (err) {
// TODO: Send a warning back to the editor. For now we just skip image resizing.
log.error('editorapi', err);
if (dynamicUrl.includes('/editorapi/img?')) {
staticUrl = url.parse(dynamicUrl, true).query.src || dynamicUrl;
} else {
staticUrl = dynamicUrl;
}
if (!/^https?:\/\/|^\/\//i.test(staticUrl)) {
staticUrl = url.resolve(serviceUrl, staticUrl);
}
}
srcs.set(key, staticUrl);
jobs--;
done();
});
}
done();
});
};
// URL structure defined by Mosaico
// /editorapi/img?src=" + encodeURIComponent(src) + "&method=" + encodeURIComponent(method) + "&params=" + encodeURIComponent(width + "," + height);
router.get('/img', (req, res) => {
getProcessedImage(req.originalUrl, (err, image) => {
if (err) {
res.status(err.status || 500);
res.send(err.message || err);
return;
}
res.set('Content-Type', 'image/' + image.format.toLowerCase());
image.stream.pipe(res);
});
});
router.post('/update', passport.parseForm, passport.csrfProtection, (req, res) => {
const sendResponse = err => {
if (err) {
return res.status(500).send(err.message || err);
}
res.send('ok');
};
prepareHtml(req.body.html, req.query.editor, (err, html) => {
if (err) {
return sendResponse(err);
}
req.body.html = html;
switch (req.query.type) {
case 'template':
return templates.update(req.body.id, req.body, sendResponse);
case 'campaign':
return campaigns.update(req.body.id, req.body, sendResponse);
default:
return sendResponse(new Error(_('Invalid resource type')));
}
});
});
// https://github.com/artf/grapesjs/wiki/API-Asset-Manager
// https://github.com/aguidrevitch/jquery-file-upload-middleware
router.get('/upload', passport.csrfProtection, (req, res) => {
getSettings('serviceUrl', (err, serviceUrl) => {
if (err) {
return res.status(500).send(err.message || err);
}
const baseDir = path.join(__dirname, '..', 'public', req.query.editor, 'uploads');
const baseDirUrl = serviceUrl + req.query.editor + '/uploads';
listImages(path.join(baseDir, '0'), baseDirUrl + '/0', (err, sharedImages) => {
if (err) {
return res.status(500).send(err.message || err);
}
if (req.query.type === 'campaign' && Number(req.query.id) > 0) {
listImages(path.join(baseDir, req.query.id), baseDirUrl + '/' + req.query.id, (err, campaignImages) => {
if (err) {
return res.status(500).send(err.message || err);
}
res.json({
files: sharedImages.concat(campaignImages)
});
});
} else {
res.json({
files: sharedImages
});
}
});
});
});
router.post('/upload', passport.csrfProtection, (req, res) => {
getSettings('serviceUrl', (err, serviceUrl) => {
if (err) {
return res.status(500).send(err.message || err);
}
const getDirName = () => {
switch (req.query.type) {
case 'template':
return '0';
case 'campaign':
return Number(req.query.id) > 0 ? req.query.id : false;
default:
return false;
}
};
const dirName = getDirName();
const serviceUrlParts = url.parse(serviceUrl);
if (dirName === false) {
return res.status(500).send(_('Invalid resource type or ID'));
}
const opts = {
tmpDir: config.www.tmpdir || os.tmpdir(),
imageVersions: req.query.editor === 'mosaico' ? {
thumbnail: {
width: 90,
height: 90
}
} : {},
uploadDir: path.join(__dirname, '..', 'public', req.query.editor, 'uploads', dirName),
uploadUrl: '/' + req.query.editor + '/uploads/' + dirName, // must be root relative
acceptFileTypes: /\.(gif|jpe?g|png)$/i,
hostname: serviceUrlParts.host, // include port
ssl: serviceUrlParts.protocol === 'https:'
};
const mockres = httpMocks.createResponse({
eventEmitter: events.EventEmitter
});
mockres.on('error', err => {
res.status(500).json({
error: err.message || err,
data: []
});
});
mockres.on('end', () => {
const data = [];
try {
JSON.parse(mockres._getData()).files.forEach(file => {
data.push({
src: file.url
});
});
res.json({
data
});
} catch(err) {
res.status(500).json({
error: err.message || err,
data
});
}
});
jqueryFileUpload.fileHandler(opts)(req, req.query.editor === 'grapejs' ? mockres : res);
});
});
router.post('/download', passport.csrfProtection, (req, res) => {
prepareHtml(req.body.html, req.query.editor, (err, html) => {
if (err) {
return res.status(500).send(err.message || err);
}
res.setHeader('Content-disposition', 'attachment; filename=' + req.body.filename);
res.setHeader('Content-type', 'text/html');
res.send(html);
});
});
const parseGrapejsMultipartTestForm = (req, res, next) => {
if (req.query.editor === 'grapejs') {
new multiparty.Form().parse(req, (err, fields) => {
if (err) {
return next(err);
}
req.body.email = fields.email[0];
req.body.subject = fields.subject[0];
req.body.html = fields.html[0];
req.body._csrf = fields._csrf[0];
next();
});
} else {
next();
}
};
router.post('/test', parseGrapejsMultipartTestForm, passport.csrfProtection, (req, res) => {
const sendError = err => {
if (req.query.editor === 'grapejs') {
res.status(500).json({
errors: err.message || err
});
} else {
res.status(500).send(err.message || err);
}
};
prepareHtml(req.body.html, req.query.editor, (err, html) => {
if (err) {
return sendError(err);
}
getSettings(['defaultAddress', 'defaultFrom'], (err, configItems) => {
if (err) {
return sendError(err);
}
mailer.getMailer((err, transport) => {
if (err) {
return sendError(err);
}
const opts = {
from: {
name: configItems.defaultFrom,
address: configItems.defaultAddress
},
to: req.body.email,
subject: req.body.subject,
text: htmlToText.fromString(html, {
wordwrap: 100
}),
html
};
transport.sendMail(opts, err => {
if (err) {
return sendError(err);
}
if (req.query.editor === 'grapejs') {
res.json({
data: 'ok'
});
} else {
res.send('ok');
}
});
});
});
});
});
*/
router.postAsync('/html-to-text', passport.loggedIn, passport.csrfProtection, async (req, res) => {
const email = await premailerPrepareAsync({
html: req.body.html,

4
routes/rest/import-runs.js
View file

@ -9,6 +9,10 @@ router.postAsync('/import-runs-table/:listId/:importId', passport.loggedIn, asyn
return res.json(await importRuns.listDTAjax(req.context, req.params.listId, req.params.importId, req.body));
});
router.postAsync('/import-run-failed-table/:listId/:importId/:importRunId', passport.loggedIn, async (req, res) => {
return res.json(await importRuns.listFailedDTAjax(req.context, req.params.listId, req.params.importId, req.params.importRunId, req.body));
});
router.getAsync('/import-runs/:listId/:importId/:runId', passport.loggedIn, async (req, res) => {
const entity = await importRuns.getById(req.context, req.params.listId, req.params.importId, req.params.runId);
return res.json(entity);

241
services/importer.js
View file

@ -6,14 +6,22 @@ const log = require('npmlog');
const fsExtra = require('fs-extra-promise');
const {ImportSource, MappingType, ImportStatus, RunStatus} = require('../shared/imports');
const imports = require('../models/imports');
const fields = require('../models/fields');
const subscriptions = require('../models/subscriptions');
const { Writable } = require('stream');
const { enforce } = require('../lib/helpers');
const { cleanupFromPost, enforce } = require('../lib/helpers');
const contextHelpers = require('../lib/context-helpers');
const tools = require('../lib/tools');
const shares = require('../models/shares');
const _ = require('../lib/translate')._;
const csvparse = require('csv-parse');
const fs = require('fs');
let running = false;
const maxInsertBatchSize = 100;
const maxPrepareBatchSize = 100;
const maxImportBatchSize = 10;
function prepareCsv(impt) {
// Processing of CSV intake
@ -71,6 +79,8 @@ function prepareCsv(impt) {
}
impt.settings.csv.columns = cols;
impt.settings.sourceTable = importTable;
await knex('imports').where({id: impt.id}).update({settings: JSON.stringify(impt.settings)});
await knex.schema.raw('CREATE TABLE `' + importTable + '` (\n' +
@ -88,7 +98,7 @@ function prepareCsv(impt) {
insertBatch.push(dbRecord);
}
if (insertBatch.length >= maxInsertBatchSize) {
if (insertBatch.length >= maxPrepareBatchSize) {
await knex(importTable).insert(insertBatch);
insertBatch = [];
}
@ -126,27 +136,220 @@ function prepareCsv(impt) {
inputStream.pipe(parser);
}
async function basicSubscribe(impt) {
let imptRun;
while (imptRun = await knex('import_runs').where('import', impt.id).whereIn('status', [RunStatus.SCHEDULED]).orderBy('created', 'asc').first()) {
await knex('import_runs').where('id', imptRun.id).update({
status: RunStatus.RUNNING
async function _execImportRun(impt, handlers) {
// FIXME - handle STOPPING
try {
let imptRun;
// It should not really happen that we have more than one run to be processed for an import. However, to be on the safe side, we process it in a while.
while (imptRun = await knex('import_runs').where('import', impt.id).whereIn('status', [RunStatus.SCHEDULED]).orderBy('created', 'asc').first()) {
try {
imptRun.mapping = JSON.parse(imptRun.mapping) || {};
log.info('Importer', `Starting BASIC_SUBSCRIBE run ${impt.id}.${imptRun.id}`);
await knex('import_runs').where('id', imptRun.id).update({
status: RunStatus.RUNNING
});
const importTable = impt.settings.sourceTable;
const flds = await fields.list(contextHelpers.getAdminContext(), impt.list);
let lastId = imptRun.last_id || 0;
let countNew = imptRun.new || 0;
let countProcessed = imptRun.processed || 0;
let countFailed = imptRun.failed || 0;
while (true) {
const rows = await knex(importTable).orderBy('id', 'asc').where('id', '>', lastId).limit(maxImportBatchSize);
log.verbose('Importer', `Processing run ${impt.id}.${imptRun.id} with id > ${lastId} ... ${rows.length} entries`);
if (rows.length === 0) {
break;
}
const subscrs = [];
const unsubscrs = [];
const failures = [];
// This should help in case we do the DNS check inside process row because it does all the checks at the same time.
await Promise.all(rows.map(row => handlers.processSourceRow(impt, imptRun, flds, row, subscrs, unsubscrs, failures)));
lastId = rows[rows.length - 1].id;
await knex.transaction(async tx => {
const groupedFieldsMap = await subscriptions.getGroupedFieldsMap(tx, impt.list);
let newRows = 0;
for (const subscr of subscrs) {
const meta = {
updateAllowed: true,
updateOfUnsubscribedAllowed: true,
subscribeIfNoExisting: true
};
try {
await subscriptions.createTxWithGroupedFieldsMap(tx, contextHelpers.getAdminContext(), impt.list, groupedFieldsMap, subscr, impt.id, meta);
if (!meta.existing) {
newRows += 1;
}
} catch (err) {
failures.push({
run: imptRun.id,
source_id: subscr.source_id,
email: subscr.email,
reason: err.message
});
}
}
for (const unsubscr of unsubscrs) {
try {
await subscriptions.unsubscribeByEmailAndGetTx(tx, contextHelpers.getAdminContext(), impt.list, unsubscr.email);
} catch (err) {
failures.push({
run: imptRun.id,
source_id: unsubscr.source_id,
email: unsubscr.email,
reason: err.message
});
}
}
countProcessed += rows.length;
countNew += newRows;
countFailed += failures.length;
if (failures.length > 0) {
await tx('import_failed').insert(failures);
}
await tx('import_runs').where('id', imptRun.id).update({
last_id: lastId,
new: countNew,
failed: countFailed,
processed: countProcessed
});
});
const imptRunStatus = await knex('import_runs').where('id', imptRun.id).select(['status']).first();
if (imptRunStatus.status === RunStatus.STOPPING) {
throw new Error('Aborted');
}
}
await knex('import_runs').where('id', imptRun.id).update({
status: RunStatus.FINISHED,
error: null,
finished: new Date()
});
log.info('Importer', `BASIC_SUBSCRIBE run ${impt.id}.${imptRun.id} finished`);
} catch (err) {
await knex('import_runs').where('id', imptRun.id).update({
status: RunStatus.FAILED,
error: err.message,
finished: new Date()
});
throw new Error(_('Last run failed'));
}
}
await knex('imports').where('id', impt.id).update({
last_run: new Date(),
error: null,
status: ImportStatus.RUN_FINISHED
});
await knex('import_runs').where('id', imptRun.id).update({
status: RunStatus.FINISHED
} catch (err) {
await knex('imports').where('id', impt.id).update({
last_run: new Date(),
error: err.message,
status: ImportStatus.RUN_FAILED
});
}
}
await knex('imports').where('id', impt.id).update({
status: ImportStatus.RUN_FINISHED
});
async function basicSubscribe(impt) {
const handlers = {
processSourceRow: async (impt, imptRun, flds, row, subscriptions, unsubscriptions, failures) => {
const mappingFields = imptRun.mapping.fields || {};
const mappingSettings = imptRun.mapping.settings || {};
const convRow = {};
for (const col in mappingFields) {
const fldMapping = mappingFields[col];
if (fldMapping && fldMapping.column) {
convRow[col] = row[fldMapping.column];
}
}
const subscription = fields.fromImport(impt.list, flds, convRow);
const email = cleanupFromPost(convRow.email);
let errorMsg;
if (!email) {
errorMsg = _('Missing email');
}
if (mappingSettings.checkEmails) {
const emailErr = await tools.validateEmail(email);
if (emailErr) {
errorMsg = tools.validateEmailGetMessage(emailErr, email);
}
}
if (!errorMsg) {
subscription.email = email;
subscription.source_id = row.id;
subscriptions.push(subscription);
} else {
failures.push({
run: imptRun.id,
source_id: row.id,
email: email,
reason: errorMsg
});
}
}
};
return await _execImportRun(impt, handlers);
}
async function basicUnsubscribe(impt) {
// FIXME
const handlers = {
processSourceRow: async (impt, imptRun, flds, row, subscriptions, unsubscriptions, failures) => {
const emailCol = imptRun.mapping.fields.email.column;
const email = cleanupFromPost(row[emailCol]);
let errorMsg;
if (!email) {
errorMsg = _('Missing email');
}
if (!errorMsg) {
unsubscriptions.push({
source_id: row.id,
email
});
} else {
failures.push({
run: imptRun.id,
source_id: row.id,
email: email,
reason: errorMsg
});
}
}
};
return await _execImportRun(impt, handlers);
}
async function getTask() {
@ -154,17 +357,17 @@ async function getTask() {
const impt = await tx('imports').whereIn('status', [ImportStatus.PREP_SCHEDULED, ImportStatus.RUN_SCHEDULED]).orderBy('created', 'asc').first();
if (impt) {
impt.settings = JSON.parse(impt.settings);
impt.settings = JSON.parse(impt.settings) || {};
if (impt.source === ImportSource.CSV_FILE && impt.status === ImportStatus.PREP_SCHEDULED) {
await tx('imports').where('id', impt.id).update('status', ImportStatus.PREP_RUNNING);
return () => prepareCsv(impt);
} else if (impt.status === ImportStatus.RUN_SCHEDULED && impt.settings.mappingType === MappingType.BASIC_SUBSCRIBE) {
} else if (impt.status === ImportStatus.RUN_SCHEDULED && impt.mapping_type === MappingType.BASIC_SUBSCRIBE) {
await tx('imports').where('id', impt.id).update('status', ImportStatus.RUN_RUNNING);
return () => basicSubscribe(impt);
} else if (impt.status === ImportStatus.RUN_SCHEDULED && impt.settings.mappingType === MappingType.BASIC_UNSUBSCRIBE) {
} else if (impt.status === ImportStatus.RUN_SCHEDULED && impt.mapping_type === MappingType.BASIC_UNSUBSCRIBE) {
await tx('imports').where('id', impt.id).update('status', ImportStatus.RUN_RUNNING);
return () => basicUnsubscribe(impt);
}

12
setup/knex/migrations/20170506102634_v1_to_v2.js
View file

@ -3,12 +3,14 @@ const files = require('../../../models/files');
const contextHelpers = require('../../../lib/context-helpers');
const mosaicoTemplates = require('../../../shared/mosaico-templates');
const {getGlobalNamespaceId} = require('../../../shared/namespaces');
const {getAdminId} = require('../../../shared/users');
const entityTypesAddNamespace = ['list', 'custom_form', 'template', 'campaign', 'report', 'report_template', 'user'];
const shareableEntityTypes = ['list', 'custom_form', 'template', 'campaign', 'report', 'report_template', 'namespace', 'send_configuration', 'mosaico_template'];
const { MailerType, getSystemSendConfigurationId } = require('../../../shared/send-configurations');
const { enforce } = require('../../../lib/helpers');
const { EntityVals: TriggerEntityVals, EventVals: TriggerEventVals } = require('../../../shared/triggers');
const { SubscriptionSource } = require('../../../shared/lists');
const crypto = require('crypto');
const entityTypesWithFiles = {
campaign: {
@ -226,7 +228,7 @@ async function migrateUsers(knex) {
});
/* The user role is set automatically in rebuild permissions, which is called upon every start */
await knex('users').where('id', 1 /* Admin user id */).update({
await knex('users').where('id', getAdminId()).update({
name: 'Administrator'
});
}
@ -1030,7 +1032,7 @@ async function migrateImporter(knex) {
table.text('settings', 'longtext');
table.integer('mapping_type').unsigned().notNullable();
table.text('mapping', 'longtext');
table.timestamp('last_run');
table.dateTime('last_run');
table.text('error');
table.timestamp('created').defaultTo(knex.fn.now());
});
@ -1040,17 +1042,19 @@ async function migrateImporter(knex) {
table.integer('import').unsigned().references('imports.id');
table.integer('status').unsigned().notNullable();
table.text('mapping', 'longtext');
table.integer('last_id');
table.integer('new').defaultTo(0);
table.integer('failed').defaultTo(0);
table.integer('processed').defaultTo(0);
table.text('error');
table.timestamp('created').defaultTo(knex.fn.now());
table.timestamp('finished');
table.dateTime('finished');
});
await knex.schema.createTable('import_failed', table => {
table.increments('id').primary();
table.integer('run').unsigned().references('import_runs.id');
table.integer('source_id').unsigned();
table.string('email').notNullable();
table.text('reason');
table.timestamp('created').defaultTo(knex.fn.now());
@ -1063,9 +1067,9 @@ exports.up = (knex, Promise) => (async() => {
await addNamespaces(knex);
await migrateUsers(knex);
await migrateSubscriptions(knex);
await migrateCustomForms(knex);
await migrateCustomFields(knex);
await migrateSubscriptions(knex);
await migrateSegments(knex);
await migrateReports(knex);
await migrateSettings(knex);

3
shared/imports.js
View file

@ -36,7 +36,8 @@ const RunStatus = {
SCHEDULED: 0,
RUNNING: 1,
STOPPING: 2,
FINISHED: 3
FINISHED: 3,
FAILED: 4
};
function prepInProgress(status) {

4
shared/lists.js
View file

@ -31,7 +31,7 @@ const SubscriptionSource = {
IMPORTED_V1: -5
};
function getFieldKey(field) {
function getFieldColumn(field) {
return field.column || 'grouped_' + field.id;
}
@ -39,5 +39,5 @@ module.exports = {
UnsubscriptionMode,
SubscriptionStatus,
SubscriptionSource,
getFieldKey
getFieldColumn
};

9
shared/users.js Normal file
View file

@ -0,0 +1,9 @@
'use strict';
function getAdminId() {
return 1;
}
module.exports = {
getAdminId
};