public/mailtrain
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Basic import seems to work

Browse source
This commit is contained in:
Tomas Bures 2018-09-01 21:29:10 +02:00
parent 16519c5353
commit d74806dde3
21 changed files with 555 additions and 749 deletions
Download patch file Download diff file Expand all files Collapse all files

50
client/src/lists/imports/CUD.js
View file

@ -175,7 +175,7 @@ export default class CUD extends Component {
} else { } else {
const mappingType = Number.parseInt(state.getIn(['mapping_type', 'value'])); const mappingType = Number.parseInt(state.getIn(['mapping_type', 'value']));
if (mappingType === MappingType.BASIC_SUBSCRIBE) { if (mappingType === MappingType.BASIC_SUBSCRIBE || mappingType === MappingType.BASIC_UNSUBSCRIBE) {
if (!state.getIn(['mapping_fields_email_column', 'value'])) { if (!state.getIn(['mapping_fields_email_column', 'value'])) {
state.setIn(['mapping_fields_email_column', 'error'], t('Email mapping has to be provided')); state.setIn(['mapping_fields_email_column', 'error'], t('Email mapping has to be provided'));
} }
@ -225,24 +225,33 @@ export default class CUD extends Component {
for (const field of this.props.fieldsGrouped) { for (const field of this.props.fieldsGrouped) {
if (field.column) { if (field.column) {
mapping.fields[field.column] = { const colMapping = data['mapping_fields_' + field.column + '_column'];
column: data['mapping_fields_' + field.column + '_column'] if (colMapping) {
}; mapping.fields[field.column] = {
column: colMapping
};
}
} else { } else {
for (const option of field.settings.options) { for (const option of field.settings.options) {
const col = field.groupedOptions[option.key].column; const col = field.groupedOptions[option.key].column;
mapping.fields[col] = { const colMapping = data['mapping_fields_' + col + '_column'];
column: data['mapping_fields_' + col + '_column'] if (colMapping) {
}; mapping.fields[col] = {
column: colMapping
};
}
} }
} }
} }
}
if (data.mapping_type === MappingType.BASIC_SUBSCRIBE || data.mapping_type === MappingType.BASIC_UNSUBSCRIBE) {
mapping.fields.email = { mapping.fields.email = {
column: data.mapping_fields_email_column column: data.mapping_fields_email_column
}; };
} }
data.mapping = mapping; data.mapping = mapping;
} }
@ -317,7 +326,7 @@ export default class CUD extends Component {
let mappingSettings = null; let mappingSettings = null;
const mappingType = Number.parseInt(this.getFormValue('mapping_type')); const mappingType = Number.parseInt(this.getFormValue('mapping_type'));
if (mappingType === MappingType.BASIC_SUBSCRIBE) { if (mappingType === MappingType.BASIC_SUBSCRIBE || mappingType === MappingType.BASIC_UNSUBSCRIBE) {
const sampleRow = this.getFormValue('sampleRow'); const sampleRow = this.getFormValue('sampleRow');
const sourceOpts = []; const sourceOpts = [];
sourceOpts.push({key: '', label: t('–– Select ––')}); sourceOpts.push({key: '', label: t('–– Select ––')});
@ -332,28 +341,33 @@ export default class CUD extends Component {
} }
} }
const settingsRows = [];
const mappingRows = [ const mappingRows = [
<Dropdown key="email" id="mapping_fields_email_column" label={t('Email')} options={sourceOpts}/> <Dropdown key="email" id="mapping_fields_email_column" label={t('Email')} options={sourceOpts}/>
]; ];
for (const field of this.props.fieldsGrouped) { if (mappingType === MappingType.BASIC_SUBSCRIBE) {
if (field.column) { settingsRows.push(<CheckBox key="checkEmails" id="mapping_settings_checkEmails" text={t('Check imported emails')}/>)
mappingRows.push(
<Dropdown key={field.column} id={'mapping_fields_' + field.column + '_column'} label={field.name} options={sourceOpts}/> for (const field of this.props.fieldsGrouped) {
); if (field.column) {
} else {
for (const option of field.settings.options) {
const col = field.groupedOptions[option.key].column;
mappingRows.push( mappingRows.push(
<Dropdown key={col} id={'mapping_fields_' + col + '_column'} label={field.groupedOptions[option.key].name} options={sourceOpts}/> <Dropdown key={field.column} id={'mapping_fields_' + field.column + '_column'} label={field.name} options={sourceOpts}/>
); );
} else {
for (const option of field.settings.options) {
const col = field.groupedOptions[option.key].column;
mappingRows.push(
<Dropdown key={col} id={'mapping_fields_' + col + '_column'} label={field.groupedOptions[option.key].name} options={sourceOpts}/>
);
}
} }
} }
} }
mappingSettings = ( mappingSettings = (
<div> <div>
<CheckBox id="mapping_settings_checkEmails" text={t('Check imported emails')}/> {settingsRows}
<Fieldset label={t('Mapping')} className={styles.mapping}> <Fieldset label={t('Mapping')} className={styles.mapping}>
{mappingRows} {mappingRows}
</Fieldset> </Fieldset>

18
client/src/lists/imports/RunStatus.js
View file

@ -18,6 +18,7 @@ import axios from "../../lib/axios";
import {getUrl} from "../../lib/urls"; import {getUrl} from "../../lib/urls";
import moment from "moment"; import moment from "moment";
import {runStatusInProgress} from "../../../../shared/imports"; import {runStatusInProgress} from "../../../../shared/imports";
import {Table} from "../../lib/table";
@translate() @translate()
@withPageHelpers @withPageHelpers
@ -52,6 +53,10 @@ export default class Status extends Component {
this.setState({ this.setState({
entity: resp.data entity: resp.data
}); });
if (this.failedTableNode) {
this.failedTableNode.refresh();
}
} }
async periodicRefreshTask() { async periodicRefreshTask() {
@ -77,7 +82,13 @@ export default class Status extends Component {
const entity = this.state.entity; const entity = this.state.entity;
const imprt = this.props.imprt; const imprt = this.props.imprt;
return ( const columns = [
{ data: 1, title: t('Row') },
{ data: 2, title: t('Email') },
{ data: 3, title: t('Reason') }
];
return (
<div> <div>
<Title>{t('Import Run Status')}</Title> <Title>{t('Import Run Status')}</Title>
@ -90,6 +101,11 @@ export default class Status extends Component {
<AlignedRow label={t('New entries')}>{entity.new}</AlignedRow> <AlignedRow label={t('New entries')}>{entity.new}</AlignedRow>
<AlignedRow label={t('Failed entries')}>{entity.failed}</AlignedRow> <AlignedRow label={t('Failed entries')}>{entity.failed}</AlignedRow>
{entity.error && <AlignedRow label={t('Error')}><pre>{entity.error}</pre></AlignedRow>} {entity.error && <AlignedRow label={t('Error')}><pre>{entity.error}</pre></AlignedRow>}
<hr/>
<h3>{t('Failed Rows')}</h3>
<Table ref={node => this.failedTableNode = node} withHeader dataUrl={`rest/import-run-failed-table/${this.props.list.id}/${this.props.imprt.id}/${this.props.entity.id}`} columns={columns} />
</div> </div>
); );
} }

10
client/src/lists/imports/Status.js
View file

@ -97,7 +97,10 @@ export default class Status extends Component {
} }
await this.refreshEntity(); await this.refreshEntity();
this.runsTableNode.refresh();
if (this.runsTableNode) {
this.runsTableNode.refresh();
}
} }
async stopRunAsync() { async stopRunAsync() {
@ -112,7 +115,10 @@ export default class Status extends Component {
} }
await this.refreshEntity(); await this.refreshEntity();
this.runsTableNode.refresh();
if (this.runsTableNode) {
this.runsTableNode.refresh();
}
} }
render() { render() {

3
client/src/lists/imports/helpers.js
View file

@ -27,7 +27,8 @@ export function getImportTypes(t) {
[RunStatus.SCHEDULED]: t('Starting'), [RunStatus.SCHEDULED]: t('Starting'),
[RunStatus.RUNNING]: t('Running'), [RunStatus.RUNNING]: t('Running'),
[RunStatus.STOPPING]: t('Stopping'), [RunStatus.STOPPING]: t('Stopping'),
[RunStatus.FINISHED]: t('Finished') [RunStatus.FINISHED]: t('Finished'),
[RunStatus.FAILED]: t('Failed')
}; };
const mappingTypeLabels = { const mappingTypeLabels = {

1
client/src/lists/subscriptions/List.js
View file

@ -86,6 +86,7 @@ export default class List extends Component {
const segments = this.props.segments; const segments = this.props.segments;
const columns = [ const columns = [
{ data: 1, title: t('CID') },
{ data: 2, title: t('Email') }, { data: 2, title: t('Email') },
{ data: 3, title: t('Status'), render: (data, display, rowData) => this.subscriptionStatusLabels[data] + (rowData[5] ? ', ' + t('Blacklisted') : '') }, { data: 3, title: t('Status'), render: (data, display, rowData) => this.subscriptionStatusLabels[data] + (rowData[5] ? ', ' + t('Blacklisted') : '') },
{ data: 4, title: t('Created'), render: data => data ? moment(data).fromNow() : '' } { data: 4, title: t('Created'), render: data => data ? moment(data).fromNow() : '' }

94
client/src/lists/subscriptions/helpers.js
View file

@ -4,7 +4,7 @@ import React from "react";
import {SubscriptionStatus} from "../../../../shared/lists"; import {SubscriptionStatus} from "../../../../shared/lists";
import {ACEEditor, CheckBoxGroup, DatePicker, Dropdown, InputField, RadioGroup, TextArea} from "../../lib/form"; import {ACEEditor, CheckBoxGroup, DatePicker, Dropdown, InputField, RadioGroup, TextArea} from "../../lib/form";
import {formatBirthday, formatDate, parseBirthday, parseDate} from "../../../../shared/date"; import {formatBirthday, formatDate, parseBirthday, parseDate} from "../../../../shared/date";
import {getFieldKey} from '../../../../shared/lists'; import {getFieldColumn} from '../../../../shared/lists';
import 'brace/mode/json'; import 'brace/mode/json';
export function getSubscriptionStatusLabels(t) { export function getSubscriptionStatusLabels(t) {
@ -24,13 +24,13 @@ export function getFieldTypes(t) {
const groupedFieldTypes = {}; const groupedFieldTypes = {};
const stringFieldType = long => ({ const stringFieldType = long => ({
form: groupedField => long ? <TextArea key={getFieldKey(groupedField)} id={getFieldKey(groupedField)} label={groupedField.name}/> : <InputField key={getFieldKey(groupedField)} id={getFieldKey(groupedField)} label={groupedField.name}/>, form: groupedField => long ? <TextArea key={getFieldColumn(groupedField)} id={getFieldColumn(groupedField)} label={groupedField.name}/> : <InputField key={getFieldColumn(groupedField)} id={getFieldColumn(groupedField)} label={groupedField.name}/>,
assignFormData: (groupedField, data) => { assignFormData: (groupedField, data) => {
const value = data[getFieldKey(groupedField)]; const value = data[getFieldColumn(groupedField)];
data[getFieldKey(groupedField)] = value || ''; data[getFieldColumn(groupedField)] = value || '';
}, },
initFormData: (groupedField, data) => { initFormData: (groupedField, data) => {
data[getFieldKey(groupedField)] = ''; data[getFieldColumn(groupedField)] = '';
}, },
assignEntity: (groupedField, data) => {}, assignEntity: (groupedField, data) => {},
validate: (groupedField, state) => {}, validate: (groupedField, state) => {},
@ -38,86 +38,86 @@ export function getFieldTypes(t) {
}); });
const numberFieldType = { const numberFieldType = {
form: groupedField => <InputField key={getFieldKey(groupedField)} id={getFieldKey(groupedField)} label={groupedField.name}/>, form: groupedField => <InputField key={getFieldColumn(groupedField)} id={getFieldColumn(groupedField)} label={groupedField.name}/>,
assignFormData: (groupedField, data) => { assignFormData: (groupedField, data) => {
const value = data[getFieldKey(groupedField)]; const value = data[getFieldColumn(groupedField)];
data[getFieldKey(groupedField)] = value ? value.toString() : ''; data[getFieldColumn(groupedField)] = value ? value.toString() : '';
}, },
initFormData: (groupedField, data) => { initFormData: (groupedField, data) => {
data[getFieldKey(groupedField)] = ''; data[getFieldColumn(groupedField)] = '';
}, },
assignEntity: (groupedField, data) => { assignEntity: (groupedField, data) => {
data[getFieldKey(groupedField)] = parseInt(data[getFieldKey(groupedField)]); data[getFieldColumn(groupedField)] = parseInt(data[getFieldColumn(groupedField)]);
}, },
validate: (groupedField, state) => { validate: (groupedField, state) => {
const value = state.getIn([getFieldKey(groupedField), 'value']).trim(); const value = state.getIn([getFieldColumn(groupedField), 'value']).trim();
if (value !== '' && isNaN(value)) { if (value !== '' && isNaN(value)) {
state.setIn([getFieldKey(groupedField), 'error'], t('Value must be a number')); state.setIn([getFieldColumn(groupedField), 'error'], t('Value must be a number'));
} else { } else {
state.setIn([getFieldKey(groupedField), 'error'], null); state.setIn([getFieldColumn(groupedField), 'error'], null);
} }
}, },
indexable: true indexable: true
}; };
const dateFieldType = { const dateFieldType = {
form: groupedField => <DatePicker key={getFieldKey(groupedField)} id={getFieldKey(groupedField)} label={groupedField.name} dateFormat={groupedField.settings.dateFormat} />, form: groupedField => <DatePicker key={getFieldColumn(groupedField)} id={getFieldColumn(groupedField)} label={groupedField.name} dateFormat={groupedField.settings.dateFormat} />,
assignFormData: (groupedField, data) => { assignFormData: (groupedField, data) => {
const value = data[getFieldKey(groupedField)]; const value = data[getFieldColumn(groupedField)];
data[getFieldKey(groupedField)] = value ? formatDate(groupedField.settings.dateFormat, value) : ''; data[getFieldColumn(groupedField)] = value ? formatDate(groupedField.settings.dateFormat, value) : '';
}, },
initFormData: (groupedField, data) => { initFormData: (groupedField, data) => {
data[getFieldKey(groupedField)] = ''; data[getFieldColumn(groupedField)] = '';
}, },
assignEntity: (groupedField, data) => { assignEntity: (groupedField, data) => {
const date = parseDate(groupedField.settings.dateFormat, data[getFieldKey(groupedField)]); const date = parseDate(groupedField.settings.dateFormat, data[getFieldColumn(groupedField)]);
data[getFieldKey(groupedField)] = date; data[getFieldColumn(groupedField)] = date;
}, },
validate: (groupedField, state) => { validate: (groupedField, state) => {
const value = state.getIn([getFieldKey(groupedField), 'value']); const value = state.getIn([getFieldColumn(groupedField), 'value']);
const date = parseDate(groupedField.settings.dateFormat, value); const date = parseDate(groupedField.settings.dateFormat, value);
if (value !== '' && !date) { if (value !== '' && !date) {
state.setIn([getFieldKey(groupedField), 'error'], t('Date is invalid')); state.setIn([getFieldColumn(groupedField), 'error'], t('Date is invalid'));
} else { } else {
state.setIn([getFieldKey(groupedField), 'error'], null); state.setIn([getFieldColumn(groupedField), 'error'], null);
} }
}, },
indexable: true indexable: true
}; };
const birthdayFieldType = { const birthdayFieldType = {
form: groupedField => <DatePicker key={getFieldKey(groupedField)} id={getFieldKey(groupedField)} label={groupedField.name} dateFormat={groupedField.settings.dateFormat} birthday />, form: groupedField => <DatePicker key={getFieldColumn(groupedField)} id={getFieldColumn(groupedField)} label={groupedField.name} dateFormat={groupedField.settings.dateFormat} birthday />,
assignFormData: (groupedField, data) => { assignFormData: (groupedField, data) => {
const value = data[getFieldKey(groupedField)]; const value = data[getFieldColumn(groupedField)];
data[getFieldKey(groupedField)] = value ? formatBirthday(groupedField.settings.dateFormat, value) : ''; data[getFieldColumn(groupedField)] = value ? formatBirthday(groupedField.settings.dateFormat, value) : '';
}, },
initFormData: (groupedField, data) => { initFormData: (groupedField, data) => {
data[getFieldKey(groupedField)] = ''; data[getFieldColumn(groupedField)] = '';
}, },
assignEntity: (groupedField, data) => { assignEntity: (groupedField, data) => {
const date = parseBirthday(groupedField.settings.dateFormat, data[getFieldKey(groupedField)]); const date = parseBirthday(groupedField.settings.dateFormat, data[getFieldColumn(groupedField)]);
data[getFieldKey(groupedField)] = date; data[getFieldColumn(groupedField)] = date;
}, },
validate: (groupedField, state) => { validate: (groupedField, state) => {
const value = state.getIn([getFieldKey(groupedField), 'value']); const value = state.getIn([getFieldColumn(groupedField), 'value']);
const date = parseBirthday(groupedField.settings.dateFormat, value); const date = parseBirthday(groupedField.settings.dateFormat, value);
if (value !== '' && !date) { if (value !== '' && !date) {
state.setIn([getFieldKey(groupedField), 'error'], t('Date is invalid')); state.setIn([getFieldColumn(groupedField), 'error'], t('Date is invalid'));
} else { } else {
state.setIn([getFieldKey(groupedField), 'error'], null); state.setIn([getFieldColumn(groupedField), 'error'], null);
} }
}, },
indexable: true indexable: true
}; };
const jsonFieldType = { const jsonFieldType = {
form: groupedField => <ACEEditor key={getFieldKey(groupedField)} id={getFieldKey(groupedField)} label={groupedField.name} mode="json" height="300px"/>, form: groupedField => <ACEEditor key={getFieldColumn(groupedField)} id={getFieldColumn(groupedField)} label={groupedField.name} mode="json" height="300px"/>,
assignFormData: (groupedField, data) => { assignFormData: (groupedField, data) => {
const value = data[getFieldKey(groupedField)]; const value = data[getFieldColumn(groupedField)];
data[getFieldKey(groupedField)] = value || ''; data[getFieldColumn(groupedField)] = value || '';
}, },
initFormData: (groupedField, data) => { initFormData: (groupedField, data) => {
data[getFieldKey(groupedField)] = ''; data[getFieldColumn(groupedField)] = '';
}, },
assignEntity: (groupedField, data) => {}, assignEntity: (groupedField, data) => {},
validate: (groupedField, state) => {}, validate: (groupedField, state) => {},
@ -125,25 +125,25 @@ export function getFieldTypes(t) {
}; };
const enumSingleFieldType = componentType => ({ const enumSingleFieldType = componentType => ({
form: groupedField => React.createElement(componentType, { key: getFieldKey(groupedField), id: getFieldKey(groupedField), label: groupedField.name, options: groupedField.settings.options }, null), form: groupedField => React.createElement(componentType, { key: getFieldColumn(groupedField), id: getFieldColumn(groupedField), label: groupedField.name, options: groupedField.settings.options }, null),
assignFormData: (groupedField, data) => { assignFormData: (groupedField, data) => {
if (data[getFieldKey(groupedField)] === null) { if (data[getFieldColumn(groupedField)] === null) {
if (groupedField.default_value) { if (groupedField.default_value) {
data[getFieldKey(groupedField)] = groupedField.default_value; data[getFieldColumn(groupedField)] = groupedField.default_value;
} else if (groupedField.settings.options.length > 0) { } else if (groupedField.settings.options.length > 0) {
data[getFieldKey(groupedField)] = groupedField.settings.options[0].key; data[getFieldColumn(groupedField)] = groupedField.settings.options[0].key;
} else { } else {
data[getFieldKey(groupedField)] = ''; data[getFieldColumn(groupedField)] = '';
} }
} }
}, },
initFormData: (groupedField, data) => { initFormData: (groupedField, data) => {
if (groupedField.default_value) { if (groupedField.default_value) {
data[getFieldKey(groupedField)] = groupedField.default_value; data[getFieldColumn(groupedField)] = groupedField.default_value;
} else if (groupedField.settings.options.length > 0) { } else if (groupedField.settings.options.length > 0) {
data[getFieldKey(groupedField)] = groupedField.settings.options[0].key; data[getFieldColumn(groupedField)] = groupedField.settings.options[0].key;
} else { } else {
data[getFieldKey(groupedField)] = ''; data[getFieldColumn(groupedField)] = '';
} }
}, },
assignEntity: (groupedField, data) => { assignEntity: (groupedField, data) => {
@ -153,14 +153,14 @@ export function getFieldTypes(t) {
}); });
const enumMultipleFieldType = componentType => ({ const enumMultipleFieldType = componentType => ({
form: groupedField => React.createElement(componentType, { key: getFieldKey(groupedField), id: getFieldKey(groupedField), label: groupedField.name, options: groupedField.settings.options }, null), form: groupedField => React.createElement(componentType, { key: getFieldColumn(groupedField), id: getFieldColumn(groupedField), label: groupedField.name, options: groupedField.settings.options }, null),
assignFormData: (groupedField, data) => { assignFormData: (groupedField, data) => {
if (data[getFieldKey(groupedField)] === null) { if (data[getFieldColumn(groupedField)] === null) {
data[getFieldKey(groupedField)] = []; data[getFieldColumn(groupedField)] = [];
} }
}, },
initFormData: (groupedField, data) => { initFormData: (groupedField, data) => {
data[getFieldKey(groupedField)] = []; data[getFieldColumn(groupedField)] = [];
}, },
assignEntity: (groupedField, data) => {}, assignEntity: (groupedField, data) => {},
validate: (groupedField, state) => {}, validate: (groupedField, state) => {},

4
lib/importer.js
View file

@ -22,10 +22,10 @@ function spawn(callback) {
await tx('imports').where('status', ImportStatus.PREP_STOPPING).update({status: ImportStatus.PREP_FAILED}); await tx('imports').where('status', ImportStatus.PREP_STOPPING).update({status: ImportStatus.PREP_FAILED});
await tx('imports').where('status', ImportStatus.RUN_RUNNING).update({status: ImportStatus.RUN_SCHEDULED}); await tx('imports').where('status', ImportStatus.RUN_RUNNING).update({status: ImportStatus.RUN_SCHEDULED});
await tx('imports').where('status', ImportStatus.RUN_STOPPING).update({status: ImportStatus.RUN_FINISHED}); await tx('imports').where('status', ImportStatus.RUN_STOPPING).update({status: ImportStatus.RUN_FAILED});
await tx('import_runs').where('status', RunStatus.RUNNING).update({status: RunStatus.SCHEDULED}); await tx('import_runs').where('status', RunStatus.RUNNING).update({status: RunStatus.SCHEDULED});
await tx('import_runs').where('status', RunStatus.STOPPING).update({status: RunStatus.FINISHED}); await tx('import_runs').where('status', RunStatus.STOPPING).update({status: RunStatus.FAILED});
}).then(() => { }).then(() => {
importerProcess = fork(path.join(__dirname, '..', 'services', 'importer.js'), [], { importerProcess = fork(path.join(__dirname, '..', 'services', 'importer.js'), [], {

4
lib/subscription-mail-helpers.js
View file

@ -7,7 +7,7 @@ const {getTrustedUrl} = require('./urls');
const _ = require('./translate')._; const _ = require('./translate')._;
const util = require('util'); const util = require('util');
const contextHelpers = require('./context-helpers'); const contextHelpers = require('./context-helpers');
const {getFieldKey} = require('../shared/lists'); const {getFieldColumn} = require('../shared/lists');
const forms = require('../models/forms'); const forms = require('../models/forms');
const mailers = require('./mailers'); const mailers = require('./mailers');
@ -101,7 +101,7 @@ async function _sendMail(list, email, template, subject, relativeUrls, subscript
const encryptionKeys = []; const encryptionKeys = [];
for (const fld of flds) { for (const fld of flds) {
if (fld.type === 'gpg' && fld.value) { if (fld.type === 'gpg' && fld.value) {
encryptionKeys.push(subscription[getFieldKey(fld)].value.trim()); encryptionKeys.push(subscription[getFieldColumn(fld)].value.trim());
} }
} }

10
lib/tools.js
View file

@ -16,7 +16,6 @@ const fsReadFile = bluebird.promisify(require('fs').readFile);
const jsdomEnv = bluebird.promisify(require('jsdom').env); const jsdomEnv = bluebird.promisify(require('jsdom').env);
const templates = new Map(); const templates = new Map();
async function getTemplate(template) { async function getTemplate(template) {
@ -74,14 +73,7 @@ async function mergeTemplateIntoLayout(template, layout) {
return source; return source;
} }
async function validateEmail(address) {
async function validateEmail(address, checkBlocked) {
let user = (address || '').toString().split('@').shift().toLowerCase().replace(/[^a-z0-9]/g, '');
if (checkBlocked && blockedUsers.indexOf(user) >= 0) {
throw new new Error(util.format(_('Blocked email address "%s"'), address));
}
const result = await new Promise(resolve => { const result = await new Promise(resolve => {
const result = isemail.validate(address, { const result = isemail.validate(address, {
checkDNS: true, checkDNS: true,

128
models/fields.js
View file

@ -11,7 +11,7 @@ const validators = require('../shared/validators');
const shortid = require('shortid'); const shortid = require('shortid');
const segments = require('./segments'); const segments = require('./segments');
const { formatDate, formatBirthday, parseDate, parseBirthday } = require('../shared/date'); const { formatDate, formatBirthday, parseDate, parseBirthday } = require('../shared/date');
const { getFieldKey } = require('../shared/lists'); const { getFieldColumn } = require('../shared/lists');
const { cleanupFromPost } = require('../lib/helpers'); const { cleanupFromPost } = require('../lib/helpers');
@ -157,7 +157,8 @@ fieldTypes.option = {
indexed: true, indexed: true,
grouped: false, grouped: false,
enumerated: false, enumerated: false,
cardinality: Cardinality.SINGLE cardinality: Cardinality.SINGLE,
parsePostValue: (field, value) => !(['false', 'no', '0', ''].indexOf((value || '').toString().trim().toLowerCase()) >= 0)
}; };
fieldTypes['date'] = { fieldTypes['date'] = {
@ -572,7 +573,7 @@ async function forHbs(context, listId, subscription) { // assumes grouped subscr
for (const fld of flds) { for (const fld of flds) {
const type = fieldTypes[fld.type]; const type = fieldTypes[fld.type];
const fldKey = getFieldKey(fld); const fldCol = getFieldColumn(fld);
const entry = { const entry = {
name: fld.name, name: fld.name,
@ -583,12 +584,12 @@ async function forHbs(context, listId, subscription) { // assumes grouped subscr
}; };
if (!type.grouped && !type.enumerated) { if (!type.grouped && !type.enumerated) {
// subscription[fldKey] may not exists because we are getting the data from "fromPost" // subscription[fldCol] may not exists because we are getting the data from "fromPost"
entry.value = (subscription ? type.forHbs(fld, subscription[fldKey]) : null) || ''; entry.value = (subscription ? type.forHbs(fld, subscription[fldCol]) : null) || '';
} else if (type.grouped) { } else if (type.grouped) {
const options = []; const options = [];
const value = (subscription ? subscription[fldKey] : null) || (type.cardinality === Cardinality.SINGLE ? null : []); const value = (subscription ? subscription[fldCol] : null) || (type.cardinality === Cardinality.SINGLE ? null : []);
for (const optCol in fld.groupedOptions) { for (const optCol in fld.groupedOptions) {
const opt = fld.groupedOptions[optCol]; const opt = fld.groupedOptions[optCol];
@ -611,7 +612,7 @@ async function forHbs(context, listId, subscription) { // assumes grouped subscr
} else if (type.enumerated) { } else if (type.enumerated) {
const options = []; const options = [];
const value = (subscription ? subscription[fldKey] : null) || null; const value = (subscription ? subscription[fldCol] : null) || null;
for (const opt of fld.settings.options) { for (const opt of fld.settings.options) {
options.push({ options.push({
@ -631,64 +632,97 @@ async function forHbs(context, listId, subscription) { // assumes grouped subscr
return customFields; return customFields;
} }
// Converts subscription data received via POST request from subscription form or via subscribe request to API v1 to subscription structure supported by subscriptions model. // Converts subscription data received via (1) POST request from subscription form, (2) via subscribe request to API v1 to subscription structure supported by subscriptions model,
// If a field is not specified in the POST data, it omits it also in the returned subscription // or (3) from import.
async function fromPost(context, listId, data, partial) { // assumes grouped subscription // If a field is not specified in the POST data, it is also omitted in the returned subscription
function _fromText(listId, data, flds, isGrouped, keyName, singleCardUsesKeyName) {
// This is to handle option values from API v1
function isSelected(value) {
return ['false', 'no', '0', ''].indexOf((value || '').toString().trim().toLowerCase()) >= 0 ? false : true;
}
const flds = await listGrouped(context, listId);
const subscription = {}; const subscription = {};
for (const fld of flds) { if (isGrouped) {
if (fld.key in data) { for (const fld of flds) {
const type = fieldTypes[fld.type]; const fldKey = fld[keyName];
const fldKey = getFieldKey(fld); if (fldKey && fldKey in data) {
const type = fieldTypes[fld.type];
const fldCol = getFieldColumn(fld);
let value = null; let value = null;
if (!type.grouped && !type.enumerated) { if (!type.grouped && !type.enumerated) {
value = type.parsePostValue(fld, cleanupFromPost(data[fld.key])); value = type.parsePostValue(fld, cleanupFromPost(data[fldKey]));
} else if (type.grouped) { } else if (type.grouped) {
if (type.cardinality === Cardinality.SINGLE) { if (type.cardinality === Cardinality.SINGLE) {
for (const optCol in fld.groupedOptions) { for (const optCol in fld.groupedOptions) {
const opt = fld.groupedOptions[optCol]; const opt = fld.groupedOptions[optCol];
const optKey = opt[keyName];
// This handles two different formats for grouped dropdowns and radios. // This handles two different formats for grouped dropdowns and radios.
// The first part of the condition handles the POST requests from the subscription form, while the // The first part of the condition handles the POST requests from the subscription form, while the
// second part handles the subscribe request to API v1 // second part handles the subscribe request to API v1
if (data[fld.key] === opt.key || isSelected(data[opt.key])) { if (singleCardUsesKeyName) {
value = opt.column if (data[fldKey] === optKey) {
value = opt.column
}
} else {
const optType = fieldTypes[opt.type];
const optValue = optType.parsePostValue(fld, cleanupFromPost(data[optKey]));
if (optValue) {
value = opt.column
}
}
}
} else {
value = [];
for (const optCol in fld.groupedOptions) {
const opt = fld.groupedOptions[optCol];
const optKey = opt[keyName];
const optType = fieldTypes[opt.type];
const optValue = optType.parsePostValue(fld, cleanupFromPost(data[optKey]));
if (optValue) {
value.push(opt.column);
}
} }
} }
} else {
value = [];
for (const optCol in fld.groupedOptions) { } else if (type.enumerated) {
const opt = fld.groupedOptions[optCol]; value = data[fldKey];
if (isSelected(data[opt.key])) {
value.push(opt.column);
}
}
} }
} else if (type.enumerated) { subscription[fldCol] = value;
value = data[fld.key];
} }
}
subscription[fldKey] = value; } else {
for (const fld of flds) {
const fldKey = fld[keyName];
if (fldKey && fldKey in data) {
const type = fieldTypes[fld.type];
const fldCol = getFieldColumn(fld);
subscription[fldCol] = type.parsePostValue(fld, cleanupFromPost(data[fldKey]));
}
} }
} }
return subscription; return subscription;
} }
async function fromPost(context, listId, data) { // assumes grouped subscription and indexation by merge key
const flds = await listGrouped(context, listId);
return _fromText(listId, data, flds, true, 'key', true);
}
async function fromAPI(context, listId, data) { // assumes grouped subscription and indexation by merge key
const flds = await listGrouped(context, listId);
return _fromText(listId, data, flds, true, 'key', false);
}
function fromImport(listId, flds, data) { // assumes ungrouped subscription and indexation by column
return _fromText(listId, data, flds, true, 'column', false);
}
// This is to handle circular dependency with segments.js // This is to handle circular dependency with segments.js
Object.assign(module.exports, { Object.assign(module.exports, {
@ -709,5 +743,7 @@ Object.assign(module.exports, {
removeAllByListIdTx, removeAllByListIdTx,
serverValidate, serverValidate,
forHbs, forHbs,
fromPost fromPost,
fromAPI,
fromImport
}); });

24
models/import-runs.js
View file

@ -35,15 +35,35 @@ async function listDTAjax(context, listId, importId, params) {
builder => builder builder => builder
.from('import_runs') .from('import_runs')
.innerJoin('imports', 'import_runs.import', 'imports.id') .innerJoin('imports', 'import_runs.import', 'imports.id')
.where({'imports.list': listId, 'imports.id': importId}), .where({'imports.list': listId, 'imports.id': importId})
.orderBy('import_runs.id', 'desc'),
[ 'import_runs.id', 'import_runs.created', 'import_runs.finished', 'import_runs.status', 'import_runs.processed', 'import_runs.new', 'import_runs.failed'] [ 'import_runs.id', 'import_runs.created', 'import_runs.finished', 'import_runs.status', 'import_runs.processed', 'import_runs.new', 'import_runs.failed']
); );
}); });
} }
async function listFailedDTAjax(context, listId, importId, importRunId, params) {
return await knex.transaction(async tx => {
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewImports');
return await dtHelpers.ajaxListTx(
tx,
params,
builder => builder
.from('import_failed')
.innerJoin('import_runs', 'import_failed.run', 'import_runs.id')
.innerJoin('imports', 'import_runs.import', 'imports.id')
.where({'imports.list': listId, 'imports.id': importId, 'import_runs.id': importRunId})
.orderBy('import_failed.source_id', 'asc'),
[ 'import_failed.id', 'import_failed.source_id', 'import_failed.email', 'import_failed.reason']
);
});
}
module.exports = { module.exports = {
getById, getById,
listDTAjax listDTAjax,
listFailedDTAjax
}; };

5
models/shares.js
View file

@ -8,6 +8,7 @@ const entitySettings = require('../lib/entity-settings');
const interoperableErrors = require('../shared/interoperable-errors'); const interoperableErrors = require('../shared/interoperable-errors');
const log = require('npmlog'); const log = require('npmlog');
const {getGlobalNamespaceId} = require('../shared/namespaces'); const {getGlobalNamespaceId} = require('../shared/namespaces');
const {getAdminId} = require('../shared/users');
// TODO: This would really benefit from some permission cache connected to rebuildPermissions // TODO: This would really benefit from some permission cache connected to rebuildPermissions
// A bit of the problem is that the cache would have to expunged as the result of other processes modifying entites/permissions // A bit of the problem is that the cache would have to expunged as the result of other processes modifying entites/permissions
@ -146,7 +147,7 @@ async function rebuildPermissionsTx(tx, restriction) {
// To prevent users locking out themselves, we consider user with id 1 to be the admin and always assign it // To prevent users locking out themselves, we consider user with id 1 to be the admin and always assign it
// the admin role. The admin role is a global role that has admin===true // the admin role. The admin role is a global role that has admin===true
// If this behavior is not desired, it is enough to delete the user with id 1. // If this behavior is not desired, it is enough to delete the user with id 1.
const adminUser = await tx('users').where('id', 1 /* Admin user id */).first(); const adminUser = await tx('users').where('id', getAdminId()).first();
if (adminUser) { if (adminUser) {
let adminRole; let adminRole;
for (const role in config.roles.global) { for (const role in config.roles.global) {
@ -157,7 +158,7 @@ async function rebuildPermissionsTx(tx, restriction) {
} }
if (adminRole) { if (adminRole) {
await tx('users').update('role', adminRole).where('id', 1 /* Admin user id */); await tx('users').update('role', adminRole).where('id', getAdminId());
} }
} }

157
models/subscriptions.js
View file

@ -7,11 +7,12 @@ const dtHelpers = require('../lib/dt-helpers');
const interoperableErrors = require('../shared/interoperable-errors'); const interoperableErrors = require('../shared/interoperable-errors');
const shares = require('./shares'); const shares = require('./shares');
const fields = require('./fields'); const fields = require('./fields');
const { SubscriptionStatus, getFieldKey } = require('../shared/lists'); const { SubscriptionStatus, getFieldColumn } = require('../shared/lists');
const segments = require('./segments'); const segments = require('./segments');
const { enforce, filterObject } = require('../lib/helpers'); const { enforce, filterObject } = require('../lib/helpers');
const moment = require('moment'); const moment = require('moment');
const { formatDate, formatBirthday } = require('../shared/date'); const { formatDate, formatBirthday } = require('../shared/date');
const crypto = require('crypto');
const allowedKeysBase = new Set(['email', 'tz', 'is_test', 'status']); const allowedKeysBase = new Set(['email', 'tz', 'is_test', 'status']);
@ -54,7 +55,7 @@ fieldTypes['radio-enum'] = fieldTypes['dropdown-enum'] = fieldTypes['radio-group
fieldTypes.date = { fieldTypes.date = {
afterJSON: (groupedField, entity) => { afterJSON: (groupedField, entity) => {
const key = getFieldKey(groupedField); const key = getFieldColumn(groupedField);
if (key in entity) { if (key in entity) {
entity[key] = entity[key] ? moment(entity[key]).toDate() : null; entity[key] = entity[key] ? moment(entity[key]).toDate() : null;
} }
@ -64,7 +65,7 @@ fieldTypes.date = {
fieldTypes.birthday = { fieldTypes.birthday = {
afterJSON: (groupedField, entity) => { afterJSON: (groupedField, entity) => {
const key = getFieldKey(groupedField); const key = getFieldColumn(groupedField);
if (key in entity) { if (key in entity) {
entity[key] = entity[key] ? moment(entity[key]).toDate() : null; entity[key] = entity[key] ? moment(entity[key]).toDate() : null;
} }
@ -86,14 +87,14 @@ async function getGroupedFieldsMap(tx, listId) {
const groupedFields = await fields.listGroupedTx(tx, listId); const groupedFields = await fields.listGroupedTx(tx, listId);
const result = {}; const result = {};
for (const fld of groupedFields) { for (const fld of groupedFields) {
result[getFieldKey(fld)] = fld; result[getFieldColumn(fld)] = fld;
} }
return result; return result;
} }
function groupSubscription(groupedFieldsMap, entity) { function groupSubscription(groupedFieldsMap, entity) {
for (const fldKey in groupedFieldsMap) { for (const fldCol in groupedFieldsMap) {
const fld = groupedFieldsMap[fldKey]; const fld = groupedFieldsMap[fldCol];
const fieldType = fields.getFieldType(fld.type); const fieldType = fields.getFieldType(fld.type);
if (fieldType.grouped) { if (fieldType.grouped) {
@ -124,7 +125,7 @@ function groupSubscription(groupedFieldsMap, entity) {
} }
} }
entity[fldKey] = value; entity[fldCol] = value;
} else if (fieldType.enumerated) { } else if (fieldType.enumerated) {
// This is enum-xxx type. We just make sure that the options we give out match the field settings. // This is enum-xxx type. We just make sure that the options we give out match the field settings.
@ -132,36 +133,36 @@ function groupSubscription(groupedFieldsMap, entity) {
const allowedKeys = new Set(fld.settings.options.map(x => x.key)); const allowedKeys = new Set(fld.settings.options.map(x => x.key));
if (!allowedKeys.has(entity[fldKey])) { if (!allowedKeys.has(entity[fldCol])) {
entity[fldKey] = null; entity[fldCol] = null;
} }
} }
} }
} }
function ungroupSubscription(groupedFieldsMap, entity) { function ungroupSubscription(groupedFieldsMap, entity) {
for (const fldKey in groupedFieldsMap) { for (const fldCol in groupedFieldsMap) {
const fld = groupedFieldsMap[fldKey]; const fld = groupedFieldsMap[fldCol];
const fieldType = fields.getFieldType(fld.type); const fieldType = fields.getFieldType(fld.type);
if (fieldType.grouped) { if (fieldType.grouped) {
if (fieldType.cardinality === fields.Cardinality.SINGLE) { if (fieldType.cardinality === fields.Cardinality.SINGLE) {
const value = entity[fldKey]; const value = entity[fldCol];
for (const optionKey in fld.groupedOptions) { for (const optionKey in fld.groupedOptions) {
const option = fld.groupedOptions[optionKey]; const option = fld.groupedOptions[optionKey];
entity[option.column] = option.column === value; entity[option.column] = option.column === value;
} }
} else { } else {
const values = entity[fldKey] || []; // The default (empty array) is here because create may be called with an entity that has some fields not filled in const values = entity[fldCol] || []; // The default (empty array) is here because create may be called with an entity that has some fields not filled in
for (const optionKey in fld.groupedOptions) { for (const optionKey in fld.groupedOptions) {
const option = fld.groupedOptions[optionKey]; const option = fld.groupedOptions[optionKey];
entity[option.column] = values.includes(option.column); entity[option.column] = values.includes(option.column);
} }
} }
delete entity[fldKey]; delete entity[fldCol];
} else if (fieldType.enumerated) { } else if (fieldType.enumerated) {
// This is enum-xxx type. We just make sure that the options we give out match the field settings. // This is enum-xxx type. We just make sure that the options we give out match the field settings.
@ -169,8 +170,8 @@ function ungroupSubscription(groupedFieldsMap, entity) {
const allowedKeys = new Set(fld.settings.options.map(x => x.key)); const allowedKeys = new Set(fld.settings.options.map(x => x.key));
if (!allowedKeys.has(entity[fldKey])) { if (!allowedKeys.has(entity[fldCol])) {
entity[fldKey] = null; entity[fldCol] = null;
} }
} }
} }
@ -259,29 +260,29 @@ async function listDTAjax(context, listId, segmentId, params) {
const idxMap = {}; const idxMap = {};
for (const listFld of listFlds) { for (const listFld of listFlds) {
const fldKey = getFieldKey(listFld); const fldCol = getFieldColumn(listFld);
const fld = groupedFieldsMap[fldKey]; const fld = groupedFieldsMap[fldCol];
if (fld.column) { if (fld.column) {
columns.push(listTable + '.' + fld.column); columns.push(listTable + '.' + fld.column);
} else { } else {
columns.push({ columns.push({
name: listTable + '.' + fldKey, name: listTable + '.' + fldCol,
raw: 0 raw: 0
}) })
} }
idxMap[fldKey] = listFldIdx; idxMap[fldCol] = listFldIdx;
listFldIdx += 1; listFldIdx += 1;
} }
for (const fldKey in groupedFieldsMap) { for (const fldCol in groupedFieldsMap) {
const fld = groupedFieldsMap[fldKey]; const fld = groupedFieldsMap[fldCol];
if (fld.column) { if (fld.column) {
if (!(fldKey in idxMap)) { if (!(fldCol in idxMap)) {
extraColumns.push(listTable + '.' + fld.column); extraColumns.push(listTable + '.' + fld.column);
idxMap[fldKey] = listFldIdx; idxMap[fldCol] = listFldIdx;
listFldIdx += 1; listFldIdx += 1;
} }
@ -313,19 +314,19 @@ async function listDTAjax(context, listId, segmentId, params) {
{ {
mapFun: data => { mapFun: data => {
const entity = {}; const entity = {};
for (const fldKey in idxMap) { for (const fldCol in idxMap) {
// This is a bit of hacking. We rely on the fact that if a field has a column, then the column is the field key. // This is a bit of hacking. We rely on the fact that if a field has a column, then the column is the field key.
// Then it has the group id with value 0. groupSubscription will be able to process the fields that have a column // Then it has the group id with value 0. groupSubscription will be able to process the fields that have a column
// and it will assign values to the fields that don't have a value (i.e. those that currently have the group id and value 0). // and it will assign values to the fields that don't have a value (i.e. those that currently have the group id and value 0).
entity[fldKey] = data[idxMap[fldKey]]; entity[fldCol] = data[idxMap[fldCol]];
} }
groupSubscription(groupedFieldsMap, entity); groupSubscription(groupedFieldsMap, entity);
for (const listFld of listFlds) { for (const listFld of listFlds) {
const fldKey = getFieldKey(listFld); const fldCol = getFieldColumn(listFld);
const fld = groupedFieldsMap[fldKey]; const fld = groupedFieldsMap[fldCol];
data[idxMap[fldKey]] = fieldTypes[fld.type].listRender(fld, entity[fldKey]); data[idxMap[fldCol]] = fieldTypes[fld.type].listRender(fld, entity[fldCol]);
} }
}, },
@ -393,7 +394,7 @@ async function serverValidate(context, listId, data) {
async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta, isCreate) { async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta, isCreate) {
enforce(entity.email, 'Email must be set'); enforce(entity.email, 'Email must be set');
const existingWithKeyQuery = tx(getSubscriptionTableName(listId)).where('email', entity.email); const existingWithKeyQuery = tx(getSubscriptionTableName(listId)).where('hash_email', hashEmail(entity.email));
if (!isCreate) { if (!isCreate) {
existingWithKeyQuery.whereNot('id', entity.id); existingWithKeyQuery.whereNot('id', entity.id);
@ -407,9 +408,11 @@ async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta
throw new interoperableErrors.DuplicitEmailError(); throw new interoperableErrors.DuplicitEmailError();
} }
} else { } else {
// This is here because of the API, which allows one to send subscriptions without caring about whether they already exist, what their status is, etc. // This is here because of the API endpoint, which allows one to submit subscriptions without caring about whether they already exist, what their status is, etc.
// In the case, the subscription is existing, we should not change the status. If it does not exist, we are fine with changing the status // The same for import where we need to subscribed only those (existing and new) that have not been unsubscribed already.
if (meta.subscribeIfNoExisting && !entity.status) { // In the case, the subscription is existing, we should not change the status. If it does not exist, we are fine with changing the status to SUBSCRIBED
if (meta && meta.subscribeIfNoExisting && !entity.status) {
entity.status = SubscriptionStatus.SUBSCRIBED; entity.status = SubscriptionStatus.SUBSCRIBED;
} }
} }
@ -425,14 +428,18 @@ async function _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta
} }
} }
function updateSourcesAndHash(subscription, source, groupedFieldsMap) { function hashEmail(email) {
return crypto.createHash('sha512').update(email).digest("base64");
}
function updateSourcesAndHashEmail(subscription, source, groupedFieldsMap) {
if ('email' in subscription) { if ('email' in subscription) {
subscription.hash_email = crypto.createHash('sha512').update(subscription.email).digest("base64"); subscription.hash_email = hashEmail(subscription.email);
subscription.source_email = source; subscription.source_email = source;
} }
for (const fldKey in groupedFieldsMap) { for (const fldCol in groupedFieldsMap) {
const fld = groupedFieldsMap[fldKey]; const fld = groupedFieldsMap[fldCol];
const fieldType = fields.getFieldType(fld.type); const fieldType = fields.getFieldType(fld.type);
if (fieldType.grouped) { if (fieldType.grouped) {
@ -444,8 +451,8 @@ function updateSourcesAndHash(subscription, source, groupedFieldsMap) {
} }
} }
} else { } else {
if (fldKey in subscription) { if (fldCol in subscription) {
subscription['source_' + fldKey] = source; subscription['source_' + fldCol] = source;
} }
} }
} }
@ -490,39 +497,42 @@ async function _create(tx, listId, filteredEntity) {
If it is unsubscribed and meta.updateOfUnsubscribedAllowed, the existing subscription is changed based on the provided data. If it is unsubscribed and meta.updateOfUnsubscribedAllowed, the existing subscription is changed based on the provided data.
If meta.updateAllowed is true, it updates even an active subscription. If meta.updateAllowed is true, it updates even an active subscription.
*/ */
async function create(context, listId, entity, source, meta /* meta is provided when called from /confirm/subscribe/:cid */) { async function createTxWithGroupedFieldsMap(tx, context, listId, groupedFieldsMap, entity, source, meta) {
return await knex.transaction(async tx => { await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageSubscriptions');
const groupedFieldsMap = await getGroupedFieldsMap(tx, listId); const allowedKeys = getAllowedKeys(groupedFieldsMap);
const allowedKeys = getAllowedKeys(groupedFieldsMap);
await _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta, true); await _validateAndPreprocess(tx, listId, groupedFieldsMap, entity, meta, true);
const filteredEntity = filterObject(entity, allowedKeys); const filteredEntity = filterObject(entity, allowedKeys);
filteredEntity.status_change = new Date(); filteredEntity.status_change = new Date();
ungroupSubscription(groupedFieldsMap, filteredEntity); ungroupSubscription(groupedFieldsMap, filteredEntity);
updateSourcesAndHash(filteredEntity, source, groupedFieldsMap); updateSourcesAndHashEmail(filteredEntity, source, groupedFieldsMap);
filteredEntity.opt_in_ip = meta && meta.ip; filteredEntity.opt_in_ip = meta && meta.ip;
filteredEntity.opt_in_country = meta && meta.country; filteredEntity.opt_in_country = meta && meta.country;
filteredEntity.imported = meta && !!meta.imported;
if (meta && meta.update) { // meta.update is set by _validateAndPreprocess if (meta && meta.update) { // meta.update is set by _validateAndPreprocess
await _update(tx, listId, meta.existing, filteredEntity); await _update(tx, listId, meta.existing, filteredEntity);
meta.cid = meta.existing.cid; // The cid is needed by /confirm/subscribe/:cid meta.cid = meta.existing.cid; // The cid is needed by /confirm/subscribe/:cid
return meta.existing.id; return meta.existing.id;
} else { } else {
filteredEntity.cid = shortid.generate(); filteredEntity.cid = shortid.generate();
if (meta) { if (meta) {
meta.cid = filteredEntity.cid; // The cid is needed by /confirm/subscribe/:cid meta.cid = filteredEntity.cid; // The cid is needed by /confirm/subscribe/:cid
}
return await _create(tx, listId, filteredEntity);
} }
return await _create(tx, listId, filteredEntity);
}
}
async function create(context, listId, entity, source, meta) {
return await knex.transaction(async tx => {
const groupedFieldsMap = await getGroupedFieldsMap(tx, listId);
return await createTxWithGroupedFieldsMap(tx, context, listId, groupedFieldsMap, entity, source, meta);
}); });
} }
@ -551,7 +561,7 @@ async function updateWithConsistencyCheck(context, listId, entity, source) {
ungroupSubscription(groupedFieldsMap, filteredEntity); ungroupSubscription(groupedFieldsMap, filteredEntity);
updateSourcesAndHash(filteredEntity, source, groupedFieldsMap); updateSourcesAndHashEmail(filteredEntity, source, groupedFieldsMap);
await _update(tx, listId, existing, filteredEntity); await _update(tx, listId, existing, filteredEntity);
}); });
@ -580,7 +590,7 @@ async function remove(context, listId, id) {
async function removeByEmailAndGet(context, listId, email) { async function removeByEmailAndGet(context, listId, email) {
return await knex.transaction(async tx => { return await knex.transaction(async tx => {
const existing = await tx(getSubscriptionTableName(listId)).where('email', email).first(); const existing = await tx(getSubscriptionTableName(listId)).where('hash_email', hashEmail(email)).first();
return await _removeAndGetTx(tx, context, listId, existing); return await _removeAndGetTx(tx, context, listId, existing);
}); });
} }
@ -624,21 +634,25 @@ async function _unsubscribeAndGetTx(tx, context, listId, existingSubscription, c
async function unsubscribeByIdAndGet(context, listId, subscriptionId) { async function unsubscribeByIdAndGet(context, listId, subscriptionId) {
return await knex.transaction(async tx => { return await knex.transaction(async tx => {
const existing = await tx(getSubscriptionTableName(listId)).where('id', subscriptionId).first(); const existing = await tx(getSubscriptionTableName(listId)).where('id', subscriptionId).first();
return _unsubscribeAndGetTx(tx, context, listId, existing); return await _unsubscribeAndGetTx(tx, context, listId, existing);
}); });
} }
async function unsubscribeByCidAndGet(context, listId, subscriptionCid, campaignCid) { async function unsubscribeByCidAndGet(context, listId, subscriptionCid, campaignCid) {
return await knex.transaction(async tx => { return await knex.transaction(async tx => {
const existing = await tx(getSubscriptionTableName(listId)).where('cid', subscriptionCid).first(); const existing = await tx(getSubscriptionTableName(listId)).where('cid', subscriptionCid).first();
return _unsubscribeAndGetTx(tx, context, listId, existing, campaignCid); return await _unsubscribeAndGetTx(tx, context, listId, existing, campaignCid);
}); });
} }
async function unsubscribeByEmailAndGetTx(tx, context, listId, email) {
const existing = await tx(getSubscriptionTableName(listId)).where('hash_email', hashEmail(email)).first();
return await _unsubscribeAndGetTx(tx, context, listId, existing);
}
async function unsubscribeByEmailAndGet(context, listId, email) { async function unsubscribeByEmailAndGet(context, listId, email) {
return await knex.transaction(async tx => { return await knex.transaction(async tx => {
const existing = await tx(getSubscriptionTableName(listId)).where('email', email).first(); return await unsubscribeByEmailAndGetTx(tx, context, listId, email);
return _unsubscribeAndGetTx(tx, context, listId, existing);
}); });
} }
@ -652,7 +666,7 @@ async function updateAddressAndGet(context, listId, subscriptionId, emailNew) {
} }
if (existing.email !== emailNew) { if (existing.email !== emailNew) {
await tx(getSubscriptionTableName(listId)).where('email', emailNew).del(); await tx(getSubscriptionTableName(listId)).where('hash_email', hashEmail(emailNew)).del();
await tx(getSubscriptionTableName(listId)).where('id', subscriptionId).update({ await tx(getSubscriptionTableName(listId)).where('id', subscriptionId).update({
email: emailNew email: emailNew
@ -717,12 +731,15 @@ module.exports = {
listDTAjax, listDTAjax,
serverValidate, serverValidate,
create, create,
getGroupedFieldsMap,
createTxWithGroupedFieldsMap,
updateWithConsistencyCheck, updateWithConsistencyCheck,
remove, remove,
removeByEmailAndGet, removeByEmailAndGet,
unsubscribeByCidAndGet, unsubscribeByCidAndGet,
unsubscribeByIdAndGet, unsubscribeByIdAndGet,
unsubscribeByEmailAndGet, unsubscribeByEmailAndGet,
unsubscribeByEmailAndGetTx,
updateAddressAndGet, updateAddressAndGet,
updateManaged, updateManaged,
getListsWithEmail getListsWithEmail

4
routes/api.js
View file

@ -37,14 +37,14 @@ router.postAsync('/subscribe/:listCid', passport.loggedIn, async (req, res) => {
throw new APIError('Missing EMAIL', 400); throw new APIError('Missing EMAIL', 400);
} }
const emailErr = await tools.validateEmail(input.EMAIL, false); const emailErr = await tools.validateEmail(input.EMAIL);
if (emailErr) { if (emailErr) {
const errMsg = tools.validateEmailGetMessage(emailErr, input.email); const errMsg = tools.validateEmailGetMessage(emailErr, input.email);
log.error('API', errMsg); log.error('API', errMsg);
throw new APIError(errMsg, 400); throw new APIError(errMsg, 400);
} }
const subscription = await fields.fromPost(req.context, list.id, input, true); const subscription = await fields.fromAPI(req.context, list.id, input);
if (input.TIMEZONE) { if (input.TIMEZONE) {
subscription.tz = (input.TIMEZONE || '').toString().trim(); subscription.tz = (input.TIMEZONE || '').toString().trim();

519
routes/rest/editors.js
View file

@ -8,525 +8,6 @@ const premailerPrepareAsync = bluebird.promisify(premailerApi.prepare);
const router = require('../../lib/router-async').create(); const router = require('../../lib/router-async').create();
/*
FIXME
const { nodeifyFunction } = require('../lib/nodeify');
const getSettings = nodeifyFunction(require('../models/settings').get);
const htmlToText = require('html-to-text');
const log = require('npmlog');
const config = require('config');
const express = require('express');
const router = new express.Router();
const passport = require('../lib/passport');
const os = require('os');
const fs = require('fs');
const path = require('path');
const mkdirp = require('mkdirp');
const crypto = require('crypto');
const events = require('events');
const httpMocks = require('node-mocks-http');
const multiparty = require('multiparty');
const escapeStringRegexp = require('escape-string-regexp');
const jqueryFileUpload = require('jquery-file-upload-middleware');
const gm = require('gm').subClass({
imageMagick: true
});
const url = require('url');
const _ = require('../lib/translate')._;
const mailer = require('../lib/mailer');
const templates = require('../lib/models/templates');
const campaigns = require('../lib/models/campaigns');
router.all('/*', (req, res, next) => {
if (!req.user) {
return res.status(403).send(_('Need to be logged in to access restricted content'));
}
if (req.originalUrl.startsWith('/editorapi/img?')) {
return next();
}
if (!config.editors.map(e => e[0]).includes(req.query.editor)) {
return res.status(500).send(_('Invalid editor name'));
}
next();
});
jqueryFileUpload.on('begin', fileInfo => {
fileInfo.name = fileInfo.name
.toLowerCase()
.replace(/ /g, '-')
.replace(/[^a-z0-9+-.]+/g, '');
});
const listImages = (dir, dirURL, callback) => {
fs.readdir(dir, (err, files = []) => {
if (err && err.code !== 'ENOENT') {
return callback(err.message || err);
}
files = files.filter(name => /\.(jpe?g|png|gif)$/i.test(name));
files = files.map(name => ({
// mosaico
name,
url: dirURL + '/' + name,
thumbnailUrl: dirURL + '/thumbnail/' + name,
// grapejs
src: dirURL + '/' + name
}));
callback(null, files);
});
};
const placeholderImage = (width, height, callback) => {
const magick = gm(width, height, '#707070');
const size = 40;
let x = 0;
let y = 0;
// stripes
while (y < height) {
magick
.fill('#808080')
.drawPolygon([x, y], [x + size, y], [x + size * 2, y + size], [x + size * 2, y + size * 2])
.drawPolygon([x, y + size], [x + size, y + size * 2], [x, y + size * 2]);
x = x + size * 2;
if (x > width) {
x = 0;
y = y + size * 2;
}
}
// text
magick
.fill('#B0B0B0')
.fontSize(20)
.drawText(0, 0, width + ' x ' + height, 'center');
magick.stream('png', (err, stream) => {
if (err) {
return callback(err);
}
const image = {
format: 'PNG',
stream
};
callback(null, image);
});
};
const resizedImage = (src, method, width, height, callback) => {
const pathname = path.join('/', url.parse(src).pathname);
const filePath = path.join(__dirname, '..', 'public', pathname);
const magick = gm(filePath);
magick.format((err, format) => {
if (err) {
return callback(err);
}
const streamHandler = (err, stream) => {
if (err) {
return callback(err);
}
const image = {
format,
stream
};
callback(null, image);
};
switch (method) {
case 'resize':
return magick
.autoOrient()
.resize(width, height)
.stream(streamHandler);
case 'cover':
return magick
.autoOrient()
.resize(width, height + '^')
.gravity('Center')
.extent(width, height + '>')
.stream(streamHandler);
default:
return callback(new Error(_('Method not supported')));
}
});
};
const getProcessedImage = (dynamicUrl, callback) => {
if (!dynamicUrl.includes('/editorapi/img?')) {
return callback(new Error('Invalid dynamicUrl'));
}
const {
src,
method,
params = '600,null'
} = url.parse(dynamicUrl, true).query;
let width = params.split(',')[0];
let height = params.split(',')[1];
const sanitizeSize = (val, min, max, defaultVal, allowNull) => {
if (val === 'null' && allowNull) {
return null;
}
val = Number(val) || defaultVal;
val = Math.max(min, val);
val = Math.min(max, val);
return val;
};
if (method === 'placeholder') {
width = sanitizeSize(width, 1, 2048, 600, false);
height = sanitizeSize(height, 1, 2048, 300, false);
placeholderImage(width, height, callback);
} else {
width = sanitizeSize(width, 1, 2048, 600, false);
height = sanitizeSize(height, 1, 2048, 300, true);
resizedImage(src, method, width, height, callback);
}
};
const getStaticImageUrl = (dynamicUrl, staticDir, staticDirUrl, callback) => {
if (!dynamicUrl.includes('/editorapi/img?')) {
return callback(null, dynamicUrl);
}
mkdirp(staticDir, err => {
if (err) {
return callback(err);
}
fs.readdir(staticDir, (err, files) => {
if (err) {
return callback(err);
}
const hash = crypto.createHash('md5').update(dynamicUrl).digest('hex');
const match = files.find(el => el.startsWith(hash));
if (match) {
return callback(null, staticDirUrl + '/' + match);
}
getProcessedImage(dynamicUrl, (err, image) => {
if (err) {
return callback(err);
}
const fileName = hash + '.' + image.format.toLowerCase();
const filePath = path.join(staticDir, fileName);
const fileUrl = staticDirUrl + '/' + fileName;
const writeStream = fs.createWriteStream(filePath);
writeStream.on('error', err => callback(err));
writeStream.on('finish', () => callback(null, fileUrl));
image.stream.pipe(writeStream);
});
});
});
};
const prepareHtml = (html, editorName, callback) => {
getSettings('serviceUrl', (err, serviceUrl) => {
if (err) {
return callback(err.message || err);
}
const srcs = new Map();
const re = /<img[^>]+src="([^"]*\/editorapi\/img\?[^"]+)"/ig;
let jobs = 0;
let result;
while ((result = re.exec(html)) !== null) {
srcs.set(result[1], result[1]);
}
const done = () => {
if (jobs === 0) {
for (const [key, value] of srcs) {
// console.log(`replace dynamicUrl: ${key} - with staticUrl: ${value}`);
html = html.replace(new RegExp(escapeStringRegexp(key), 'g'), value);
}
return callback(null, html);
}
};
const staticDir = path.join(__dirname, '..', 'public', editorName, 'uploads', 'static');
const staticDirUrl = url.resolve(serviceUrl, editorName + '/uploads/static');
for (const key of srcs.keys()) {
jobs++;
const dynamicUrl = key.replace(/&amp;/g, '&');
getStaticImageUrl(dynamicUrl, staticDir, staticDirUrl, (err, staticUrl) => {
if (err) {
// TODO: Send a warning back to the editor. For now we just skip image resizing.
log.error('editorapi', err);
if (dynamicUrl.includes('/editorapi/img?')) {
staticUrl = url.parse(dynamicUrl, true).query.src || dynamicUrl;
} else {
staticUrl = dynamicUrl;
}
if (!/^https?:\/\/|^\/\//i.test(staticUrl)) {
staticUrl = url.resolve(serviceUrl, staticUrl);
}
}
srcs.set(key, staticUrl);
jobs--;
done();
});
}
done();
});
};
// URL structure defined by Mosaico
// /editorapi/img?src=" + encodeURIComponent(src) + "&method=" + encodeURIComponent(method) + "&params=" + encodeURIComponent(width + "," + height);
router.get('/img', (req, res) => {
getProcessedImage(req.originalUrl, (err, image) => {
if (err) {
res.status(err.status || 500);
res.send(err.message || err);
return;
}
res.set('Content-Type', 'image/' + image.format.toLowerCase());
image.stream.pipe(res);
});
});
router.post('/update', passport.parseForm, passport.csrfProtection, (req, res) => {
const sendResponse = err => {
if (err) {
return res.status(500).send(err.message || err);
}
res.send('ok');
};
prepareHtml(req.body.html, req.query.editor, (err, html) => {
if (err) {
return sendResponse(err);
}
req.body.html = html;
switch (req.query.type) {
case 'template':
return templates.update(req.body.id, req.body, sendResponse);
case 'campaign':
return campaigns.update(req.body.id, req.body, sendResponse);
default:
return sendResponse(new Error(_('Invalid resource type')));
}
});
});
// https://github.com/artf/grapesjs/wiki/API-Asset-Manager
// https://github.com/aguidrevitch/jquery-file-upload-middleware
router.get('/upload', passport.csrfProtection, (req, res) => {
getSettings('serviceUrl', (err, serviceUrl) => {
if (err) {
return res.status(500).send(err.message || err);
}
const baseDir = path.join(__dirname, '..', 'public', req.query.editor, 'uploads');
const baseDirUrl = serviceUrl + req.query.editor + '/uploads';
listImages(path.join(baseDir, '0'), baseDirUrl + '/0', (err, sharedImages) => {
if (err) {
return res.status(500).send(err.message || err);
}
if (req.query.type === 'campaign' && Number(req.query.id) > 0) {
listImages(path.join(baseDir, req.query.id), baseDirUrl + '/' + req.query.id, (err, campaignImages) => {
if (err) {
return res.status(500).send(err.message || err);
}
res.json({
files: sharedImages.concat(campaignImages)
});
});
} else {
res.json({
files: sharedImages
});
}
});
});
});
router.post('/upload', passport.csrfProtection, (req, res) => {
getSettings('serviceUrl', (err, serviceUrl) => {
if (err) {
return res.status(500).send(err.message || err);
}
const getDirName = () => {
switch (req.query.type) {
case 'template':
return '0';
case 'campaign':
return Number(req.query.id) > 0 ? req.query.id : false;
default:
return false;
}
};
const dirName = getDirName();
const serviceUrlParts = url.parse(serviceUrl);
if (dirName === false) {
return res.status(500).send(_('Invalid resource type or ID'));
}
const opts = {
tmpDir: config.www.tmpdir || os.tmpdir(),
imageVersions: req.query.editor === 'mosaico' ? {
thumbnail: {
width: 90,
height: 90
}
} : {},
uploadDir: path.join(__dirname, '..', 'public', req.query.editor, 'uploads', dirName),
uploadUrl: '/' + req.query.editor + '/uploads/' + dirName, // must be root relative
acceptFileTypes: /\.(gif|jpe?g|png)$/i,
hostname: serviceUrlParts.host, // include port
ssl: serviceUrlParts.protocol === 'https:'
};
const mockres = httpMocks.createResponse({
eventEmitter: events.EventEmitter
});
mockres.on('error', err => {
res.status(500).json({
error: err.message || err,
data: []
});
});
mockres.on('end', () => {
const data = [];
try {
JSON.parse(mockres._getData()).files.forEach(file => {
data.push({
src: file.url
});
});
res.json({
data
});
} catch(err) {
res.status(500).json({
error: err.message || err,
data
});
}
});
jqueryFileUpload.fileHandler(opts)(req, req.query.editor === 'grapejs' ? mockres : res);
});
});
router.post('/download', passport.csrfProtection, (req, res) => {
prepareHtml(req.body.html, req.query.editor, (err, html) => {
if (err) {
return res.status(500).send(err.message || err);
}
res.setHeader('Content-disposition', 'attachment; filename=' + req.body.filename);
res.setHeader('Content-type', 'text/html');
res.send(html);
});
});
const parseGrapejsMultipartTestForm = (req, res, next) => {
if (req.query.editor === 'grapejs') {
new multiparty.Form().parse(req, (err, fields) => {
if (err) {
return next(err);
}
req.body.email = fields.email[0];
req.body.subject = fields.subject[0];
req.body.html = fields.html[0];
req.body._csrf = fields._csrf[0];
next();
});
} else {
next();
}
};
router.post('/test', parseGrapejsMultipartTestForm, passport.csrfProtection, (req, res) => {
const sendError = err => {
if (req.query.editor === 'grapejs') {
res.status(500).json({
errors: err.message || err
});
} else {
res.status(500).send(err.message || err);
}
};
prepareHtml(req.body.html, req.query.editor, (err, html) => {
if (err) {
return sendError(err);
}
getSettings(['defaultAddress', 'defaultFrom'], (err, configItems) => {
if (err) {
return sendError(err);
}
mailer.getMailer((err, transport) => {
if (err) {
return sendError(err);
}
const opts = {
from: {
name: configItems.defaultFrom,
address: configItems.defaultAddress
},
to: req.body.email,
subject: req.body.subject,
text: htmlToText.fromString(html, {
wordwrap: 100
}),
html
};
transport.sendMail(opts, err => {
if (err) {
return sendError(err);
}
if (req.query.editor === 'grapejs') {
res.json({
data: 'ok'
});
} else {
res.send('ok');
}
});
});
});
});
});
*/
router.postAsync('/html-to-text', passport.loggedIn, passport.csrfProtection, async (req, res) => { router.postAsync('/html-to-text', passport.loggedIn, passport.csrfProtection, async (req, res) => {
const email = await premailerPrepareAsync({ const email = await premailerPrepareAsync({
html: req.body.html, html: req.body.html,

4
routes/rest/import-runs.js
View file

@ -9,6 +9,10 @@ router.postAsync('/import-runs-table/:listId/:importId', passport.loggedIn, asyn
return res.json(await importRuns.listDTAjax(req.context, req.params.listId, req.params.importId, req.body)); return res.json(await importRuns.listDTAjax(req.context, req.params.listId, req.params.importId, req.body));
}); });
router.postAsync('/import-run-failed-table/:listId/:importId/:importRunId', passport.loggedIn, async (req, res) => {
return res.json(await importRuns.listFailedDTAjax(req.context, req.params.listId, req.params.importId, req.params.importRunId, req.body));
});
router.getAsync('/import-runs/:listId/:importId/:runId', passport.loggedIn, async (req, res) => { router.getAsync('/import-runs/:listId/:importId/:runId', passport.loggedIn, async (req, res) => {
const entity = await importRuns.getById(req.context, req.params.listId, req.params.importId, req.params.runId); const entity = await importRuns.getById(req.context, req.params.listId, req.params.importId, req.params.runId);
return res.json(entity); return res.json(entity);

241
services/importer.js
View file

@ -6,14 +6,22 @@ const log = require('npmlog');
const fsExtra = require('fs-extra-promise'); const fsExtra = require('fs-extra-promise');
const {ImportSource, MappingType, ImportStatus, RunStatus} = require('../shared/imports'); const {ImportSource, MappingType, ImportStatus, RunStatus} = require('../shared/imports');
const imports = require('../models/imports'); const imports = require('../models/imports');
const fields = require('../models/fields');
const subscriptions = require('../models/subscriptions');
const { Writable } = require('stream'); const { Writable } = require('stream');
const { enforce } = require('../lib/helpers'); const { cleanupFromPost, enforce } = require('../lib/helpers');
const contextHelpers = require('../lib/context-helpers');
const tools = require('../lib/tools');
const shares = require('../models/shares');
const _ = require('../lib/translate')._;
const csvparse = require('csv-parse'); const csvparse = require('csv-parse');
const fs = require('fs'); const fs = require('fs');
let running = false; let running = false;
const maxInsertBatchSize = 100; const maxPrepareBatchSize = 100;
const maxImportBatchSize = 10;
function prepareCsv(impt) { function prepareCsv(impt) {
// Processing of CSV intake // Processing of CSV intake
@ -71,6 +79,8 @@ function prepareCsv(impt) {
} }
impt.settings.csv.columns = cols; impt.settings.csv.columns = cols;
impt.settings.sourceTable = importTable;
await knex('imports').where({id: impt.id}).update({settings: JSON.stringify(impt.settings)}); await knex('imports').where({id: impt.id}).update({settings: JSON.stringify(impt.settings)});
await knex.schema.raw('CREATE TABLE `' + importTable + '` (\n' + await knex.schema.raw('CREATE TABLE `' + importTable + '` (\n' +
@ -88,7 +98,7 @@ function prepareCsv(impt) {
insertBatch.push(dbRecord); insertBatch.push(dbRecord);
} }
if (insertBatch.length >= maxInsertBatchSize) { if (insertBatch.length >= maxPrepareBatchSize) {
await knex(importTable).insert(insertBatch); await knex(importTable).insert(insertBatch);
insertBatch = []; insertBatch = [];
} }
@ -126,27 +136,220 @@ function prepareCsv(impt) {
inputStream.pipe(parser); inputStream.pipe(parser);
} }
async function basicSubscribe(impt) { async function _execImportRun(impt, handlers) {
let imptRun; // FIXME - handle STOPPING
while (imptRun = await knex('import_runs').where('import', impt.id).whereIn('status', [RunStatus.SCHEDULED]).orderBy('created', 'asc').first()) { try {
await knex('import_runs').where('id', imptRun.id).update({ let imptRun;
status: RunStatus.RUNNING
// It should not really happen that we have more than one run to be processed for an import. However, to be on the safe side, we process it in a while.
while (imptRun = await knex('import_runs').where('import', impt.id).whereIn('status', [RunStatus.SCHEDULED]).orderBy('created', 'asc').first()) {
try {
imptRun.mapping = JSON.parse(imptRun.mapping) || {};
log.info('Importer', `Starting BASIC_SUBSCRIBE run ${impt.id}.${imptRun.id}`);
await knex('import_runs').where('id', imptRun.id).update({
status: RunStatus.RUNNING
});
const importTable = impt.settings.sourceTable;
const flds = await fields.list(contextHelpers.getAdminContext(), impt.list);
let lastId = imptRun.last_id || 0;
let countNew = imptRun.new || 0;
let countProcessed = imptRun.processed || 0;
let countFailed = imptRun.failed || 0;
while (true) {
const rows = await knex(importTable).orderBy('id', 'asc').where('id', '>', lastId).limit(maxImportBatchSize);
log.verbose('Importer', `Processing run ${impt.id}.${imptRun.id} with id > ${lastId} ... ${rows.length} entries`);
if (rows.length === 0) {
break;
}
const subscrs = [];
const unsubscrs = [];
const failures = [];
// This should help in case we do the DNS check inside process row because it does all the checks at the same time.
await Promise.all(rows.map(row => handlers.processSourceRow(impt, imptRun, flds, row, subscrs, unsubscrs, failures)));
lastId = rows[rows.length - 1].id;
await knex.transaction(async tx => {
const groupedFieldsMap = await subscriptions.getGroupedFieldsMap(tx, impt.list);
let newRows = 0;
for (const subscr of subscrs) {
const meta = {
updateAllowed: true,
updateOfUnsubscribedAllowed: true,
subscribeIfNoExisting: true
};
try {
await subscriptions.createTxWithGroupedFieldsMap(tx, contextHelpers.getAdminContext(), impt.list, groupedFieldsMap, subscr, impt.id, meta);
if (!meta.existing) {
newRows += 1;
}
} catch (err) {
failures.push({
run: imptRun.id,
source_id: subscr.source_id,
email: subscr.email,
reason: err.message
});
}
}
for (const unsubscr of unsubscrs) {
try {
await subscriptions.unsubscribeByEmailAndGetTx(tx, contextHelpers.getAdminContext(), impt.list, unsubscr.email);
} catch (err) {
failures.push({
run: imptRun.id,
source_id: unsubscr.source_id,
email: unsubscr.email,
reason: err.message
});
}
}
countProcessed += rows.length;
countNew += newRows;
countFailed += failures.length;
if (failures.length > 0) {
await tx('import_failed').insert(failures);
}
await tx('import_runs').where('id', imptRun.id).update({
last_id: lastId,
new: countNew,
failed: countFailed,
processed: countProcessed
});
});
const imptRunStatus = await knex('import_runs').where('id', imptRun.id).select(['status']).first();
if (imptRunStatus.status === RunStatus.STOPPING) {
throw new Error('Aborted');
}
}
await knex('import_runs').where('id', imptRun.id).update({
status: RunStatus.FINISHED,
error: null,
finished: new Date()
});
log.info('Importer', `BASIC_SUBSCRIBE run ${impt.id}.${imptRun.id} finished`);
} catch (err) {
await knex('import_runs').where('id', imptRun.id).update({
status: RunStatus.FAILED,
error: err.message,
finished: new Date()
});
throw new Error(_('Last run failed'));
}
}
await knex('imports').where('id', impt.id).update({
last_run: new Date(),
error: null,
status: ImportStatus.RUN_FINISHED
}); });
} catch (err) {
await knex('imports').where('id', impt.id).update({
await knex('import_runs').where('id', imptRun.id).update({ last_run: new Date(),
status: RunStatus.FINISHED error: err.message,
status: ImportStatus.RUN_FAILED
}); });
} }
}
await knex('imports').where('id', impt.id).update({ async function basicSubscribe(impt) {
status: ImportStatus.RUN_FINISHED const handlers = {
}); processSourceRow: async (impt, imptRun, flds, row, subscriptions, unsubscriptions, failures) => {
const mappingFields = imptRun.mapping.fields || {};
const mappingSettings = imptRun.mapping.settings || {};
const convRow = {};
for (const col in mappingFields) {
const fldMapping = mappingFields[col];
if (fldMapping && fldMapping.column) {
convRow[col] = row[fldMapping.column];
}
}
const subscription = fields.fromImport(impt.list, flds, convRow);
const email = cleanupFromPost(convRow.email);
let errorMsg;
if (!email) {
errorMsg = _('Missing email');
}
if (mappingSettings.checkEmails) {
const emailErr = await tools.validateEmail(email);
if (emailErr) {
errorMsg = tools.validateEmailGetMessage(emailErr, email);
}
}
if (!errorMsg) {
subscription.email = email;
subscription.source_id = row.id;
subscriptions.push(subscription);
} else {
failures.push({
run: imptRun.id,
source_id: row.id,
email: email,
reason: errorMsg
});
}
}
};
return await _execImportRun(impt, handlers);
} }
async function basicUnsubscribe(impt) { async function basicUnsubscribe(impt) {
// FIXME const handlers = {
processSourceRow: async (impt, imptRun, flds, row, subscriptions, unsubscriptions, failures) => {
const emailCol = imptRun.mapping.fields.email.column;
const email = cleanupFromPost(row[emailCol]);
let errorMsg;
if (!email) {
errorMsg = _('Missing email');
}
if (!errorMsg) {
unsubscriptions.push({
source_id: row.id,
email
});
} else {
failures.push({
run: imptRun.id,
source_id: row.id,
email: email,
reason: errorMsg
});
}
}
};
return await _execImportRun(impt, handlers);
} }
async function getTask() { async function getTask() {
@ -154,17 +357,17 @@ async function getTask() {
const impt = await tx('imports').whereIn('status', [ImportStatus.PREP_SCHEDULED, ImportStatus.RUN_SCHEDULED]).orderBy('created', 'asc').first(); const impt = await tx('imports').whereIn('status', [ImportStatus.PREP_SCHEDULED, ImportStatus.RUN_SCHEDULED]).orderBy('created', 'asc').first();
if (impt) { if (impt) {
impt.settings = JSON.parse(impt.settings); impt.settings = JSON.parse(impt.settings) || {};
if (impt.source === ImportSource.CSV_FILE && impt.status === ImportStatus.PREP_SCHEDULED) { if (impt.source === ImportSource.CSV_FILE && impt.status === ImportStatus.PREP_SCHEDULED) {
await tx('imports').where('id', impt.id).update('status', ImportStatus.PREP_RUNNING); await tx('imports').where('id', impt.id).update('status', ImportStatus.PREP_RUNNING);
return () => prepareCsv(impt); return () => prepareCsv(impt);
} else if (impt.status === ImportStatus.RUN_SCHEDULED && impt.settings.mappingType === MappingType.BASIC_SUBSCRIBE) { } else if (impt.status === ImportStatus.RUN_SCHEDULED && impt.mapping_type === MappingType.BASIC_SUBSCRIBE) {
await tx('imports').where('id', impt.id).update('status', ImportStatus.RUN_RUNNING); await tx('imports').where('id', impt.id).update('status', ImportStatus.RUN_RUNNING);
return () => basicSubscribe(impt); return () => basicSubscribe(impt);
} else if (impt.status === ImportStatus.RUN_SCHEDULED && impt.settings.mappingType === MappingType.BASIC_UNSUBSCRIBE) { } else if (impt.status === ImportStatus.RUN_SCHEDULED && impt.mapping_type === MappingType.BASIC_UNSUBSCRIBE) {
await tx('imports').where('id', impt.id).update('status', ImportStatus.RUN_RUNNING); await tx('imports').where('id', impt.id).update('status', ImportStatus.RUN_RUNNING);
return () => basicUnsubscribe(impt); return () => basicUnsubscribe(impt);
} }

12
setup/knex/migrations/20170506102634_v1_to_v2.js
View file

@ -3,12 +3,14 @@ const files = require('../../../models/files');
const contextHelpers = require('../../../lib/context-helpers'); const contextHelpers = require('../../../lib/context-helpers');
const mosaicoTemplates = require('../../../shared/mosaico-templates'); const mosaicoTemplates = require('../../../shared/mosaico-templates');
const {getGlobalNamespaceId} = require('../../../shared/namespaces'); const {getGlobalNamespaceId} = require('../../../shared/namespaces');
const {getAdminId} = require('../../../shared/users');
const entityTypesAddNamespace = ['list', 'custom_form', 'template', 'campaign', 'report', 'report_template', 'user']; const entityTypesAddNamespace = ['list', 'custom_form', 'template', 'campaign', 'report', 'report_template', 'user'];
const shareableEntityTypes = ['list', 'custom_form', 'template', 'campaign', 'report', 'report_template', 'namespace', 'send_configuration', 'mosaico_template']; const shareableEntityTypes = ['list', 'custom_form', 'template', 'campaign', 'report', 'report_template', 'namespace', 'send_configuration', 'mosaico_template'];
const { MailerType, getSystemSendConfigurationId } = require('../../../shared/send-configurations'); const { MailerType, getSystemSendConfigurationId } = require('../../../shared/send-configurations');
const { enforce } = require('../../../lib/helpers'); const { enforce } = require('../../../lib/helpers');
const { EntityVals: TriggerEntityVals, EventVals: TriggerEventVals } = require('../../../shared/triggers'); const { EntityVals: TriggerEntityVals, EventVals: TriggerEventVals } = require('../../../shared/triggers');
const { SubscriptionSource } = require('../../../shared/lists'); const { SubscriptionSource } = require('../../../shared/lists');
const crypto = require('crypto');
const entityTypesWithFiles = { const entityTypesWithFiles = {
campaign: { campaign: {
@ -226,7 +228,7 @@ async function migrateUsers(knex) {
}); });
/* The user role is set automatically in rebuild permissions, which is called upon every start */ /* The user role is set automatically in rebuild permissions, which is called upon every start */
await knex('users').where('id', 1 /* Admin user id */).update({ await knex('users').where('id', getAdminId()).update({
name: 'Administrator' name: 'Administrator'
}); });
} }
@ -1030,7 +1032,7 @@ async function migrateImporter(knex) {
table.text('settings', 'longtext'); table.text('settings', 'longtext');
table.integer('mapping_type').unsigned().notNullable(); table.integer('mapping_type').unsigned().notNullable();
table.text('mapping', 'longtext'); table.text('mapping', 'longtext');
table.timestamp('last_run'); table.dateTime('last_run');
table.text('error'); table.text('error');
table.timestamp('created').defaultTo(knex.fn.now()); table.timestamp('created').defaultTo(knex.fn.now());
}); });
@ -1040,17 +1042,19 @@ async function migrateImporter(knex) {
table.integer('import').unsigned().references('imports.id'); table.integer('import').unsigned().references('imports.id');
table.integer('status').unsigned().notNullable(); table.integer('status').unsigned().notNullable();
table.text('mapping', 'longtext'); table.text('mapping', 'longtext');
table.integer('last_id');
table.integer('new').defaultTo(0); table.integer('new').defaultTo(0);
table.integer('failed').defaultTo(0); table.integer('failed').defaultTo(0);
table.integer('processed').defaultTo(0); table.integer('processed').defaultTo(0);
table.text('error'); table.text('error');
table.timestamp('created').defaultTo(knex.fn.now()); table.timestamp('created').defaultTo(knex.fn.now());
table.timestamp('finished'); table.dateTime('finished');
}); });
await knex.schema.createTable('import_failed', table => { await knex.schema.createTable('import_failed', table => {
table.increments('id').primary(); table.increments('id').primary();
table.integer('run').unsigned().references('import_runs.id'); table.integer('run').unsigned().references('import_runs.id');
table.integer('source_id').unsigned();
table.string('email').notNullable(); table.string('email').notNullable();
table.text('reason'); table.text('reason');
table.timestamp('created').defaultTo(knex.fn.now()); table.timestamp('created').defaultTo(knex.fn.now());
@ -1063,9 +1067,9 @@ exports.up = (knex, Promise) => (async() => {
await addNamespaces(knex); await addNamespaces(knex);
await migrateUsers(knex); await migrateUsers(knex);
await migrateSubscriptions(knex);
await migrateCustomForms(knex); await migrateCustomForms(knex);
await migrateCustomFields(knex); await migrateCustomFields(knex);
await migrateSubscriptions(knex);
await migrateSegments(knex); await migrateSegments(knex);
await migrateReports(knex); await migrateReports(knex);
await migrateSettings(knex); await migrateSettings(knex);

3
shared/imports.js
View file

@ -36,7 +36,8 @@ const RunStatus = {
SCHEDULED: 0, SCHEDULED: 0,
RUNNING: 1, RUNNING: 1,
STOPPING: 2, STOPPING: 2,
FINISHED: 3 FINISHED: 3,
FAILED: 4
}; };
function prepInProgress(status) { function prepInProgress(status) {

4
shared/lists.js
View file

@ -31,7 +31,7 @@ const SubscriptionSource = {
IMPORTED_V1: -5 IMPORTED_V1: -5
}; };
function getFieldKey(field) { function getFieldColumn(field) {
return field.column || 'grouped_' + field.id; return field.column || 'grouped_' + field.id;
} }
@ -39,5 +39,5 @@ module.exports = {
UnsubscriptionMode, UnsubscriptionMode,
SubscriptionStatus, SubscriptionStatus,
SubscriptionSource, SubscriptionSource,
getFieldKey getFieldColumn
}; };

9
shared/users.js Normal file
View file

@ -0,0 +1,9 @@
'use strict';
function getAdminId() {
return 1;
}
module.exports = {
getAdminId
};