public/mailtrain
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Some additions to import UI to cover the basic subscribe and unsubscribe cases.

Browse source
This commit is contained in:
Tomas Bures 2018-08-26 15:32:03 +02:00
parent 739b9452de
commit 16519c5353
13 changed files with 250 additions and 139 deletions
Download patch file Download diff file Expand all files Collapse all files

4
client/src/lib/page.js
View file

@ -326,7 +326,9 @@ class SectionContent extends Component {
errorHandler(error) {
if (error instanceof interoperableErrors.NotLoggedInError) {
this.navigateTo('/account/login?next=' + encodeURIComponent(window.location.pathname));
if (window.location.pathname !== '/account/login') { // There may be multiple async requests failing at the same time. So we take the pathname only from the first one.
this.navigateTo('/account/login?next=' + encodeURIComponent(window.location.pathname));
}
} else if (error.response && error.response.data && error.response.data.message) {
console.error(error);
this.navigateToWithFlashMessage(this.props.root, 'danger', error.response.data.message);

229
client/src/lists/imports/CUD.js
View file

@ -12,6 +12,7 @@ import {
import {
Button,
ButtonRow,
CheckBox,
Dropdown,
Fieldset,
Form,
@ -28,8 +29,9 @@ import {
import {DeleteModalDialog} from "../../lib/modals";
import {getImportTypes} from './helpers';
import {
ImportType,
ImportSource,
inProgress,
MappingType,
prepInProgress,
runInProgress
} from '../../../../shared/imports';
@ -60,13 +62,18 @@ export default class CUD extends Component {
this.state = {};
const {importTypeLabels} = getImportTypes(props.t);
const {importSourceLabels, mappingTypeLabels} = getImportTypes(props.t);
this.importTypeLabels = importTypeLabels;
this.importSourceLabels = importSourceLabels;
this.importTypeOptions = [
{key: ImportType.CSV_FILE, label: importTypeLabels[ImportType.CSV_FILE]},
// {key: ImportType.LIST, label: importTypeLabels[ImportType.LIST]}
this.importSourceOptions = [
{key: ImportSource.CSV_FILE, label: importSourceLabels[ImportSource.CSV_FILE]},
// {key: ImportSource.LIST, label: importSourceLabels[ImportSource.LIST]}
];
this.mappingOptions = [
{key: MappingType.BASIC_SUBSCRIBE, label: mappingTypeLabels[MappingType.BASIC_SUBSCRIBE]},
{key: MappingType.BASIC_UNSUBSCRIBE, label: mappingTypeLabels[MappingType.BASIC_UNSUBSCRIBE]},
];
this.refreshTimeoutHandler = ::this.refreshEntity;
@ -87,26 +94,30 @@ export default class CUD extends Component {
data.settings = data.settings || {};
const mapping = data.mapping || {};
if (data.type === ImportType.CSV_FILE) {
if (data.source === ImportSource.CSV_FILE) {
data.csvFileName = data.settings.csv.originalname;
data.csvDelimiter = data.settings.csv.delimiter;
}
const mappingSettings = mapping.settings || {};
data.mapping_settings_checkEmails = 'checkEmails' in mappingSettings ? !!mappingSettings.checkEmails : true;
const mappingFlds = mapping.fields || {};
for (const field of this.props.fieldsGrouped) {
if (field.column) {
const colMapping = mapping[field.column] || {};
data['mapping_' + field.column + '_column'] = colMapping.column || '';
const colMapping = mappingFlds[field.column] || {};
data['mapping_fields_' + field.column + '_column'] = colMapping.column || '';
} else {
for (const option of field.settings.options) {
const col = field.groupedOptions[option.key].column;
const colMapping = mapping[col] || {};
data['mapping_' + col + '_column'] = colMapping.column || '';
const colMapping = mappingFlds[col] || {};
data['mapping_fields_' + col + '_column'] = colMapping.column || '';
}
}
}
const emailMapping = mapping.email || {};
data.mapping_email_column = emailMapping.column || '';
const emailMapping = mappingFlds.email || {};
data.mapping_fields_email_column = emailMapping.column || '';
});
if (inProgress(entity.status)) {
@ -121,9 +132,9 @@ export default class CUD extends Component {
this.populateFormValues({
name: '',
description: '',
type: ImportType.CSV_FILE,
source: ImportSource.CSV_FILE,
csvFileName: '',
csvDelimiter: ','
csvDelimiter: ',',
});
}
}
@ -141,8 +152,7 @@ export default class CUD extends Component {
localValidateFormValues(state) {
const t = this.props.t;
const isEdit = !!this.props.entity;
const type = Number.parseInt(state.getIn(['type', 'value']));
const status = this.getFormValue('status');
const source = Number.parseInt(state.getIn(['source', 'value']));
for (const key of state.keys()) {
state.setIn([key, 'error'], null);
@ -152,19 +162,23 @@ export default class CUD extends Component {
state.setIn(['name', 'error'], t('Name must not be empty'));
}
if (!isEdit && type === ImportType.CSV_FILE) {
if (!this.csvFile || this.csvFile.files.length === 0) {
state.setIn(['csvFileName', 'error'], t('File must be selected'));
}
if (!isEdit) {
if (source === ImportSource.CSV_FILE) {
if (!this.csvFile || this.csvFile.files.length === 0) {
state.setIn(['csvFileName', 'error'], t('File must be selected'));
}
if (!state.getIn(['csvDelimiter', 'value']).trim()) {
state.setIn(['csvDelimiter', 'error'], t('CSV delimiter must not be empty'));
if (!state.getIn(['csvDelimiter', 'value']).trim()) {
state.setIn(['csvDelimiter', 'error'], t('CSV delimiter must not be empty'));
}
}
}
} else {
const mappingType = Number.parseInt(state.getIn(['mapping_type', 'value']));
if (isEdit) {
if (!state.getIn(['mapping_email_column', 'value'])) {
state.setIn(['mapping_email_column', 'error'], t('Email mapping has to be provided'));
if (mappingType === MappingType.BASIC_SUBSCRIBE) {
if (!state.getIn(['mapping_fields_email_column', 'value'])) {
state.setIn(['mapping_fields_email_column', 'error'], t('Email mapping has to be provided'));
}
}
}
}
@ -173,7 +187,6 @@ export default class CUD extends Component {
const t = this.props.t;
const isEdit = !!this.props.entity;
const type = Number.parseInt(this.getFormValue('type'));
let sendMethod, url;
if (this.props.entity) {
@ -189,44 +202,56 @@ export default class CUD extends Component {
this.setFormStatusMessage('info', t('Saving ...'));
const submitResponse = await this.validateAndSendFormValuesToURL(sendMethod, url, data => {
data.type = Number.parseInt(data.type);
data.source = Number.parseInt(data.source);
data.settings = {};
const formData = new FormData();
if (!isEdit && data.type === ImportType.CSV_FILE) {
data.settings.csv = {};
formData.append('csvFile', this.csvFile.files[0]);
data.settings.csv.delimiter = data.csvDelimiter.trim();
}
if (isEdit) {
const mapping = {};
for (const field of this.props.fieldsGrouped) {
if (field.column) {
mapping[field.column] = {
column: data['mapping_' + field.column + '_column']
};
delete data['mapping_' + field.column + '_column'];
} else {
for (const option of field.settings.options) {
const col = field.groupedOptions[option.key].column;
mapping[col] = {
column: data['mapping_' + col + '_column']
};
delete data['mapping_' + col + '_column'];
}
}
if (!isEdit) {
if (data.source === ImportSource.CSV_FILE) {
data.settings.csv = {};
formData.append('csvFile', this.csvFile.files[0]);
data.settings.csv.delimiter = data.csvDelimiter.trim();
}
mapping.email = {
column: data.mapping_email_column
} else {
data.mapping_type = Number.parseInt(data.mapping_type);
const mapping = {
fields: {},
settings: {}
};
if (data.mapping_type === MappingType.BASIC_SUBSCRIBE) {
mapping.settings.checkEmails = data.mapping_settings_checkEmails;
for (const field of this.props.fieldsGrouped) {
if (field.column) {
mapping.fields[field.column] = {
column: data['mapping_fields_' + field.column + '_column']
};
} else {
for (const option of field.settings.options) {
const col = field.groupedOptions[option.key].column;
mapping.fields[col] = {
column: data['mapping_fields_' + col + '_column']
};
}
}
}
mapping.fields.email = {
column: data.mapping_fields_email_column
};
}
data.mapping = mapping;
}
for (const key in data) {
if (key.startsWith('mapping_fields') || key.startsWith('mapping_settings')) {
delete data[key];
}
}
delete data.csvFile;
delete data.csvDelimiter;
delete data.sampleRow;
@ -237,7 +262,7 @@ export default class CUD extends Component {
});
if (submitResponse) {
if (!isEdit && type === ImportType.CSV_FILE) {
if (!isEdit) {
this.navigateTo(`/lists/${this.props.list.id}/imports/${submitResponse}/edit`);
} else {
this.navigateToWithFlashMessage(`/lists/${this.props.list.id}/imports/${this.props.entity.id}/status`, 'success', t('Import saved'));
@ -260,12 +285,12 @@ export default class CUD extends Component {
const t = this.props.t;
const isEdit = !!this.props.entity;
const type = Number.parseInt(this.getFormValue('type'));
const source = Number.parseInt(this.getFormValue('source'));
const status = this.getFormValue('status');
const settings = this.getFormValue('settings');
let settingsEdit = null;
if (type === ImportType.CSV_FILE) {
if (source === ImportSource.CSV_FILE) {
if (isEdit) {
settingsEdit =
<div>
@ -284,49 +309,69 @@ export default class CUD extends Component {
let mappingEdit;
if (isEdit) {
if (prepInProgress(status)) {
mappingEdit = <div>{t('Preparation in progress. Please wait till it is done or visit this page later.')}</div>;
} else if (runInProgress(status)) {
mappingEdit = <div>{t('Run in progress. Please wait till it is done or visit this page later.')}</div>;
mappingEdit = (
<div>{t('Preparation in progress. Please wait till it is done or visit this page later.')}</div>
);
} else {
const sampleRow = this.getFormValue('sampleRow');
const sourceOpts = [];
sourceOpts.push({key: '', label: t('–– Select ––')});
if (type === ImportType.CSV_FILE) {
for (const csvCol of settings.csv.columns) {
let help = '';
if (sampleRow) {
help = ' (' + t('e.g.:', {keySeparator: '>', nsSeparator: '|'}) + ' ' + truncate(sampleRow[csvCol.column], 50) + ')';
let mappingSettings = null;
const mappingType = Number.parseInt(this.getFormValue('mapping_type'));
if (mappingType === MappingType.BASIC_SUBSCRIBE) {
const sampleRow = this.getFormValue('sampleRow');
const sourceOpts = [];
sourceOpts.push({key: '', label: t('–– Select ––')});
if (source === ImportSource.CSV_FILE) {
for (const csvCol of settings.csv.columns) {
let help = '';
if (sampleRow) {
help = ' (' + t('e.g.:', {keySeparator: '>', nsSeparator: '|'}) + ' ' + truncate(sampleRow[csvCol.column], 50) + ')';
}
sourceOpts.push({key: csvCol.column, label: csvCol.name + help});
}
sourceOpts.push({key: csvCol.column, label: csvCol.name + help});
}
}
const mappingRows = [
<Dropdown key="email" id="mapping_email_column" label={t('Email')} options={sourceOpts}/>
];
const mappingRows = [
<Dropdown key="email" id="mapping_fields_email_column" label={t('Email')} options={sourceOpts}/>
];
for (const field of this.props.fieldsGrouped) {
if (field.column) {
mappingRows.push(
<Dropdown key={field.column} id={'mapping_' + field.column + '_column'} label={field.name} options={sourceOpts}/>
);
} else {
for (const option of field.settings.options) {
const col = field.groupedOptions[option.key].column;
for (const field of this.props.fieldsGrouped) {
if (field.column) {
mappingRows.push(
<Dropdown key={col} id={'mapping_' + col + '_column'} label={field.groupedOptions[option.key].name} options={sourceOpts}/>
<Dropdown key={field.column} id={'mapping_fields_' + field.column + '_column'} label={field.name} options={sourceOpts}/>
);
} else {
for (const option of field.settings.options) {
const col = field.groupedOptions[option.key].column;
mappingRows.push(
<Dropdown key={col} id={'mapping_fields_' + col + '_column'} label={field.groupedOptions[option.key].name} options={sourceOpts}/>
);
}
}
}
mappingSettings = (
<div>
<CheckBox id="mapping_settings_checkEmails" text={t('Check imported emails')}/>
<Fieldset label={t('Mapping')} className={styles.mapping}>
{mappingRows}
</Fieldset>
</div>
);
}
mappingEdit = mappingRows;
mappingEdit = (
<div>
<Dropdown id="mapping_type" label={t('Type')} options={this.mappingOptions}/>
{mappingSettings}
</div>
);
}
}
let saveButtonLabel;
if (!isEdit && type === ImportType.CSV_FILE) {
if (!isEdit) {
saveButtonLabel = t('Save and edit mapping');
} else {
saveButtonLabel = t('Save');
@ -352,18 +397,14 @@ export default class CUD extends Component {
<TextArea id="description" label={t('Description')}/>
{isEdit ?
<StaticField id="type" className={styles.formDisabled} label={t('Type')}>{this.importTypeLabels[this.getFormValue('type')]}</StaticField>
<StaticField id="source" className={styles.formDisabled} label={t('Source')}>{this.importSourceLabels[this.getFormValue('source')]}</StaticField>
:
<Dropdown id="type" label={t('Type')} options={this.importTypeOptions}/>
<Dropdown id="source" label={t('Source')} options={this.importSourceOptions}/>
}
{settingsEdit}
{mappingEdit &&
<Fieldset label={t('Mapping')} className={styles.mapping}>
{mappingEdit}
</Fieldset>
}
{mappingEdit}
<ButtonRow>

6
client/src/lists/imports/List.js
View file

@ -28,8 +28,8 @@ export default class List extends Component {
this.state = {};
const {importTypeLabels, importStatusLabels} = getImportTypes(props.t);
this.importTypeLabels = importTypeLabels;
const {importSourceLabels, importStatusLabels} = getImportTypes(props.t);
this.importSourceLabels = importSourceLabels;
this.importStatusLabels = importStatusLabels;
}
@ -46,7 +46,7 @@ export default class List extends Component {
const columns = [
{ data: 1, title: t('Name') },
{ data: 2, title: t('Description') },
{ data: 3, title: t('Source'), render: data => this.importTypeLabels[data], sortable: false, searchable: false },
{ data: 3, title: t('Source'), render: data => this.importSourceLabels[data], sortable: false, searchable: false },
{ data: 4, title: t('Status'), render: data => this.importStatusLabels[data], sortable: false, searchable: false },
{ data: 5, title: t('Last run'), render: data => moment(data).fromNow() },
{

11
client/src/lists/imports/RunStatus.js
View file

@ -31,8 +31,8 @@ export default class Status extends Component {
entity: props.entity
};
const {importTypeLabels, importStatusLabels, runStatusLabels} = getImportTypes(props.t);
this.importTypeLabels = importTypeLabels;
const {importSourceLabels, importStatusLabels, runStatusLabels} = getImportTypes(props.t);
this.importSourceLabels = importSourceLabels;
this.importStatusLabels = importStatusLabels;
this.runStatusLabels = runStatusLabels;
@ -57,7 +57,9 @@ export default class Status extends Component {
async periodicRefreshTask() {
if (runStatusInProgress(this.state.entity.status)) {
await this.refreshEntity();
this.refreshTimeoutId = setTimeout(this.refreshTimeoutHandler, 2000);
if (this.refreshTimeoutHandler) { // For some reason the task gets rescheduled if server is restarted while the page is shown. That why we have this check here.
this.refreshTimeoutId = setTimeout(this.refreshTimeoutHandler, 2000);
}
}
}
@ -67,6 +69,7 @@ export default class Status extends Component {
componentWillUnmount() {
clearTimeout(this.refreshTimeoutId);
this.refreshTimeoutHandler = null;
}
render() {
@ -79,7 +82,7 @@ export default class Status extends Component {
<Title>{t('Import Run Status')}</Title>
<AlignedRow label={t('Import name')}>{imprt.name}</AlignedRow>
<AlignedRow label={t('Import type')}>{this.importTypeLabels[imprt.type]}</AlignedRow>
<AlignedRow label={t('Import source')}>{this.importSourceLabels[imprt.source]}</AlignedRow>
<AlignedRow label={t('Run started')}>{moment(entity.created).fromNow()}</AlignedRow>
{entity.finished && <AlignedRow label={t('Run finished')}>{moment(entity.finished).fromNow()}</AlignedRow>}
<AlignedRow label={t('Run status')}>{this.runStatusLabels[entity.status]}</AlignedRow>

15
client/src/lists/imports/Status.js
View file

@ -46,8 +46,8 @@ export default class Status extends Component {
entity: props.entity
};
const {importTypeLabels, importStatusLabels, runStatusLabels} = getImportTypes(props.t);
this.importTypeLabels = importTypeLabels;
const {importSourceLabels, importStatusLabels, runStatusLabels} = getImportTypes(props.t);
this.importSourceLabels = importSourceLabels;
this.importStatusLabels = importStatusLabels;
this.runStatusLabels = runStatusLabels;
@ -71,7 +71,9 @@ export default class Status extends Component {
async periodicRefreshTask() {
// The periodic task runs all the time, so that we don't have to worry about starting/stopping it as a reaction to the buttons.
await this.refreshEntity();
this.refreshTimeoutId = setTimeout(this.refreshTimeoutHandler, 2000);
if (this.refreshTimeoutHandler) { // For some reason the task gets rescheduled if server is restarted while the page is shown. That why we have this check here.
this.refreshTimeoutId = setTimeout(this.refreshTimeoutHandler, 2000);
}
}
componentDidMount() {
@ -80,6 +82,7 @@ export default class Status extends Component {
componentWillUnmount() {
clearTimeout(this.refreshTimeoutId);
this.refreshTimeoutHandler = null;
}
async startRunAsync() {
@ -94,6 +97,7 @@ export default class Status extends Component {
}
await this.refreshEntity();
this.runsTableNode.refresh();
}
async stopRunAsync() {
@ -108,6 +112,7 @@ export default class Status extends Component {
}
await this.refreshEntity();
this.runsTableNode.refresh();
}
render() {
@ -147,7 +152,7 @@ export default class Status extends Component {
<Title>{t('Import Status')}</Title>
<AlignedRow label={t('Name')}>{entity.name}</AlignedRow>
<AlignedRow label={t('Type')}>{this.importTypeLabels[entity.type]}</AlignedRow>
<AlignedRow label={t('Source')}>{this.importSourceLabels[entity.source]}</AlignedRow>
<AlignedRow label={t('Status')}>{this.importStatusLabels[entity.status]}</AlignedRow>
{entity.error && <AlignedRow label={t('Error')}><pre>{entity.error}</pre></AlignedRow>}
@ -158,7 +163,7 @@ export default class Status extends Component {
<hr/>
<h3>{t('Import Runs')}</h3>
<Table withHeader dataUrl={`rest/import-runs-table/${this.props.list.id}/${this.props.entity.id}`} columns={columns} />
<Table ref={node => this.runsTableNode = node} withHeader dataUrl={`rest/import-runs-table/${this.props.list.id}/${this.props.entity.id}`} columns={columns} />
</div>
);
}

16
client/src/lists/imports/helpers.js
View file

@ -1,13 +1,13 @@
'use strict';
import React from 'react';
import {ImportType, ImportStatus, RunStatus} from '../../../../shared/imports';
import {ImportSource, MappingType, ImportStatus, RunStatus} from '../../../../shared/imports';
export function getImportTypes(t) {
const importTypeLabels = {
[ImportType.CSV_FILE]: t('CSV file'),
[ImportType.LIST]: t('List'),
const importSourceLabels = {
[ImportSource.CSV_FILE]: t('CSV file'),
[ImportSource.LIST]: t('List'),
};
const importStatusLabels = {
@ -30,9 +30,15 @@ export function getImportTypes(t) {
[RunStatus.FINISHED]: t('Finished')
};
const mappingTypeLabels = {
[MappingType.BASIC_SUBSCRIBE]: t('Basic import of subscribers'),
[MappingType.BASIC_UNSUBSCRIBE]: t('Unsubscribe emails'),
}
return {
importStatusLabels,
importTypeLabels,
mappingTypeLabels,
importSourceLabels,
runStatusLabels
};
}

8
lib/importer.js
View file

@ -4,7 +4,7 @@ const knex = require('./knex');
const fork = require('child_process').fork;
const log = require('npmlog');
const path = require('path');
const {ImportStatus} = require('../shared/imports');
const {ImportStatus, RunStatus} = require('../shared/imports');
let messageTid = 0;
let importerProcess;
@ -19,7 +19,13 @@ function spawn(callback) {
knex.transaction(async tx => {
await tx('imports').where('status', ImportStatus.PREP_RUNNING).update({status: ImportStatus.PREP_SCHEDULED});
await tx('imports').where('status', ImportStatus.PREP_STOPPING).update({status: ImportStatus.PREP_FAILED});
await tx('imports').where('status', ImportStatus.RUN_RUNNING).update({status: ImportStatus.RUN_SCHEDULED});
await tx('imports').where('status', ImportStatus.RUN_STOPPING).update({status: ImportStatus.RUN_FINISHED});
await tx('import_runs').where('status', RunStatus.RUNNING).update({status: RunStatus.SCHEDULED});
await tx('import_runs').where('status', RunStatus.STOPPING).update({status: RunStatus.FINISHED});
}).then(() => {
importerProcess = fork(path.join(__dirname, '..', 'services', 'importer.js'), [], {

42
models/imports.js
View file

@ -6,15 +6,15 @@ const { enforce, filterObject } = require('../lib/helpers');
const dtHelpers = require('../lib/dt-helpers');
const interoperableErrors = require('../shared/interoperable-errors');
const shares = require('./shares');
const {ImportType, ImportStatus, RunStatus, prepFinished, prepFinishedAndNotInProgress, runInProgress} = require('../shared/imports');
const {ImportSource, MappingType, ImportStatus, RunStatus, prepFinished, prepFinishedAndNotInProgress, runInProgress} = require('../shared/imports');
const fs = require('fs-extra-promise');
const path = require('path');
const importer = require('../lib/importer');
const filesDir = path.join(__dirname, '..', 'files', 'imports');
const allowedKeysCreate = new Set(['name', 'description', 'type', 'settings', 'mapping']);
const allowedKeysUpdate = new Set(['name', 'description', 'mapping']);
const allowedKeysCreate = new Set(['name', 'description', 'source', 'settings']);
const allowedKeysUpdate = new Set(['name', 'description', 'mapping_type', 'mapping']);
function hash(entity) {
return hasher.hash(filterObject(entity, allowedKeysUpdate));
@ -34,7 +34,7 @@ async function getById(context, listId, id, withSampleRow = false) {
entity.mapping = JSON.parse(entity.mapping);
if (withSampleRow && prepFinished(entity.status)) {
if (entity.type === ImportType.CSV_FILE) {
if (entity.source === ImportSource.CSV_FILE) {
const importTable = 'import_file__' + id;
const row = await tx(importTable).first();
@ -58,21 +58,28 @@ async function listDTAjax(context, listId, params) {
builder => builder
.from('imports')
.where('imports.list', listId),
[ 'imports.id', 'imports.name', 'imports.description', 'imports.type', 'imports.status', 'imports.last_run' ]
[ 'imports.id', 'imports.name', 'imports.description', 'imports.source', 'imports.status', 'imports.last_run' ]
);
});
}
async function _validateAndPreprocess(tx, listId, entity, isCreate) {
enforce(Number.isInteger(entity.type));
enforce(entity.type >= ImportType.MIN && entity.type <= ImportType.MAX, 'Invalid import type');
if (isCreate) {
enforce(Number.isInteger(entity.source));
enforce(entity.source >= ImportSource.MIN && entity.source <= ImportSource.MAX, 'Invalid import source');
entity.settings = entity.settings || {};
entity.mapping = entity.mapping || {};
entity.settings = entity.settings || {};
if (isCreate && entity.type === ImportType.CSV_FILE) {
entity.settings.csv = entity.settings.csv || {};
enforce(entity.settings.csv.delimiter && entity.settings.csv.delimiter.trim(), 'CSV delimiter must not be empty');
if (entity.source === ImportSource.CSV_FILE) {
entity.settings.csv = entity.settings.csv || {};
enforce(entity.settings.csv.delimiter && entity.settings.csv.delimiter.trim(), 'CSV delimiter must not be empty');
}
} else {
enforce(Number.isInteger(entity.mapping_type));
enforce(entity.mapping_type >= MappingType.MIN && entity.mapping_type <= MappingType.MAX, 'Invalid mapping type');
entity.mapping = entity.mapping || { settings: {}, fields: {} };
}
}
@ -83,7 +90,7 @@ async function create(context, listId, entity, files) {
await _validateAndPreprocess(tx, listId, entity, true);
if (entity.type === ImportType.CSV_FILE) {
if (entity.source === ImportSource.CSV_FILE) {
enforce(files.csvFile, 'File must be included');
const csvFile = files.csvFile[0];
const filePath = path.join(filesDir, csvFile.filename);
@ -98,11 +105,12 @@ async function create(context, listId, entity, files) {
entity.status = ImportStatus.PREP_SCHEDULED;
}
const filteredEntity = filterObject(entity, allowedKeysCreate);
filteredEntity.list = listId;
filteredEntity.settings = JSON.stringify(filteredEntity.settings);
filteredEntity.mapping = JSON.stringify(filteredEntity.mapping);
filteredEntity.mapping_type = MappingType.BASIC_SUBSCRIBE; // This is not set in the create form. It can be changed in the update form.
filteredEntity.mapping = JSON.stringify({});
const ids = await tx('imports').insert(filteredEntity);
const id = ids[0];
@ -130,13 +138,11 @@ async function updateWithConsistencyCheck(context, listId, entity) {
throw new interoperableErrors.ChangedError();
}
enforce(prepFinishedAndNotInProgress(existing.status), 'Cannot save updates until preparation or run is finished');
enforce(prepFinished(existing.status), 'Cannot save updates until preparation is finished');
enforce(entity.type === existing.type, 'Import type cannot be changed');
await _validateAndPreprocess(tx, listId, entity, false);
const filteredEntity = filterObject(entity, allowedKeysUpdate);
filteredEntity.list = listId;
filteredEntity.mapping = JSON.stringify(filteredEntity.mapping);
await tx('imports').where({list: listId, id: entity.id}).update(filteredEntity);

2
obsolete/routes/lists-legacy.js
View file

@ -297,7 +297,7 @@ router.get('/view/:id', passport.csrfProtection, (req, res) => {
list.imports = imports.map((entry, i) => {
entry.index = i + 1;
entry.importType = entry.type === 0 ? _('Subscribe') : (entry.type === 1 ? _('Force Subscribe') : _('Unsubscribe'));
entry.importSource = entry.type === 0 ? _('Subscribe') : (entry.type === 1 ? _('Force Subscribe') : _('Unsubscribe'));
switch (entry.status) {
case 0:
entry.importStatus = _('Initializing');

2
obsolete/views/lists/view.hbs
View file

@ -164,7 +164,7 @@
</td>
<td>
{{importType}}
{{importSource}}
</td>
<td>
{{new}}

37
services/importer.js
View file

@ -4,9 +4,10 @@ const knex = require('../lib/knex');
const path = require('path');
const log = require('npmlog');
const fsExtra = require('fs-extra-promise');
const {ImportType, ImportStatus, RunStatus} = require('../shared/imports');
const {ImportSource, MappingType, ImportStatus, RunStatus} = require('../shared/imports');
const imports = require('../models/imports');
const { Writable } = require('stream');
const { enforce } = require('../lib/helpers');
const csvparse = require('csv-parse');
const fs = require('fs');
@ -50,7 +51,6 @@ function prepareCsv(impt) {
};
const processRows = async (chunks) => {
console.log('process row');
let insertBatch = [];
for (const chunkEntry of chunks) {
const record = chunkEntry.chunk;
@ -126,6 +126,29 @@ function prepareCsv(impt) {
inputStream.pipe(parser);
}
async function basicSubscribe(impt) {
let imptRun;
while (imptRun = await knex('import_runs').where('import', impt.id).whereIn('status', [RunStatus.SCHEDULED]).orderBy('created', 'asc').first()) {
await knex('import_runs').where('id', imptRun.id).update({
status: RunStatus.RUNNING
});
await knex('import_runs').where('id', imptRun.id).update({
status: RunStatus.FINISHED
});
}
await knex('imports').where('id', impt.id).update({
status: ImportStatus.RUN_FINISHED
});
}
async function basicUnsubscribe(impt) {
// FIXME
}
async function getTask() {
return await knex.transaction(async tx => {
const impt = await tx('imports').whereIn('status', [ImportStatus.PREP_SCHEDULED, ImportStatus.RUN_SCHEDULED]).orderBy('created', 'asc').first();
@ -133,9 +156,17 @@ async function getTask() {
if (impt) {
impt.settings = JSON.parse(impt.settings);
if (impt.type === ImportType.CSV_FILE && impt.status === ImportStatus.PREP_SCHEDULED) {
if (impt.source === ImportSource.CSV_FILE && impt.status === ImportStatus.PREP_SCHEDULED) {
await tx('imports').where('id', impt.id).update('status', ImportStatus.PREP_RUNNING);
return () => prepareCsv(impt);
} else if (impt.status === ImportStatus.RUN_SCHEDULED && impt.settings.mappingType === MappingType.BASIC_SUBSCRIBE) {
await tx('imports').where('id', impt.id).update('status', ImportStatus.RUN_RUNNING);
return () => basicSubscribe(impt);
} else if (impt.status === ImportStatus.RUN_SCHEDULED && impt.settings.mappingType === MappingType.BASIC_UNSUBSCRIBE) {
await tx('imports').where('id', impt.id).update('status', ImportStatus.RUN_RUNNING);
return () => basicUnsubscribe(impt);
}
} else {

3
setup/knex/migrations/20170506102634_v1_to_v2.js
View file

@ -1025,9 +1025,10 @@ async function migrateImporter(knex) {
table.string('name');
table.text('description');
table.integer('list').unsigned().references('lists.id');
table.integer('type').unsigned().notNullable();
table.integer('source').unsigned().notNullable();
table.integer('status').unsigned().notNullable();
table.text('settings', 'longtext');
table.integer('mapping_type').unsigned().notNullable();
table.text('mapping', 'longtext');
table.timestamp('last_run');
table.text('error');

14
shared/imports.js
View file

@ -1,6 +1,6 @@
'use strict';
const ImportType = {
const ImportSource = {
MIN: 0,
CSV_FILE: 0,
@ -9,6 +9,15 @@ const ImportType = {
MAX: 1
};
const MappingType = {
MIN: 0,
BASIC_SUBSCRIBE: 0,
BASIC_UNSUBSCRIBE: 1,
MAX: 1
};
const ImportStatus = {
PREP_SCHEDULED: 0,
PREP_RUNNING: 1,
@ -59,7 +68,8 @@ function runStatusInProgress(status) {
}
module.exports = {
ImportType,
ImportSource,
MappingType,
ImportStatus,
RunStatus,
prepInProgress,