UI for basic import and preparation phase of CSV.

app-builder.js

@@ -40,6 +40,7 @@ const listsRest = require('./routes/rest/lists');
 const formsRest = require('./routes/rest/forms');
 const fieldsRest = require('./routes/rest/fields');
 const importsRest = require('./routes/rest/imports');
+const importRunsRest = require('./routes/rest/import-runs');
 const sharesRest = require('./routes/rest/shares');
 const segmentsRest = require('./routes/rest/segments');
 const subscriptionsRest = require('./routes/rest/subscriptions');
@@ -283,6 +284,7 @@ function createApp(trusted) {
     app.use('/rest', formsRest);
     app.use('/rest', fieldsRest);
     app.use('/rest', importsRest);
+    app.use('/rest', importRunsRest);
     app.use('/rest', sharesRest);
     app.use('/rest', segmentsRest);
     app.use('/rest', subscriptionsRest);

client/src/lib/form.js

@@ -112,7 +112,8 @@ class Fieldset extends Component {
         id: PropTypes.string,
         label: PropTypes.string,
         help: PropTypes.oneOfType([PropTypes.string, PropTypes.object]),
-        flat: PropTypes.bool
+        flat: PropTypes.bool,
+        className: PropTypes.string
     }
 
     static contextTypes = {
@@ -125,7 +126,10 @@ class Fieldset extends Component {
         const id = this.props.id;
         const htmlId = 'form_' + id;
 
-        const className = id ? owner.addFormValidationClass('', id) : null;
+        let className = id ? owner.addFormValidationClass('', id) : null;
+        if (this.props.className) {
+            className = (className || '') + ' ' + this.props.className;
+        }
 
         let helpBlock = null;
         if (this.props.help) {
@@ -154,7 +158,17 @@ class Fieldset extends Component {
 }
 
 function wrapInput(id, htmlId, owner, format, rightContainerClass, label, help, input) {
-    const className = id ? owner.addFormValidationClass('form-group', id) : 'form-group';
+    // wrapInput may be used also outside forms to make a kind of fake read-only forms
+    let className;
+    if (owner) {
+        if (id) {
+            className = owner.addFormValidationClass('form-group', id);
+        } else {
+            className = 'form-group';
+        }
+    } else {
+        className = 'row ' + styles.staticFormGroup;
+    }
 
     let colLeft = '';
     let colRight = '';
@@ -580,7 +594,6 @@ class Dropdown extends Component {
         label: PropTypes.string.isRequired,
         help: PropTypes.oneOfType([PropTypes.string, PropTypes.object]),
         options: PropTypes.array,
-        optGroups: PropTypes.array,
         className: PropTypes.string,
         format: PropTypes.string
     }
@@ -595,16 +608,20 @@ class Dropdown extends Component {
         const owner = this.context.formStateOwner;
         const id = this.props.id;
         const htmlId = 'form_' + id;
-        let options = [];
+        const options = [];
 
         if (this.props.options) {
-            options = props.options.map(option => <option key={option.key} value={option.key}>{option.label}</option>);
+            for (const optOrGrp of props.options) {
-        } else if (this.props.optGroups) {
+                if (optOrGrp.options) {
-            options = props.optGroups.map(optGroup =>
+                    options.push(
-                <optgroup key={optGroup.key} label={optGroup.label}>
+                        <optgroup key={optOrGrp.key} label={optOrGrp.label}>
-                    {optGroup.options.map(option => <option key={option.key} value={option.key}>{option.label}</option>)}
+                            {optOrGrp.options.map(opt => <option key={opt.key} value={opt.key}>{opt.label}</option>)}
-                </optgroup>
+                        </optgroup>
-            );
+                    )
+                } else {
+                    options.push(<option key={optOrGrp.key} value={optOrGrp.key}>{optOrGrp.label}</option>)
+                }
+            }
         }
 
         let className = 'form-control';
@@ -629,7 +646,7 @@ class AlignedRow extends Component {
     }
 
     static contextTypes = {
-        formStateOwner: PropTypes.object.isRequired
+        formStateOwner: PropTypes.object // AlignedRow may be used also outside forms to make a kind of fake read-only forms
     }
 
     static defaultProps = {

client/src/lib/styles.scss

@@ -27,6 +27,10 @@
   }
 }
 
+.staticFormGroup {
+  margin-bottom: 15px;
+}
+
 .dayPickerWrapper {
   text-align: right;
 }

client/src/lib/table.js

@@ -347,6 +347,11 @@ class Table extends Component {
         this.fetchAndNotifySelectionData();
     }
 
+    componentWillUnmount() {
+        clearInterval(this.refreshIntervalId);
+        clearTimeout(this.refreshTimeoutId);
+    }
+
     async notifySelection(eventCallback, newSelectionMap) {
         if (eventCallback) {
             const selPairs = Array.from(newSelectionMap).sort((l, r) => l[0] - r[0]);

client/src/lists/List.js

@@ -95,7 +95,7 @@ export default class List extends Component {
 
                     if (perms.includes('viewImports')) {
                         actions.push({
-                            label: <Icon icon="arrow-down" title={t('Imports')}/>,
+                            label: <Icon icon="sort" title={t('Imports')}/>,
                             link: `/lists/${data[0]}/imports`
                         });
                     }

client/src/lists/forms/CUD.js

@@ -427,7 +427,7 @@ export default class CUD extends Component {
 
                     { selectedTemplate &&
                         <Fieldset label={t('Templates')}>
-                            <Dropdown id="selectedTemplate" label={t('Edit')} optGroups={templateOptGroups} help={this.templateSettings[selectedTemplate].help}/>
+                            <Dropdown id="selectedTemplate" label={t('Edit')} options={templateOptGroups} help={this.templateSettings[selectedTemplate].help}/>
                             <ACEEditor id={selectedTemplate} height="500px" mode={this.templateSettings[selectedTemplate].mode}/>
                         </Fieldset>
                     }

client/src/lists/imports/CUD.js

@@ -10,10 +10,10 @@ import {
     withPageHelpers
 } from '../../lib/page';
 import {
-    AlignedRow,
     Button,
     ButtonRow,
     Dropdown,
+    Fieldset,
     Form,
     FormSendMethod,
     InputField,
@@ -21,11 +21,33 @@ import {
     TextArea,
     withForm
 } from '../../lib/form';
-import {withErrorHandling} from '../../lib/error-handling';
+import {
+    withAsyncErrorHandler,
+    withErrorHandling
+} from '../../lib/error-handling';
 import {DeleteModalDialog} from "../../lib/modals";
 import {getImportTypes} from './helpers';
-import styles from "../../lib/styles.scss";
+import {
-import {ImportType} from '../../../../shared/imports';
+    ImportType,
+    inProgress,
+    prepInProgress,
+    runInProgress
+} from '../../../../shared/imports';
+import axios from "../../lib/axios";
+import {getUrl} from "../../lib/urls";
+import styles from "../styles.scss";
+
+
+function truncate(str, len, ending = '...') {
+    str = str.trim();
+
+    if (str.length > len) {
+        return str.substring(0, len - ending.length) + ending;
+    } else {
+        return str;
+    }
+}
+
 
 @translate()
 @withForm
@@ -47,26 +69,54 @@ export default class CUD extends Component {
             // {key: ImportType.LIST, label: importTypeLabels[ImportType.LIST]}
         ];
 
+        this.refreshTimeoutHandler = ::this.refreshEntity;
+        this.refreshTimeoutId = 0;
+
         this.initForm();
     }
 
     static propTypes = {
         action: PropTypes.string.isRequired,
         list: PropTypes.object,
+        fieldsGrouped: PropTypes.array,
         entity: PropTypes.object
     }
 
+    initFromEntity(entity) {
+        this.getFormValuesFromEntity(entity, data => {
+            data.settings = data.settings || {};
+            const mapping = data.mapping || {};
+
+            if (data.type === ImportType.CSV_FILE) {
+                data.csvFileName = data.settings.csv.originalname;
+                data.csvDelimiter = data.settings.csv.delimiter;
+            }
+
+            for (const field of this.props.fieldsGrouped) {
+                if (field.column) {
+                    const colMapping = mapping[field.column] || {};
+                    data['mapping_' + field.column + '_column'] = colMapping.column || '';
+                } else {
+                    for (const option of field.settings.options) {
+                        const col = field.groupedOptions[option.key].column;
+                        const colMapping = mapping[col] || {};
+                        data['mapping_' + col + '_column'] = colMapping.column || '';
+                    }
+                }
+            }
+
+            const emailMapping = mapping.email || {};
+            data.mapping_email_column = emailMapping.column || '';
+        });
+
+        if (inProgress(entity.status)) {
+            this.refreshTimeoutId = setTimeout(this.refreshTimeoutHandler, 1000);
+        }
+    }
+
     componentDidMount() {
         if (this.props.entity) {
-            this.getFormValuesFromEntity(this.props.entity, data => {
+            this.initFromEntity(this.props.entity);
-                data.settings = data.settings || {};
-
-                if (data.type === ImportType.CSV_FILE) {
-                    data.csvFileName = data.settings.csv.originalname;
-                    data.csvDelimiter = data.settings.csv.delimiter;
-                }
-            });
-
         } else {
             this.populateFormValues({
                 name: '',
@@ -78,10 +128,21 @@ export default class CUD extends Component {
         }
     }
 
+    componentWillUnmount() {
+        clearTimeout(this.refreshTimeoutId);
+    }
+
+    @withAsyncErrorHandler
+    async refreshEntity() {
+        const resp = await axios.get(getUrl(`rest/imports/${this.props.list.id}/${this.props.entity.id}`));
+        this.initFromEntity(resp.data);
+    }
+
     localValidateFormValues(state) {
         const t = this.props.t;
         const isEdit = !!this.props.entity;
         const type = Number.parseInt(state.getIn(['type', 'value']));
+        const status = this.getFormValue('status');
 
         for (const key of state.keys()) {
             state.setIn([key, 'error'], null);
@@ -100,12 +161,20 @@ export default class CUD extends Component {
                 state.setIn(['csvDelimiter', 'error'], t('CSV delimiter must not be empty'));
             }
         }
+
+        if (isEdit) {
+            if (!state.getIn(['mapping_email_column', 'value'])) {
+                state.setIn(['mapping_email_column', 'error'], t('Email mapping has to be provided'));
+            }
+        }
     }
 
     async submitHandler() {
         const t = this.props.t;
         const isEdit = !!this.props.entity;
 
+        const type = Number.parseInt(this.getFormValue('type'));
+
         let sendMethod, url;
         if (this.props.entity) {
             sendMethod = FormSendMethod.PUT;
@@ -119,7 +188,7 @@ export default class CUD extends Component {
             this.disableForm();
             this.setFormStatusMessage('info', t('Saving ...'));
 
-            const submitSuccessful = await this.validateAndSendFormValuesToURL(sendMethod, url, data => {
+            const submitResponse = await this.validateAndSendFormValuesToURL(sendMethod, url, data => {
                 data.type = Number.parseInt(data.type);
                 data.settings = {};
 
@@ -128,18 +197,52 @@ export default class CUD extends Component {
                     data.settings.csv = {};
                     formData.append('csvFile', this.csvFile.files[0]);
                     data.settings.csv.delimiter = data.csvDelimiter.trim();
-
-                    delete data.csvFile;
-                    delete data.csvDelimiter;
                 }
 
+                if (isEdit) {
+                    const mapping = {};
+                    for (const field of this.props.fieldsGrouped) {
+                        if (field.column) {
+                            mapping[field.column] = {
+                                column: data['mapping_' + field.column + '_column']
+                            };
+
+                            delete data['mapping_' + field.column + '_column'];
+                        } else {
+                            for (const option of field.settings.options) {
+                                const col = field.groupedOptions[option.key].column;
+                                mapping[col] = {
+                                    column: data['mapping_' + col + '_column']
+                                };
+
+                                delete data['mapping_' + col + '_column'];
+                            }
+                        }
+                    }
+
+                    mapping.email = {
+                        column: data.mapping_email_column
+                    };
+
+                    data.mapping = mapping;
+                }
+
+                delete data.csvFile;
+                delete data.csvDelimiter;
+                delete data.sampleRow;
+
                 formData.append('entity', JSON.stringify(data));
 
                 return formData;
             });
 
-            if (submitSuccessful) {
+            if (submitResponse) {
-                this.navigateToWithFlashMessage(`/lists/${this.props.list.id}/imports`, 'success', t('Import saved'));
+                if (!isEdit && type === ImportType.CSV_FILE) {
+                    this.navigateTo(`/lists/${this.props.list.id}/imports/${submitResponse}/edit`);
+                } else {
+                    this.navigateToWithFlashMessage(`/lists/${this.props.list.id}/imports/${this.props.entity.id}/status`, 'success', t('Import saved'));
+                }
+
             } else {
                 this.enableForm();
                 this.setFormStatusMessage('warning', t('There are errors in the form. Please fix them and submit again.'));
@@ -158,17 +261,19 @@ export default class CUD extends Component {
         const isEdit = !!this.props.entity;
 
         const type = Number.parseInt(this.getFormValue('type'));
+        const status = this.getFormValue('status');
+        const settings = this.getFormValue('settings');
 
-        let settings = null;
+        let settingsEdit = null;
         if (type === ImportType.CSV_FILE) {
             if (isEdit) {
-                settings =
+                settingsEdit =
                     <div>
                         <StaticField id="csvFileName" className={styles.formDisabled} label={t('File')}>{this.getFormValue('csvFileName')}</StaticField>
                         <StaticField id="csvDelimiter" className={styles.formDisabled} label={t('Delimiter')}>{this.getFormValue('csvDelimiter')}</StaticField>
                     </div>;
             } else {
-                settings =
+                settingsEdit =
                     <div>
                         <StaticField withValidation id="csvFileName" label={t('File')}><input ref={node => this.csvFile = node} type="file" onChange={::this.onFileSelected}/></StaticField>
                         <InputField id="csvDelimiter" label={t('Delimiter')}/>
@@ -176,6 +281,57 @@ export default class CUD extends Component {
             }
         }
 
+        let mappingEdit;
+        if (isEdit) {
+            if (prepInProgress(status)) {
+                mappingEdit = <div>{t('Preparation in progress. Please wait till it is done or visit this page later.')}</div>;
+            } else if (runInProgress(status)) {
+                    mappingEdit = <div>{t('Run in progress. Please wait till it is done or visit this page later.')}</div>;
+            } else {
+                const sampleRow = this.getFormValue('sampleRow');
+                const sourceOpts = [];
+                sourceOpts.push({key: '', label: t('–– Select ––')});
+                if (type === ImportType.CSV_FILE) {
+                    for (const csvCol of settings.csv.columns) {
+                        let help = '';
+                        if (sampleRow) {
+                            help = ' (' + t('e.g.:', {keySeparator: '>', nsSeparator: '|'}) + ' ' + truncate(sampleRow[csvCol.column], 50) + ')';
+                        }
+
+                        sourceOpts.push({key: csvCol.column, label: csvCol.name + help});
+                    }
+                }
+
+                const mappingRows = [
+                    <Dropdown key="email" id="mapping_email_column" label={t('Email')} options={sourceOpts}/>
+                ];
+
+                for (const field of this.props.fieldsGrouped) {
+                    if (field.column) {
+                        mappingRows.push(
+                            <Dropdown key={field.column} id={'mapping_' + field.column + '_column'} label={field.name} options={sourceOpts}/>
+                        );
+                    } else {
+                        for (const option of field.settings.options) {
+                            const col = field.groupedOptions[option.key].column;
+                            mappingRows.push(
+                                <Dropdown key={col} id={'mapping_' + col + '_column'} label={field.groupedOptions[option.key].name} options={sourceOpts}/>
+                            );
+                        }
+                    }
+                }
+
+                mappingEdit = mappingRows;
+            }
+        }
+
+        let saveButtonLabel;
+        if (!isEdit && type === ImportType.CSV_FILE) {
+            saveButtonLabel = t('Save and edit mapping');
+        } else {
+            saveButtonLabel = t('Save');
+        }
+
         return (
             <div>
                 {isEdit &&
@@ -201,10 +357,17 @@ export default class CUD extends Component {
                         <Dropdown id="type" label={t('Type')} options={this.importTypeOptions}/>
                     }
 
-                    {settings}
+                    {settingsEdit}
+
+                    {mappingEdit &&
+                        <Fieldset label={t('Mapping')} className={styles.mapping}>
+                            {mappingEdit}
+                        </Fieldset>
+                    }
+
 
                     <ButtonRow>
-                        <Button type="submit" className="btn-primary" icon="ok" label={t('Save')}/>
+                        <Button type="submit" className="btn-primary" icon="ok" label={saveButtonLabel}/>
                         {isEdit && <NavButton className="btn-danger" icon="remove" label={t('Delete')} linkTo={`/lists/${this.props.list.id}/imports/${this.props.entity.id}/delete`}/>}
                     </ButtonRow>
                 </Form>

client/src/lists/imports/List.js

@@ -1,15 +1,22 @@
 'use strict';
 
-import React, { Component } from 'react';
+import React, {Component} from 'react';
 import PropTypes from 'prop-types';
-import { translate } from 'react-i18next';
+import {translate} from 'react-i18next';
-import {requiresAuthenticatedUser, withPageHelpers, Title, Toolbar, NavButton} from '../../lib/page';
+import {
-import { withErrorHandling } from '../../lib/error-handling';
+    NavButton,
-import { Table } from '../../lib/table';
+    requiresAuthenticatedUser,
-import { getImportTypes } from './helpers';
+    Title,
+    Toolbar,
+    withPageHelpers
+} from '../../lib/page';
+import {withErrorHandling} from '../../lib/error-handling';
+import {Table} from '../../lib/table';
+import {getImportTypes} from './helpers';
 import {Icon} from "../../lib/bootstrap-components";
 import mailtrainConfig from 'mailtrainConfig';
 import moment from "moment";
+import {inProgress} from '../../../../shared/imports';
 
 @translate()
 @withPageHelpers
@@ -39,12 +46,19 @@ export default class List extends Component {
         const columns = [
             { data: 1, title: t('Name') },
             { data: 2, title: t('Description') },
-            { data: 3, title: t('Source'), render: data => this.importTypeLabels[data].label, sortable: false, searchable: false },
+            { data: 3, title: t('Source'), render: data => this.importTypeLabels[data], sortable: false, searchable: false },
-            { data: 4, title: t('Status'), render: data => this.importStatusLabels[data].label, sortable: false, searchable: false },
+            { data: 4, title: t('Status'), render: data => this.importStatusLabels[data], sortable: false, searchable: false },
             { data: 5, title: t('Last run'), render: data => moment(data).fromNow() },
             {
                 actions: data => {
                     const actions = [];
+                    const status = data[4];
+                    
+                    let refreshTimeout;
+                    
+                    if (inProgress(status)) {
+                        refreshTimeout = 1000;
+                    }
 
                     if (mailtrainConfig.globalPermissions.includes('setupAutomation') && this.props.list.permissions.includes('manageImports')) {
                         actions.push({
@@ -53,7 +67,12 @@ export default class List extends Component {
                         });
                     }
 
-                    return actions;
+                    actions.push({
+                        label: <Icon icon="eye-open" title={t('Detailed status')}/>,
+                        link: `/lists/${this.props.list.id}/imports/${data[0]}/status`
+                    });
+
+                    return { refreshTimeout, actions };
                 }
             }
         ];

client/src/lists/imports/RunStatus.js

@@ -0,0 +1,93 @@
+'use strict';
+
+import React, {Component} from 'react';
+import PropTypes from 'prop-types';
+import {translate} from 'react-i18next';
+import {
+    requiresAuthenticatedUser,
+    Title,
+    withPageHelpers
+} from '../../lib/page';
+import {AlignedRow} from '../../lib/form';
+import {
+    withAsyncErrorHandler,
+    withErrorHandling
+} from '../../lib/error-handling';
+import {getImportTypes} from './helpers';
+import axios from "../../lib/axios";
+import {getUrl} from "../../lib/urls";
+import moment from "moment";
+import {runStatusInProgress} from "../../../../shared/imports";
+
+@translate()
+@withPageHelpers
+@withErrorHandling
+@requiresAuthenticatedUser
+export default class Status extends Component {
+    constructor(props) {
+        super(props);
+
+        this.state = {
+            entity: props.entity
+        };
+
+        const {importTypeLabels, importStatusLabels, runStatusLabels} = getImportTypes(props.t);
+        this.importTypeLabels = importTypeLabels;
+        this.importStatusLabels = importStatusLabels;
+        this.runStatusLabels = runStatusLabels;
+
+        this.refreshTimeoutHandler = ::this.periodicRefreshTask;
+        this.refreshTimeoutId = 0;
+    }
+
+    static propTypes = {
+        entity: PropTypes.object,
+        imprt: PropTypes.object,
+        list: PropTypes.object
+    }
+
+    @withAsyncErrorHandler
+    async refreshEntity() {
+        const resp = await axios.get(getUrl(`rest/import-runs/${this.props.list.id}/${this.props.imprt.id}/${this.props.entity.id}`));
+        this.setState({
+            entity: resp.data
+        });
+    }
+
+    async periodicRefreshTask() {
+        if (runStatusInProgress(this.state.entity.status)) {
+            await this.refreshEntity();
+            this.refreshTimeoutId = setTimeout(this.refreshTimeoutHandler, 2000);
+        }
+    }
+
+    componentDidMount() {
+        this.periodicRefreshTask();
+    }
+
+    componentWillUnmount() {
+        clearTimeout(this.refreshTimeoutId);
+    }
+
+    render() {
+        const t = this.props.t;
+        const entity = this.state.entity;
+        const imprt = this.props.imprt;
+
+       return (
+            <div>
+                <Title>{t('Import Run Status')}</Title>
+
+                <AlignedRow label={t('Import name')}>{imprt.name}</AlignedRow>
+                <AlignedRow label={t('Import type')}>{this.importTypeLabels[imprt.type]}</AlignedRow>
+                <AlignedRow label={t('Run started')}>{moment(entity.created).fromNow()}</AlignedRow>
+                {entity.finished && <AlignedRow label={t('Run finished')}>{moment(entity.finished).fromNow()}</AlignedRow>}
+                <AlignedRow label={t('Run status')}>{this.runStatusLabels[entity.status]}</AlignedRow>
+                <AlignedRow label={t('Processed entries')}>{entity.processed}</AlignedRow>
+                <AlignedRow label={t('New entries')}>{entity.new}</AlignedRow>
+                <AlignedRow label={t('Failed entries')}>{entity.failed}</AlignedRow>
+                {entity.error && <AlignedRow label={t('Error')}><pre>{entity.error}</pre></AlignedRow>}
+            </div>
+        );
+    }
+}

client/src/lists/imports/Status.js

@@ -0,0 +1,165 @@
+'use strict';
+
+import React, {Component} from 'react';
+import PropTypes from 'prop-types';
+import {translate} from 'react-i18next';
+import {
+    requiresAuthenticatedUser,
+    Title,
+    withPageHelpers
+} from '../../lib/page';
+import {
+    AlignedRow,
+    ButtonRow,
+    Fieldset
+} from '../../lib/form';
+import {
+    withAsyncErrorHandler,
+    withErrorHandling
+} from '../../lib/error-handling';
+import {getImportTypes} from './helpers';
+import {
+    prepFinishedAndNotInProgress,
+    runInProgress,
+    RunStatus,
+    runStatusInProgress
+} from '../../../../shared/imports';
+import {Table} from "../../lib/table";
+import {
+    Button,
+    Icon
+} from "../../lib/bootstrap-components";
+import axios from "../../lib/axios";
+import {getUrl} from "../../lib/urls";
+import moment from "moment";
+import interoperableErrors from '../../../../shared/interoperable-errors';
+
+@translate()
+@withPageHelpers
+@withErrorHandling
+@requiresAuthenticatedUser
+export default class Status extends Component {
+    constructor(props) {
+        super(props);
+
+        this.state = {
+            entity: props.entity
+        };
+
+        const {importTypeLabels, importStatusLabels, runStatusLabels} = getImportTypes(props.t);
+        this.importTypeLabels = importTypeLabels;
+        this.importStatusLabels = importStatusLabels;
+        this.runStatusLabels = runStatusLabels;
+
+        this.refreshTimeoutHandler = ::this.periodicRefreshTask;
+        this.refreshTimeoutId = 0;
+    }
+
+    static propTypes = {
+        entity: PropTypes.object,
+        list: PropTypes.object
+    }
+
+    @withAsyncErrorHandler
+    async refreshEntity() {
+        const resp = await axios.get(getUrl(`rest/imports/${this.props.list.id}/${this.props.entity.id}`));
+        this.setState({
+            entity: resp.data
+        });
+    }
+
+    async periodicRefreshTask() {
+        // The periodic task runs all the time, so that we don't have to worry about starting/stopping it as a reaction to the buttons.
+        await this.refreshEntity();
+        this.refreshTimeoutId = setTimeout(this.refreshTimeoutHandler, 2000);
+    }
+
+    componentDidMount() {
+        this.periodicRefreshTask();
+    }
+
+    componentWillUnmount() {
+        clearTimeout(this.refreshTimeoutId);
+    }
+
+    async startRunAsync() {
+        try {
+            await axios.post(getUrl(`rest/import-start/${this.props.list.id}/${this.props.entity.id}`));
+        } catch (err) {
+            if (err instanceof interoperableErrors.InvalidStateError) {
+                // Just mask the fact that it's not possible to start anything and refresh instead.
+            } else {
+                throw err;
+            }
+        }
+
+        await this.refreshEntity();
+    }
+
+    async stopRunAsync() {
+        try {
+            await axios.post(getUrl(`rest/import-stop/${this.props.list.id}/${this.props.entity.id}`));
+        } catch (err) {
+            if (err instanceof interoperableErrors.InvalidStateError) {
+                // Just mask the fact that it's not possible to stop anything and refresh instead.
+            } else {
+                throw err;
+            }
+        }
+
+        await this.refreshEntity();
+    }
+
+    render() {
+        const t = this.props.t;
+        const entity = this.state.entity;
+
+        const columns = [
+            { data: 1, title: t('Started'), render: data => moment(data).fromNow() },
+            { data: 2, title: t('Finished'), render: data => data ? moment(data).fromNow() : '' },
+            { data: 3, title: t('Status'), render: data => this.runStatusLabels[data], sortable: false, searchable: false },
+            { data: 4, title: t('Processed') },
+            { data: 5, title: t('New') },
+            { data: 6, title: t('Failed') },
+            {
+                actions: data => {
+                    const actions = [];
+                    const status = data[3];
+
+                    let refreshTimeout;
+
+                    if (runStatusInProgress(status)) {
+                        refreshTimeout = 1000;
+                    }
+
+                    actions.push({
+                        label: <Icon icon="eye-open" title={t('Run status')}/>,
+                        link: `/lists/${this.props.list.id}/imports/${this.props.entity.id}/status/${data[0]}`
+                    });
+
+                    return { refreshTimeout, actions };
+                }
+            }
+        ];
+
+        return (
+            <div>
+                <Title>{t('Import Status')}</Title>
+
+                <AlignedRow label={t('Name')}>{entity.name}</AlignedRow>
+                <AlignedRow label={t('Type')}>{this.importTypeLabels[entity.type]}</AlignedRow>
+                <AlignedRow label={t('Status')}>{this.importStatusLabels[entity.status]}</AlignedRow>
+                {entity.error && <AlignedRow label={t('Error')}><pre>{entity.error}</pre></AlignedRow>}
+
+                <ButtonRow label={t('Actions')}>
+                    {prepFinishedAndNotInProgress(entity.status) && <Button className="btn-primary" icon="play" label={t('Start')} onClickAsync={::this.startRunAsync}/>}
+                    {runInProgress(entity.status) && <Button className="btn-primary" icon="stop" label={t('Stop')} onClickAsync={::this.stopRunAsync}/>}
+                </ButtonRow>
+
+                <hr/>
+                <h3>{t('Import Runs')}</h3>
+                <Table withHeader dataUrl={`rest/import-runs-table/${this.props.list.id}/${this.props.entity.id}`} columns={columns} />
+            </div>
+        );
+    }
+}

client/src/lists/imports/helpers.js

@@ -1,7 +1,7 @@
 'use strict';
 
 import React from 'react';
-import {ImportType, ImportStatus} from '../../../../shared/imports';
+import {ImportType, ImportStatus, RunStatus} from '../../../../shared/imports';
 
 export function getImportTypes(t) {
 
@@ -13,17 +13,27 @@ export function getImportTypes(t) {
     const importStatusLabels = {
         [ImportStatus.PREP_SCHEDULED]: t('Created'),
         [ImportStatus.PREP_RUNNING]: t('Preparing'),
+        [ImportStatus.PREP_STOPPING]: t('Stopping'),
         [ImportStatus.PREP_FINISHED]: t('Ready'),
         [ImportStatus.PREP_FAILED]: t('Preparation failed'),
         [ImportStatus.RUN_SCHEDULED]: t('Scheduled'),
         [ImportStatus.RUN_RUNNING]: t('Running'),
+        [ImportStatus.RUN_STOPPING]: t('Stopping'),
         [ImportStatus.RUN_FINISHED]: t('Finished'),
         [ImportStatus.RUN_FAILED]: t('Failed')
     };
 
+    const runStatusLabels = {
+        [RunStatus.SCHEDULED]: t('Starting'),
+        [RunStatus.RUNNING]: t('Running'),
+        [RunStatus.STOPPING]: t('Stopping'),
+        [RunStatus.FINISHED]: t('Finished')
+    };
+
     return {
         importStatusLabels,
-        importTypeLabels
+        importTypeLabels,
+        runStatusLabels
     };
 }
 

client/src/lists/root.js

@@ -14,6 +14,8 @@ import SegmentsList from './segments/List';
 import SegmentsCUD from './segments/CUD';
 import ImportsList from './imports/List';
 import ImportsCUD from './imports/CUD';
+import ImportsStatus from './imports/Status';
+import ImportRunsStatus from './imports/RunStatus';
 import Share from '../shares/Share';
 import TriggersList from './TriggersList';
 
@@ -140,18 +142,36 @@ function getMenus(t) {
                                     resolve: {
                                         import: params => `rest/imports/${params.listId}/${params.importId}`,
                                     },
-                                    link: params => `/lists/${params.listId}/imports/${params.importId}/edit`,
+                                    link: params => `/lists/${params.listId}/imports/${params.importId}/status`,
                                     navs: {
                                         ':action(edit|delete)': {
                                             title: t('Edit'),
+                                            resolve: {
+                                                fieldsGrouped: params => `rest/fields-grouped/${params.listId}`
+                                            },
                                             link: params => `/lists/${params.listId}/imports/${params.importId}/edit`,
-                                            panelRender: props => <ImportsCUD action={props.match.params.action} entity={props.resolved.import} list={props.resolved.list} imports={props.resolved.imports} />
+                                            panelRender: props => <ImportsCUD action={props.match.params.action} entity={props.resolved.import} list={props.resolved.list} fieldsGrouped={props.resolved.fieldsGrouped}/>
+                                        },
+                                        'status': {
+                                            title: t('Status'),
+                                            link: params => `/lists/${params.listId}/imports/${params.importId}/status`,
+                                            panelRender: props => <ImportsStatus entity={props.resolved.import} list={props.resolved.list} />,
+                                            children: {
+                                                ':importRunId([0-9]+)': {
+                                                    title: resolved => t('Run'),
+                                                    resolve: {
+                                                        importRun: params => `rest/import-runs/${params.listId}/${params.importId}/${params.importRunId}`,
+                                                    },
+                                                    link: params => `/lists/${params.listId}/imports/${params.importId}/status/${params.importRunId}`,
+                                                    panelRender: props => <ImportRunsStatus entity={props.resolved.importRun} imprt={props.resolved.import} list={props.resolved.list} />
+                                                }
+                                            }
                                         }
                                     }
                                 },
                                 create: {
                                     title: t('Create'),
-                                    panelRender: props => <ImportsCUD action="create" list={props.resolved.list} imports={props.resolved.imports} />
+                                    panelRender: props => <ImportsCUD action="create" list={props.resolved.list} />
                                 }
                             }
                         },

client/src/lists/styles.scss

@@ -0,0 +1,3 @@
+.mapping {
+  margin-top: 30px;
+}

client/src/lists/subscriptions/List.js

@@ -15,6 +15,7 @@ import {
 import {Icon, Button} from "../../lib/bootstrap-components";
 import axios from '../../lib/axios';
 import {getFieldTypes, getSubscriptionStatusLabels} from './helpers';
+import {getUrl} from "../../lib/urls";
 
 @translate()
 @withForm

models/import-runs.js

@@ -0,0 +1,49 @@
+'use strict';
+
+const knex = require('../lib/knex');
+const { enforce, filterObject } = require('../lib/helpers');
+const dtHelpers = require('../lib/dt-helpers');
+const interoperableErrors = require('../shared/interoperable-errors');
+const shares = require('./shares');
+
+async function getById(context, listId, importId, id) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewImports');
+
+        const entity = await tx('import_runs')
+            .innerJoin('imports', 'import_runs.import', 'imports.id')
+            .where({'imports.list': listId, 'imports.id': importId, 'import_runs.id': id})
+            .select('import_runs.id', 'import_runs.import', 'import_runs.status', 'import_runs.new',
+                'import_runs.failed', 'import_runs.processed', 'import_runs.error', 'import_runs.created', 'import_runs.finished')
+            .first();
+
+        if (!entity) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        return entity;
+    });
+}
+
+async function listDTAjax(context, listId, importId, params) {
+    return await knex.transaction(async tx => {
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewImports');
+
+        return await dtHelpers.ajaxListTx(
+            tx,
+            params,
+            builder => builder
+                .from('import_runs')
+                .innerJoin('imports', 'import_runs.import', 'imports.id')
+                .where({'imports.list': listId, 'imports.id': importId}),
+            [ 'import_runs.id', 'import_runs.created', 'import_runs.finished', 'import_runs.status', 'import_runs.processed', 'import_runs.new', 'import_runs.failed']
+        );
+    });
+}
+
+
+
+module.exports = {
+    getById,
+    listDTAjax
+};

models/imports.js

@@ -6,24 +6,43 @@ const { enforce, filterObject } = require('../lib/helpers');
 const dtHelpers = require('../lib/dt-helpers');
 const interoperableErrors = require('../shared/interoperable-errors');
 const shares = require('./shares');
-const {ImportType, ImportStatus, RunStatus} = require('../shared/imports');
+const {ImportType, ImportStatus, RunStatus, prepFinished, prepFinishedAndNotInProgress, runInProgress} = require('../shared/imports');
 const fs = require('fs-extra-promise');
 const path = require('path');
+const importer = require('../lib/importer');
 
 const filesDir = path.join(__dirname, '..', 'files', 'imports');
 
-const allowedKeys = new Set(['name', 'description', 'type', 'settings']);
+const allowedKeysCreate = new Set(['name', 'description', 'type', 'settings', 'mapping']);
+const allowedKeysUpdate = new Set(['name', 'description', 'mapping']);
 
 function hash(entity) {
-    return hasher.hash(filterObject(entity, allowedKeys));
+    return hasher.hash(filterObject(entity, allowedKeysUpdate));
 }
 
-async function getById(context, listId, id) {
+async function getById(context, listId, id, withSampleRow = false) {
     return await knex.transaction(async tx => {
         await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'viewImports');
 
         const entity = await tx('imports').where({list: listId, id}).first();
+
+        if (!entity) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
         entity.settings = JSON.parse(entity.settings);
+        entity.mapping = JSON.parse(entity.mapping);
+
+        if (withSampleRow && prepFinished(entity.status)) {
+            if (entity.type === ImportType.CSV_FILE) {
+                const importTable = 'import_file__' + id;
+
+                const row = await tx(importTable).first();
+                delete row.id;
+
+                entity.sampleRow = row;
+            }
+        }
 
         return entity;
     });
@@ -49,22 +68,21 @@ async function _validateAndPreprocess(tx, listId, entity, isCreate) {
     enforce(entity.type >= ImportType.MIN && entity.type <= ImportType.MAX, 'Invalid import type');
 
     entity.settings = entity.settings || {};
+    entity.mapping = entity.mapping || {};
 
-    if (entity.type === ImportType.CSV_FILE) {
+    if (isCreate && entity.type === ImportType.CSV_FILE) {
         entity.settings.csv = entity.settings.csv || {};
-        enforce(entity.settings.csv.delimiter.trim(), 'CSV delimiter must not be empty');
+        enforce(entity.settings.csv.delimiter && entity.settings.csv.delimiter.trim(), 'CSV delimiter must not be empty');
     }
 }
 
 async function create(context, listId, entity, files) {
-    return await knex.transaction(async tx => {
+    const res = await knex.transaction(async tx => {
         shares.enforceGlobalPermission(context, 'setupAutomation');
         await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageImports');
 
         await _validateAndPreprocess(tx, listId, entity, true);
 
-        // FIXME - set status
-
         if (entity.type === ImportType.CSV_FILE) {
             enforce(files.csvFile, 'File must be included');
             const csvFile = files.csvFile[0];
@@ -81,18 +99,22 @@ async function create(context, listId, entity, files) {
         }
 
 
-        const filteredEntity = filterObject(entity, allowedKeys);
+        const filteredEntity = filterObject(entity, allowedKeysCreate);
         filteredEntity.list = listId;
         filteredEntity.settings = JSON.stringify(filteredEntity.settings);
+        filteredEntity.mapping = JSON.stringify(filteredEntity.mapping);
 
         const ids = await tx('imports').insert(filteredEntity);
         const id = ids[0];
 
         return id;
     });
+
+    importer.scheduleCheck();
+    return res;
 }
 
-async function updateWithConsistencyCheck(context, listId, entity, files) {
+async function updateWithConsistencyCheck(context, listId, entity) {
     await knex.transaction(async tx => {
         shares.enforceGlobalPermission(context, 'setupAutomation');
         await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageImports');
@@ -102,25 +124,20 @@ async function updateWithConsistencyCheck(context, listId, entity, files) {
             throw new interoperableErrors.NotFoundError();
         }
 
-        existing.settings = JSON.parse(existing.settings);
+        existing.mapping = JSON.parse(existing.mapping);
         const existingHash = hash(existing);
         if (existingHash !== entity.originalHash) {
             throw new interoperableErrors.ChangedError();
         }
 
+        enforce(prepFinishedAndNotInProgress(existing.status), 'Cannot save updates until preparation or run is finished');
+
         enforce(entity.type === existing.type, 'Import type cannot be changed');
         await _validateAndPreprocess(tx, listId, entity, false);
 
-        if (entity.type === ImportType.CSV_FILE) {
+        const filteredEntity = filterObject(entity, allowedKeysUpdate);
-            entity.settings.csv =  existing.settings.csv;
-        }
-
-        // FIXME - set status
-        // FIXME - create CSV import table
-
-        const filteredEntity = filterObject(entity, allowedKeys);
         filteredEntity.list = listId;
-        filteredEntity.settings = JSON.stringify(filteredEntity.settings);
+        filteredEntity.mapping = JSON.stringify(filteredEntity.mapping);
 
         await tx('imports').where({list: listId, id: entity.id}).update(filteredEntity);
     });
@@ -134,9 +151,15 @@ async function removeTx(tx, context, listId, id) {
         throw new interoperableErrors.NotFoundError();
     }
 
-    // FIXME - remove csv import table
+    existing.settings = JSON.parse(existing.settings);
 
-    await tx('import_failed').whereIn('run', function() {this.from('import_runs').select('id').where('import', id)});
+    const filePath = path.join(filesDir, existing.settings.csv.filename);
+    await fs.removeAsync(filePath);
+
+    const importTable = 'import_file__' + id;
+    await knex.schema.dropTableIfExists(importTable);
+
+    await tx('import_failed').whereIn('run', function() {this.from('import_runs').select('id').where('import', id)}).del();
     await tx('import_runs').where('import', id).del();
     await tx('imports').where({list: listId, id}).del();
 }
@@ -154,6 +177,60 @@ async function removeAllByListIdTx(tx, context, listId) {
     }
 }
 
+async function start(context, listId, id) {
+    await knex.transaction(async tx => {
+        shares.enforceGlobalPermission(context, 'setupAutomation');
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageImports');
+
+        const entity = await tx('imports').where({list: listId, id}).first();
+        if (!entity) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        if (!prepFinishedAndNotInProgress(entity.status)) {
+            throw new interoperableErrors.InvalidStateError('Cannot start until preparation or run is finished');
+        }
+
+        await tx('imports').where({list: listId, id}).update({
+            status: ImportStatus.RUN_SCHEDULED
+        });
+
+        await tx('import_runs').insert({
+            import: id,
+            status: RunStatus.SCHEDULED,
+            mapping: entity.mapping
+        });
+    });
+
+    importer.scheduleCheck();
+}
+
+async function stop(context, listId, id) {
+    await knex.transaction(async tx => {
+        shares.enforceGlobalPermission(context, 'setupAutomation');
+        await shares.enforceEntityPermissionTx(tx, context, 'list', listId, 'manageImports');
+
+        const entity = await tx('imports').where({list: listId, id}).first();
+        if (!entity) {
+            throw new interoperableErrors.NotFoundError();
+        }
+
+        if (!runInProgress(entity.status)) {
+            throw new interoperableErrors.InvalidStateError('No import is currently running');
+        }
+
+        await tx('imports').where({list: listId, id}).update({
+            status: ImportStatus.RUN_STOPPING
+        });
+
+        await tx('import_runs').where('import', id).whereIn('status', [RunStatus.SCHEDULED, RunStatus.RUNNING]).update({
+            status: RunStatus.STOPPING
+        });
+    });
+
+    importer.scheduleCheck();
+}
+
 
 // This is to handle circular dependency with segments.js
 module.exports = {
@@ -164,5 +241,7 @@ module.exports = {
     create,
     updateWithConsistencyCheck,
     remove,
-    removeAllByListIdTx
+    removeAllByListIdTx,
+    start,
+    stop
 };

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "mailtrain",
-  "version": "1.23.2",
+  "version": "1.24.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -1655,6 +1655,11 @@
       "resolved": "https://registry.npmjs.org/double-ended-queue/-/double-ended-queue-2.1.0-0.tgz",
       "integrity": "sha1-ED01J/0xUo9AGIEwyEHv3XgmTlw="
     },
+    "easy-stack": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/easy-stack/-/easy-stack-1.0.0.tgz",
+      "integrity": "sha1-EskbMIWjfwuqM26UhurEv5Tj54g="
+    },
     "ecc-jsbn": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz",
@@ -1969,6 +1974,11 @@
       "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
       "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc="
     },
+    "event-pubsub": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/event-pubsub/-/event-pubsub-4.3.0.tgz",
+      "integrity": "sha512-z7IyloorXvKbFx9Bpie2+vMJKKx1fH1EN5yiTfp8CiLOTptSYy1g8H4yDpGlEdshL1PBiFtBHepF2cNsqeEeFQ=="
+    },
     "eventemitter2": {
       "version": "0.4.14",
       "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz",
@@ -4074,6 +4084,19 @@
         "nopt": "~3.0.1"
       }
     },
+    "js-message": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/js-message/-/js-message-1.0.5.tgz",
+      "integrity": "sha1-IwDSSxrwjondCVvBpMnJz8uJLRU="
+    },
+    "js-queue": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/js-queue/-/js-queue-2.0.0.tgz",
+      "integrity": "sha1-NiITz4YPRo8BJfxslqvBdCUx+Ug=",
+      "requires": {
+        "easy-stack": "^1.0.0"
+      }
+    },
     "js-tokens": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-3.0.2.tgz",
@@ -5624,6 +5647,16 @@
         "lodash.get": "^4.4.2"
       }
     },
+    "node-ipc": {
+      "version": "9.1.1",
+      "resolved": "https://registry.npmjs.org/node-ipc/-/node-ipc-9.1.1.tgz",
+      "integrity": "sha512-FAyICv0sIRJxVp3GW5fzgaf9jwwRQxAKDJlmNFUL5hOy+W4X/I5AypyHoq0DXXbo9o/gt79gj++4cMr4jVWE/w==",
+      "requires": {
+        "event-pubsub": "4.3.0",
+        "js-message": "1.0.5",
+        "js-queue": "2.0.0"
+      }
+    },
     "node-localstorage": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/node-localstorage/-/node-localstorage-1.3.0.tgz",

package.json

@@ -99,6 +99,7 @@
     "multiparty": "^4.1.3",
     "mysql2": "^1.3.5",
     "node-gettext": "^2.0.0-rc.1",
+    "node-ipc": "^9.1.1",
     "node-mocks-http": "^1.6.5",
     "node-object-hash": "^1.2.0",
     "nodeify": "^1.0.1",

routes/rest/import-runs.js

@@ -0,0 +1,17 @@
+'use strict';
+
+const passport = require('../../lib/passport');
+const importRuns = require('../../models/import-runs');
+
+const router = require('../../lib/router-async').create();
+
+router.postAsync('/import-runs-table/:listId/:importId', passport.loggedIn, async (req, res) => {
+    return res.json(await importRuns.listDTAjax(req.context, req.params.listId, req.params.importId, req.body));
+});
+
+router.getAsync('/import-runs/:listId/:importId/:runId', passport.loggedIn, async (req, res) => {
+    const entity = await importRuns.getById(req.context, req.params.listId, req.params.importId, req.params.runId);
+    return res.json(entity);
+});
+
+module.exports = router;

routes/rest/imports.js

@@ -19,7 +19,7 @@ router.postAsync('/imports-table/:listId', passport.loggedIn, async (req, res) =
 });
 
 router.getAsync('/imports/:listId/:importId', passport.loggedIn, async (req, res) => {
-    const entity = await imports.getById(req.context, req.params.listId, req.params.importId);
+    const entity = await imports.getById(req.context, req.params.listId, req.params.importId, true);
     entity.hash = imports.hash(entity);
     return res.json(entity);
 });
@@ -47,5 +47,12 @@ router.deleteAsync('/imports/:listId/:importId', passport.loggedIn, passport.csr
     return res.json();
 });
 
+router.postAsync('/import-start/:listId/:importId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await imports.start(req.context, req.params.listId, req.params.importId));
+});
+
+router.postAsync('/import-stop/:listId/:importId', passport.loggedIn, passport.csrfProtection, async (req, res) => {
+    return res.json(await imports.stop(req.context, req.params.listId, req.params.importId));
+});
 
 module.exports = router;

services/importer.js

@@ -6,35 +6,39 @@ const log = require('npmlog');
 const fsExtra = require('fs-extra-promise');
 const {ImportType, ImportStatus, RunStatus} = require('../shared/imports');
 const imports = require('../models/imports');
+const { Writable } = require('stream');
 
 const csvparse = require('csv-parse');
 const fs = require('fs');
 
 let running = false;
+const maxInsertBatchSize = 100;
 
 function prepareCsv(impt) {
-    async function finishWithError(msg, err) {
+    // Processing of CSV intake
-        if (finished) {
+    const filePath = path.join(imports.filesDir, impt.settings.csv.filename);
-            return;
+    const importTable = 'import_file__' + impt.id;
-        }
 
-        finished = true;
+    let finishedWithError = false;
+    let firstRow;
+
+    const finishWithError = async (msg, err) => {
+        finishedWithError = true;
         log.error('Importer (CSV)', err.stack);
 
         await knex('imports').where('id', impt.id).update({
             status: ImportStatus.PREP_FAILED,
-            error: msg + '\n' + err.stack
+            error: msg + '\n' + err.message
         });
 
         await fsExtra.removeAsync(filePath);
-    }
+    };
 
-    async function finishWithSuccess() {
+    const finishWithSuccess = async () => {
-        if (finished) {
+        if (finishedWithError) {
             return;
         }
 
-        finished = true;
         log.info('Importer (CSV)', 'Preparation finished');
 
         await knex('imports').where('id', impt.id).update({
@@ -43,59 +47,87 @@ function prepareCsv(impt) {
         });
 
         await fsExtra.removeAsync(filePath);
-    }
+    };
 
-    // Processing of CSV intake
+    const processRows = async (chunks) => {
-    const filePath = path.join(imports.filesDir, impt.settings.csv.filename);
+        console.log('process row');
+        let insertBatch = [];
+        for (const chunkEntry of chunks) {
+            const record = chunkEntry.chunk;
 
+            if (!firstRow) {
+                firstRow = true;
+
+                const cols = [];
+                let colsDef = '';
+                for (let idx = 0; idx < record.length; idx++) {
+                    const colName = 'column_' + idx;
+                    cols.push({
+                        column: colName,
+                        name: record[idx]
+                    });
+
+                    colsDef += '  `' + colName + '` text DEFAULT NULL,\n';
+                }
+
+                impt.settings.csv.columns = cols;
+                await knex('imports').where({id: impt.id}).update({settings: JSON.stringify(impt.settings)});
+
+                await knex.schema.raw('CREATE TABLE `' + importTable + '` (\n' +
+                    '  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,\n' +
+                    colsDef +
+                    '  PRIMARY KEY (`id`)\n' +
+                    ') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\n');
+
+            } else {
+                const dbRecord = {};
+                for (let idx = 0; idx < record.length; idx++) {
+                    dbRecord['column_' + idx] = record[idx];
+                }
+
+                insertBatch.push(dbRecord);
+            }
+
+            if (insertBatch.length >= maxInsertBatchSize) {
+                await knex(importTable).insert(insertBatch);
+                insertBatch = [];
+            }
+        }
+
+        if (insertBatch.length > 0) {
+            await knex(importTable).insert(insertBatch);
+        }
+    };
+
+
+    const inputStream = fs.createReadStream(filePath);
     const parser = csvparse({
         comment: '#',
         delimiter: impt.settings.csv.delimiter
     });
 
-    const inputStream = fs.createReadStream(filePath);
-    let finished;
-
     inputStream.on('error', err => finishWithError('Error reading CSV file.', err));
     parser.on('error', err => finishWithError('Error parsing CSV file.', err));
 
-    let firstRow;
+    const importProcessor = new Writable({
-    let processing = false;
+        write(chunk, encoding, callback) {
-    const processRows = () => {
+            processRows([{chunk, encoding}]).then(() => callback());
-        const record = parser.read();
+        },
-        if (record === null) {
+        writev(chunks, callback) {
-            processing = false;
+            processRows(chunks).then(() => callback());
-            return;
+        },
-        }
+        final(callback) {
-        processing = true;
+            finishWithSuccess().then(() => callback());
-
+        },
-        if (!firstRow) {
+        objectMode: true
-            firstRow = record;
-            console.log(record);
-            return setImmediate(processRows);
-
-        }
-
-        console.log(record);
-        return setImmediate(processRows);
-    };
-
-    parser.on('readable', () => {
-        if (finished || processing) {
-            return;
-        }
-        processRows();
-    });
-
-    parser.on('finish', () => {
-        finishWithSuccess();
     });
 
+    parser.pipe(importProcessor);
     inputStream.pipe(parser);
 }
 
 async function getTask() {
-    await knex.transaction(async tx => {
+    return await knex.transaction(async tx => {
         const impt = await tx('imports').whereIn('status', [ImportStatus.PREP_SCHEDULED, ImportStatus.RUN_SCHEDULED]).orderBy('created', 'asc').first();
 
         if (impt) {
@@ -109,7 +141,7 @@ async function getTask() {
         } else {
             return null;
         }
-    })
+    });
 }
 
 async function run() {
@@ -132,7 +164,7 @@ process.on('message', msg => {
         const type = msg.type;
 
         if (type === 'scheduleCheck') {
-            run()
+            run();
         }
     }
 });
@@ -141,3 +173,5 @@ process.send({
     type: 'importer-started'
 });
 
+run();
+

setup/knex/migrations/20170506102634_v1_to_v2.js

@@ -1028,6 +1028,7 @@ async function migrateImporter(knex) {
         table.integer('type').unsigned().notNullable();
         table.integer('status').unsigned().notNullable();
         table.text('settings', 'longtext');
+        table.text('mapping', 'longtext');
         table.timestamp('last_run');
         table.text('error');
         table.timestamp('created').defaultTo(knex.fn.now());
@@ -1037,6 +1038,7 @@ async function migrateImporter(knex) {
         table.increments('id').primary();
         table.integer('import').unsigned().references('imports.id');
         table.integer('status').unsigned().notNullable();
+        table.text('mapping', 'longtext');
         table.integer('new').defaultTo(0);
         table.integer('failed').defaultTo(0);
         table.integer('processed').defaultTo(0);

shared/imports.js

@@ -12,22 +12,60 @@ const ImportType = {
 const ImportStatus = {
     PREP_SCHEDULED: 0,
     PREP_RUNNING: 1,
-    PREP_FINISHED: 2,
+    PREP_STOPPING: 2,
-    PREP_FAILED: 3,
+    PREP_FINISHED: 3,
+    PREP_FAILED: 4,
 
-    RUN_SCHEDULED: 4,
+    RUN_SCHEDULED: 5,
-    RUN_RUNNING: 5,
+    RUN_RUNNING: 6,
-    RUN_FINISHED: 6,
+    RUN_STOPPING: 7,
-    RUN_FAILED: 7
+    RUN_FINISHED: 8,
+    RUN_FAILED: 9
 };
 
 const RunStatus = {
-    RUNNING: 0,
+    SCHEDULED: 0,
-    FINISHED: 1
+    RUNNING: 1,
+    STOPPING: 2,
+    FINISHED: 3
 };
 
+function prepInProgress(status) {
+    return status === ImportStatus.PREP_SCHEDULED || status === ImportStatus.PREP_RUNNING || status === ImportStatus.PREP_STOPPING;
+}
+
+function runInProgress(status) {
+    return status === ImportStatus.RUN_SCHEDULED || status === ImportStatus.RUN_RUNNING || status === ImportStatus.RUN_STOPPING;
+}
+
+function inProgress(status) {
+    return status === ImportStatus.PREP_SCHEDULED || status === ImportStatus.PREP_RUNNING || status === ImportStatus.PREP_STOPPING ||
+        status === ImportStatus.RUN_SCHEDULED || status === ImportStatus.RUN_RUNNING || status === ImportStatus.RUN_STOPPING;
+}
+
+function prepFinished(status) {
+    return status === ImportStatus.PREP_FINISHED ||
+        status === ImportStatus.RUN_SCHEDULED || status === ImportStatus.RUN_RUNNING || status === ImportStatus.RUN_STOPPING ||
+        status === ImportStatus.RUN_FINISHED || status === ImportStatus.RUN_FAILED;
+}
+
+function prepFinishedAndNotInProgress(status) {
+    return status === ImportStatus.PREP_FINISHED ||
+        status === ImportStatus.RUN_FINISHED || status === ImportStatus.RUN_FAILED;
+}
+
+function runStatusInProgress(status) {
+    return status === RunStatus.SCHEDULED || status === RunStatus.RUNNING || status === RunStatus.STOPPING;
+}
+
 module.exports = {
     ImportType,
     ImportStatus,
-    RunStatus
+    RunStatus,
+    prepInProgress,
+    runInProgress,
+    prepFinished,
+    prepFinishedAndNotInProgress,
+    inProgress,
+    runStatusInProgress
 };

shared/interoperable-errors.js

@@ -112,6 +112,12 @@ class DependencyPresentError extends InteroperableError {
     }
 }
 
+class InvalidStateError extends InteroperableError {
+    constructor(msg, data) {
+        super('InvalidStateError', msg, data);
+    }
+}
+
 
 const errorTypes = {
     InteroperableError,
@@ -131,7 +137,8 @@ const errorTypes = {
     InvalidConfirmationForSubscriptionError,
     InvalidConfirmationForAddressChangeError,
     InvalidConfirmationForUnsubscriptionError,
-    DependencyPresentError
+    DependencyPresentError,
+    InvalidStateError
 };
 
 function deserialize(errorObj) {