UI for basic import and preparation phase of CSV.

services/importer.js

@@ -6,35 +6,39 @@ const log = require('npmlog');
 const fsExtra = require('fs-extra-promise');
 const {ImportType, ImportStatus, RunStatus} = require('../shared/imports');
 const imports = require('../models/imports');
+const { Writable } = require('stream');
 
 const csvparse = require('csv-parse');
 const fs = require('fs');
 
 let running = false;
+const maxInsertBatchSize = 100;
 
 function prepareCsv(impt) {
-    async function finishWithError(msg, err) {
-        if (finished) {
-            return;
-        }
+    // Processing of CSV intake
+    const filePath = path.join(imports.filesDir, impt.settings.csv.filename);
+    const importTable = 'import_file__' + impt.id;
 
-        finished = true;
+    let finishedWithError = false;
+    let firstRow;
+
+    const finishWithError = async (msg, err) => {
+        finishedWithError = true;
         log.error('Importer (CSV)', err.stack);
 
         await knex('imports').where('id', impt.id).update({
             status: ImportStatus.PREP_FAILED,
-            error: msg + '\n' + err.stack
+            error: msg + '\n' + err.message
         });
 
         await fsExtra.removeAsync(filePath);
-    }
+    };
 
-    async function finishWithSuccess() {
-        if (finished) {
+    const finishWithSuccess = async () => {
+        if (finishedWithError) {
             return;
         }
 
-        finished = true;
         log.info('Importer (CSV)', 'Preparation finished');
 
         await knex('imports').where('id', impt.id).update({
@@ -43,59 +47,87 @@ function prepareCsv(impt) {
         });
 
         await fsExtra.removeAsync(filePath);
-    }
+    };
 
-    // Processing of CSV intake
-    const filePath = path.join(imports.filesDir, impt.settings.csv.filename);
+    const processRows = async (chunks) => {
+        console.log('process row');
+        let insertBatch = [];
+        for (const chunkEntry of chunks) {
+            const record = chunkEntry.chunk;
 
+            if (!firstRow) {
+                firstRow = true;
+
+                const cols = [];
+                let colsDef = '';
+                for (let idx = 0; idx < record.length; idx++) {
+                    const colName = 'column_' + idx;
+                    cols.push({
+                        column: colName,
+                        name: record[idx]
+                    });
+
+                    colsDef += '  `' + colName + '` text DEFAULT NULL,\n';
+                }
+
+                impt.settings.csv.columns = cols;
+                await knex('imports').where({id: impt.id}).update({settings: JSON.stringify(impt.settings)});
+
+                await knex.schema.raw('CREATE TABLE `' + importTable + '` (\n' +
+                    '  `id` int(10) unsigned NOT NULL AUTO_INCREMENT,\n' +
+                    colsDef +
+                    '  PRIMARY KEY (`id`)\n' +
+                    ') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;\n');
+
+            } else {
+                const dbRecord = {};
+                for (let idx = 0; idx < record.length; idx++) {
+                    dbRecord['column_' + idx] = record[idx];
+                }
+
+                insertBatch.push(dbRecord);
+            }
+
+            if (insertBatch.length >= maxInsertBatchSize) {
+                await knex(importTable).insert(insertBatch);
+                insertBatch = [];
+            }
+        }
+
+        if (insertBatch.length > 0) {
+            await knex(importTable).insert(insertBatch);
+        }
+    };
+
+
+    const inputStream = fs.createReadStream(filePath);
     const parser = csvparse({
         comment: '#',
         delimiter: impt.settings.csv.delimiter
     });
 
-    const inputStream = fs.createReadStream(filePath);
-    let finished;
-
     inputStream.on('error', err => finishWithError('Error reading CSV file.', err));
     parser.on('error', err => finishWithError('Error parsing CSV file.', err));
 
-    let firstRow;
-    let processing = false;
-    const processRows = () => {
-        const record = parser.read();
-        if (record === null) {
-            processing = false;
-            return;
-        }
-        processing = true;
-
-        if (!firstRow) {
-            firstRow = record;
-            console.log(record);
-            return setImmediate(processRows);
-
-        }
-
-        console.log(record);
-        return setImmediate(processRows);
-    };
-
-    parser.on('readable', () => {
-        if (finished || processing) {
-            return;
-        }
-        processRows();
-    });
-
-    parser.on('finish', () => {
-        finishWithSuccess();
+    const importProcessor = new Writable({
+        write(chunk, encoding, callback) {
+            processRows([{chunk, encoding}]).then(() => callback());
+        },
+        writev(chunks, callback) {
+            processRows(chunks).then(() => callback());
+        },
+        final(callback) {
+            finishWithSuccess().then(() => callback());
+        },
+        objectMode: true
     });
 
+    parser.pipe(importProcessor);
     inputStream.pipe(parser);
 }
 
 async function getTask() {
-    await knex.transaction(async tx => {
+    return await knex.transaction(async tx => {
         const impt = await tx('imports').whereIn('status', [ImportStatus.PREP_SCHEDULED, ImportStatus.RUN_SCHEDULED]).orderBy('created', 'asc').first();
 
         if (impt) {
@@ -109,7 +141,7 @@ async function getTask() {
         } else {
             return null;
         }
-    })
+    });
 }
 
 async function run() {
@@ -132,7 +164,7 @@ process.on('message', msg => {
         const type = msg.type;
 
         if (type === 'scheduleCheck') {
-            run()
+            run();
         }
     }
 });
@@ -141,3 +173,5 @@ process.send({
     type: 'importer-started'
 });
 
+run();
+