Work in progress on integration of IVIS.

Some fixes.

client/src/campaigns/Status.js

@@ -615,7 +615,7 @@ export default class Status extends Component {
                 <Title>{t('campaignStatus')}</Title>
 
                 <AlignedRow label={t('name')}>{entity.name}</AlignedRow>
-                <AlignedRow label={t('delivered')}>{entity.delivered}</AlignedRow>
+                <AlignedRow label={t('Sent')}>{entity.delivered}</AlignedRow>
                 <AlignedRow label={t('status')}>{this.campaignStatusLabels[entity.status]}</AlignedRow>
 
                 {sendSettings}

mvis/client/webpack.config.js

@@ -1,12 +1,11 @@
-const webpack = require('webpack');
 const path = require('path');
 
 const webpackConf = require('../ivis-core/client/webpack.config');
 
 webpackConf.resolve.modules = ['node_modules', '../ivis-core/client/node_modules'];
 webpackConf.entry = {
-    'index-trusted': ['babel-polyfill', './src/root-trusted.js'],
+    'index-trusted': ['@babel/polyfill', './src/root-trusted.js'],
-    'index-sandbox': ['babel-polyfill', '../ivis-core/client/src/root-sandbox.js']
+    'index-sandbox': ['@babel/polyfill', '../ivis-core/client/src/root-sandbox.js']
 };
 webpackConf.output = {
     filename: '[name].js',

mvis/ivis-core

@@ -1 +1 @@
-Subproject commit 052fbff6e4eaba52eee2f984f2b8d947ebfa7298
+Subproject commit 5ea4783f3ec5140ad68637c31e230a410e493170

mvis/server/config/default.yaml

@@ -6,8 +6,7 @@ mysql:
 
 mailtrain:
   url: http://localhost:3000/
-  namespaces:
+  namespace: 1
-    campaigns: 2
   userRole: mailtrainUser
 
 www:

mvis/server/extensions-common.js

@@ -5,6 +5,7 @@ const path = require('path');
 
 em.set('config.extraDirs', [ path.join(__dirname, 'config') ]);
 em.set('builder.exec', path.join(__dirname, 'builder.js'));
+em.set('task-handler.exec', path.join(__dirname, 'task-handler.js'));
 em.set('indexer.elasticsearch.exec', path.join(__dirname, 'indexer-elasticsearch.js'));
 em.set('app.title', 'Mailtrain IVIS');
 

mvis/server/index.js

@@ -15,6 +15,9 @@ em.on('knex.migrate', async () => {
 em.on('app.installAPIRoutes', app => {
     const embedApi = require('./routes/api/embed');
     app.use('/api', embedApi);
+
+    const eventsApi = require('./routes/api/events');
+    app.use('/api', eventsApi);
 });
 
 require('../ivis-core/server/index');

mvis/server/routes/api/events.js

@@ -0,0 +1,123 @@
+'use strict';
+
+const config = require('../../../ivis-core/server/lib/config');
+const moment = require('moment');
+const knex = require('../../../ivis-core/server/lib/knex');
+const router = require('../../../ivis-core/server/lib/router-async').create();
+const log = require('../../../ivis-core/server/lib/log');
+const signalSets = require('../../../ivis-core/server/models/signal-sets');
+const { SignalType } = require('../../../ivis-core/shared/signals');
+const contextHelpers = require('../../../ivis-core/server/lib/context-helpers');
+const namespaces = require('../../../ivis-core/server/models/namespaces');
+
+/*
+async function ensureCampaignTracker() {
+    const schema = {
+        type: {
+            type: SignalType.INTEGER,
+            name: 'Type',
+            settings: {},
+            indexed: true,
+            weight_list: 0,
+            weight_edit: 0
+        },
+        timestamp: {
+            type: SignalType.DATE_TIME,
+            name: 'Timestamp',
+            settings: {},
+            indexed: true,
+            weight_list: 1,
+            weight_edit: 1
+        },
+        campaignId: {
+            type: SignalType.INTEGER,
+            name: 'Campaign ID',
+            settings: {},
+            indexed: true,
+            weight_list: 2,
+            weight_edit: 2
+        },
+        listId: {
+            type: SignalType.INTEGER,
+            name: 'List ID',
+            settings: {},
+            indexed: true,
+            weight_list: 3,
+            weight_edit: 3
+        },
+        subscriptionId: {
+            type: SignalType.INTEGER,
+            name: 'Subscription ID',
+            settings: {},
+            indexed: true,
+            weight_list: 4,
+            weight_edit: 4
+        },
+
+    };
+
+    return await signalSets.ensure(
+        req.context,
+        'campaignTracker',
+        schema,
+        'Campaign Tracker',
+        '',
+        config.mailtrain.namespace
+    );
+}
+
+async function ingestCampaignTrackerRecord(record) {
+    return {
+        id: TODO
+    };
+}
+
+const types = {
+    campaign_tracker: {
+        ensure: ensureCampaignTracker,
+        ingest: ingestCampaignTrackerRecord
+    }
+}
+
+router.postAsync('/events', async (req, res) => {
+    const batch = req.body;
+
+    const recordsByType = {};
+    const signalSetWithSignalMapByType = {};
+
+    for (const type in types) {
+        recordsByType[type] = [];
+        signalSetWithSignalMapByType[type] = await types[type].ensure();
+    }
+
+    for (const dataEntry of batch.data) {
+        const record = {
+            id: dataEntry[idField],
+            signals: {}
+        };
+
+        for (const fieldId in dataEntry) {
+            if (fieldId !== idField) {
+                if (!(fieldId in schema)) {
+                    throw new Error(`Unknown data field "${fieldId}`);
+                }
+
+                let value = dataEntry[fieldId];
+                if (schema[fieldId].type === SignalType.DATE_TIME) {
+                    value = moment(value);
+                }
+
+                record.signals[fieldId] = value;
+            }
+        }
+
+        records.push(record);
+    }
+
+    await signalSets.insertRecords(req.context, signalSetWithSignalMap, records);
+
+    return res.json();
+});
+*/
+
+module.exports = router;

mvis/server/task-handler.js

@@ -0,0 +1,5 @@
+'use strict';
+
+require('./extensions-common');
+require('../ivis-core/server/services/task-handler');
+

server/lib/activity-log.js

@@ -1,7 +1,37 @@
 'use strict';
 
+const moment = require('moment');
+
+const activityQueueLenthThreshold = 100;
+const actitivyQueue = [];
+
+let processQueueIsRunning = false;
+
+async function processQueue() {
+    if (processQueueIsRunning) {
+        return;
+    }
+
+    processQueueIsRunning = true;
+
+    // XXX submit data to IVIS if configured in config
+
+    actitivyQueue.splice(0);
+
+    processQueueIsRunning = false;
+}
+
 async function _logActivity(typeId, data) {
-    // TODO
+    actitivyQueue.push({
+        typeId,
+        data,
+        timestamp: moment.utc().toISOString()
+    });
+
+    if (actitivyQueue.length >= activityQueueLenthThreshold) {
+        // noinspection ES6MissingAwait
+        processQueue();
+    }
 }
 
 /*

server/services/sender-worker.js

@@ -4,8 +4,12 @@ const config = require('../lib/config');
 const log = require('../lib/log');
 const mailers = require('../lib/mailers');
 const messageSender = require('../lib/message-sender');
+const {CampaignTrackerActivityType} = require('../../shared/activity-log');
+const activityLog = require('../lib/activity-log');
 require('../lib/fork');
 
+const MessageType = messageSender.MessageType;
+
 const workerId = Number.parseInt(process.argv[2]);
 let running = false;
 
@@ -26,6 +30,8 @@ async function processCampaignMessages(campaignId, messages) {
         try {
             await cs.sendRegularCampaignMessage(campaignMessage);
 
+            await activityLog.logCampaignTrackerActivity(CampaignTrackerActivityType.SENT, campaignId, campaignMessage.list, campaignMessage.subscription);
+
             log.verbose('Senders', 'Message sent and status updated for %s:%s', campaignMessage.list, campaignMessage.subscription);
         } catch (err) {
 
@@ -58,6 +64,8 @@ async function processQueuedMessages(sendConfigurationId, messages) {
 
     for (const queuedMessage of messages) {
 
+        const messageType = queuedMessage.type;
+
         const msgData = queuedMessage.data;
         let target = '';
         if (msgData.listId && msgData.subscriptionId) {
@@ -74,6 +82,11 @@ async function processQueuedMessages(sendConfigurationId, messages) {
 
         try {
             await messageSender.sendQueuedMessage(queuedMessage);
+
+            if ((messageType === MessageType.TRIGGERED || messageType === MessageType.TEST) && msgData.campaignId && msgData.listId && msgData.subscriptionId) {
+                await activityLog.logCampaignTrackerActivity(CampaignTrackerActivityType.SENT, msgData.campaignId, msgData.listId, msgData.subscriptionId);
+            }
+
             log.verbose('Senders', `Message sent and status updated for ${target}`);
         } catch (err) {
             if (err instanceof mailers.SendConfigurationError) {

shared/activity-log.js

@@ -30,10 +30,12 @@ const ListActivityType = {
 };
 
 const CampaignTrackerActivityType = {
-    DELIVERED: 1,
+    SENT: 1,
     BOUNCED: 2,
-    OPENED: 3,
+    UNSUBSCRIBED: 3,
-    CLICKED: 4
+    COMPLAINED: 4,
+    OPENED: 5,
+    CLICKED: 6
 };
 
 const BlacklistActivityType = {
@@ -46,3 +48,4 @@ module.exports.EntityActivityType = EntityActivityType;
 module.exports.BlacklistActivityType = BlacklistActivityType;
 module.exports.CampaignActivityType = CampaignActivityType;
 module.exports.ListActivityType = ListActivityType;
+module.exports.CampaignTrackerActivityType = CampaignTrackerActivityType;