From 01d1a903a2c0d4d120e942dfa47e5e66dc8e1e39 Mon Sep 17 00:00:00 2001
From: Tomas Bures <bures@d3s.mff.cuni.cz>
Date: Mon, 10 Sep 2018 20:15:59 +0200
Subject: [PATCH] Work in progress on Campaign Status

---
 client/src/campaigns/CUD.js                   |   2 +-
 client/src/campaigns/Status.js                |  14 +-
 client/src/lists/subscriptions/List.js        |   2 +-
 lib/knex.js                                   |   2 +-
 models/campaigns.js                           | 177 +++++++++++-------
 services/importer.js                          |   1 -
 .../migrations/20170506102634_v1_to_v2.js     |   2 +-
 7 files changed, 116 insertions(+), 84 deletions(-)

diff --git a/client/src/campaigns/CUD.js b/client/src/campaigns/CUD.js
index d2cad797..64c4b853 100644
--- a/client/src/campaigns/CUD.js
+++ b/client/src/campaigns/CUD.js
@@ -170,7 +170,7 @@ export default class CUD extends Component {
                 }
 
                 for (const overridable of campaignOverridables) {
-                    data[overridable + '_overriden'] = data[overridable + '_override'] === null;
+                    data[overridable + '_overriden'] = data[overridable + '_override'] !== null;
                 }
 
                 const lsts = [];
diff --git a/client/src/campaigns/Status.js b/client/src/campaigns/Status.js
index 6f305da0..379c6399 100644
--- a/client/src/campaigns/Status.js
+++ b/client/src/campaigns/Status.js
@@ -243,11 +243,11 @@ class SendControls extends Component {
         } else if (entity.status === CampaignStatus.SENDING || (entity.status === CampaignStatus.SCHEDULED && !entity.scheduled)) {
             return (
                 <div>
-                    <div>
+                    <AlignedRow label={t('Send status')}>
                         {t('Campaign is being sent out.')}
-                    </div>
+                    </AlignedRow>
                     <ButtonRow>
-                        <Button className="btn-primary" icon="stop" label={t('Stop')} onClickAsync={::this.pauseAsync}/>
+                        <Button className="btn-primary" icon="stop" label={t('Stop')} onClickAsync={::this.stopAsync}/>
                     </ButtonRow>
                 </div>
             );
@@ -255,9 +255,9 @@ class SendControls extends Component {
         } else if (entity.status === CampaignStatus.FINISHED) {
             return (
                 <div>
-                    <div>
+                    <AlignedRow label={t('Send status')}>
                         {t('All messages sent! Hit "Continue" if you you want to send this campaign to new subscribers.')}
-                    </div>
+                    </AlignedRow>
                     <ButtonRow>
                         <Button className="btn-primary" icon="play" label={t('Continue')} onClickAsync={::this.startAsync}/>
                         <Button className="btn-primary" icon="refresh" label={t('Reset')} onClickAsync={::this.resetAsync}/>
@@ -356,8 +356,8 @@ export default class Status extends Component {
         const listsColumns = [
             { data: 1, title: t('Name') },
             { data: 2, title: t('ID'), render: data => <code>{data}</code> },
-            { data: 3, title: t('Namespace') },
-            { data: 4, title: t('Segment') }
+            { data: 4, title: t('Segment') },
+            { data: 3, title: t('List namespace') }
         ];
 
         return (
diff --git a/client/src/lists/subscriptions/List.js b/client/src/lists/subscriptions/List.js
index e166cf75..50fe95df 100644
--- a/client/src/lists/subscriptions/List.js
+++ b/client/src/lists/subscriptions/List.js
@@ -86,7 +86,7 @@ export default class List extends Component {
         const segments = this.props.segments;
 
         const columns = [
-            { data: 1, title: t('CID') },
+            { data: 1, title: t('ID'), render: data => <code>{data}</code> },
             { data: 2, title: t('Email') },
             { data: 3, title: t('Status'), render: (data, display, rowData) => this.subscriptionStatusLabels[data] + (rowData[5] ? ', ' + t('Blacklisted') : '') },
             { data: 4, title: t('Created'), render: data => data ? moment(data).fromNow() : '' }
diff --git a/lib/knex.js b/lib/knex.js
index fd0bfb5b..327c55ac 100644
--- a/lib/knex.js
+++ b/lib/knex.js
@@ -8,7 +8,7 @@ const knex = require('knex')({
     migrations: {
         directory: __dirname + '/../setup/knex/migrations'
     }
-    , debug: true
+    //, debug: true
 });
 
 module.exports = knex;
diff --git a/models/campaigns.js b/models/campaigns.js
index d9e2a669..ca588e7d 100644
--- a/models/campaigns.js
+++ b/models/campaigns.js
@@ -97,46 +97,66 @@ async function listTestUsersDTAjax(context, campaignId, params) {
     return await knex.transaction(async tx => {
         await shares.enforceEntityPermissionTx(tx, context, 'campaign', campaignId, 'view');
 
-        const subscriptionsQueries = [];
+        /*
+        This is supposed to produce queries like this:
+
+        select * from (
+          (select `subscription__1`.`email`, 2 AS campaign_list_id, 1 AS list, NULL AS segment from `subscription__1` left join `campaign_messages` on
+            `campaign_messages`.`subscription` = `subscription__1`.`id` where `subscription__1`.`status` = 1 and `subscription__1`.`is_test` = true)
+        UNION ALL
+          (select `subscription__2`.`email`, 4 AS campaign_list_id, 2 AS list, NULL AS segment from `subscription__2` left join `campaign_messages` on
+          `campaign_messages`.`subscription` = `subscription__2`.`id` where `subscription__2`.`status` = 1 and `subscription__2`.`is_test` = true)
+        ) as `test_subscriptions` inner join `lists` on `test_subscriptions`.`list` = `lists`.`id` inner join `segments` on `test_subscriptions`.`segment` = `segments`.`id`
+          inner join `namespaces` on `lists`.`namespace` = `namespaces`.`id`
+
+        This was too much for Knex, so we partially construct these queries directly as strings;
+        */
+
+        const subsQrys = [];
         const cpgLists = await tx('campaign_lists').where('campaign', campaignId);
 
         for (const cpgList of cpgLists) {
-            const addSegmentQuery = cpgList.segment ? await segments.getQueryGeneratorTx(tx, cpgList.list, cpgList.segment) : () => {
-            };
+            const addSegmentQuery = cpgList.segment ? await segments.getQueryGeneratorTx(tx, cpgList.list, cpgList.segment) : () => {};
             const subsTable = subscriptions.getSubscriptionTableName(cpgList.list);
 
-            subscriptionsQueries.push(function () {
-                this.from(subsTable)
-                    .where(subsTable + '.status', SubscriptionStatus.SUBSCRIBED)
-                    .where(subsTable + '.is_test', true)
-                    .where(function() {
-                        addSegmentQuery(this);
-                    })
-                    .select([subsTable + '.email', knex.raw('? AS campaign_list_id', [cpgList.id]), knex.raw('? AS list', [cpgList.list]), knex.raw('? AS segment', [cpgList.segment])]);
-            });
+            const sqlQry = knex.from(subsTable)
+                .where(subsTable + '.status', SubscriptionStatus.SUBSCRIBED)
+                .where(subsTable + '.is_test', true)
+                .where(function() {
+                    addSegmentQuery(this);
+                })
+                .select([subsTable + '.email', knex.raw('? AS campaign_list_id', [cpgList.id]), knex.raw('? AS list', [cpgList.list]), knex.raw('? AS segment', [cpgList.segment])])
+                .toSQL().toNative();
+
+            subsQrys.push(sqlQry);
         }
 
-        if (subscriptionsQueries.length > 0) {
+        if (subsQrys.length > 0) {
+            let subsQry;
+
+            if (subsQrys.length === 1) {
+                const subsUnionSql = '(' + subsQrys[0].sql + ') as `test_subscriptions`'
+                subsQry = knex.raw(subsUnionSql, subsQrys[0].bindings);
+
+            } else {
+                const subsUnionSql = '(' +
+                    subsQrys.map(qry => '(' + qry.sql + ')').join(' UNION ALL ') +
+                    ') as `test_subscriptions`';
+                const subsUnionBindings = Array.prototype.concat(...subsQrys.map(qry => qry.bindings));
+                subsQry = knex.raw(subsUnionSql, subsUnionBindings);
+            }
+
             return await dtHelpers.ajaxListWithPermissions(
                 context,
                 [{ entityTypeId: 'list', requiredOperations: ['viewSubscriptions'] }],
                 params,
                 builder => {
-                    let ret;
-                    if (subscriptionsQueries.length > 1) {
-                        ret = builder.unionAll(subscriptionsQueries, true)
-                            .as('test_subscriptions');
-                    } else {
-                        ret = builder.from(function () { subscriptionsQueries[0].apply(this); this.as('test_subscriptions'); })
-                            .as('test_subscriptions');
-                    }
-
-                    ret = ret
+                    const qry = builder.from(subsQry)
                         .innerJoin('lists', 'test_subscriptions.list', 'lists.id')
-                        .innerJoin('segments', 'test_subscriptions.segment', 'segments.id')
-                        .innerJoin('namespaces', 'lists.namespace', 'namespaces.id');
+                        .leftJoin('segments', 'test_subscriptions.segment', 'segments.id')
+                        .innerJoin('namespaces', 'lists.namespace', 'namespaces.id')
 
-                    return ret;
+                    return qry
                 },
                 ['test_subscriptions.campaign_list_id', 'test_subscriptions.email', 'test_subscriptions.list', 'test_subscriptions.segment', 'lists.cid', 'lists.name', 'segments.name', 'namespaces.name']
             );
@@ -156,7 +176,7 @@ async function listTestUsersDTAjax(context, campaignId, params) {
 
 async function rawGetByIdTx(tx, id) {
     const entity = await tx('campaigns').where('campaigns.id', id)
-        .innerJoin('campaign_lists', 'campaigns.id', 'campaign_lists.campaign')
+        .leftJoin('campaign_lists', 'campaigns.id', 'campaign_lists.campaign')
         .groupBy('campaigns.id')
         .select([
             'campaigns.id', 'campaigns.name', 'campaigns.description', 'campaigns.namespace', 'campaigns.status', 'campaigns.type', 'campaigns.source',
@@ -170,12 +190,16 @@ async function rawGetByIdTx(tx, id) {
         throw new interoperableErrors.NotFoundError();
     }
 
-    entity.lists = entity.lists.split(';').map(x => {
-        const entries = x.split(':');
-        const list = Number.parseInt(entries[0]);
-        const segment = entries[1] ? Number.parseInt(entries[1]) : null;
-        return {list, segment};
-    });
+    if (entity.lists) {
+        entity.lists = entity.lists.split(';').map(x => {
+            const entries = x.split(':');
+            const list = Number.parseInt(entries[0]);
+            const segment = entries[1] ? Number.parseInt(entries[1]) : null;
+            return {list, segment};
+        });
+    } else {
+        entity.lists = [];
+    }
 
     entity.data = JSON.parse(entity.data);
 
@@ -546,54 +570,63 @@ async function changeStatusByMessage(context, message, subscriptionStatus, updat
 }
 
 async function getSubscribersQueryGeneratorTx(tx, campaignId, onlyUnsent, batchSize) {
-    const subscriptionsQueries = [];
+    /*
+    This is supposed to produce queries like this:
+
+    select count(*) as `subscriptionsToSend` from `campaign_lists` inner join (
+        select `email`, min(`campaign_list_id`) as `campaign_list_id`, max(`sent`) as `sent` from (
+            (select `subscription__1`.`email`, 2 AS campaign_list_id, campaign_messages.id IS NOT NULL AS sent from `subscription__1` left join `campaign_messages` on
+            `campaign_messages`.`subscription` = `subscription__1`.`id` where `campaign_messages`.`campaign` = 1 and `campaign_messages`.`list` = 1 and `subscription__1`.`status` = 1)
+         UNION ALL
+            (select `subscription__2`.`email`, 4 AS campaign_list_id, campaign_messages.id IS NOT NULL AS sent from `subscription__2` left join `campaign_messages` on
+            `campaign_messages`.`subscription` = `subscription__2`.`id` where `campaign_messages`.`campaign` = 1 and `campaign_messages`.`list` = 2 and `subscription__2`.`status` = 1)
+         )
+         as `pending_subscriptions_all` where `sent` = false group by `email`
+     ) as `pending_subscriptions` on `campaign_lists`.`id` = `pending_subscriptions`.`campaign_list_id` where `campaign_lists`.`campaign` = 1 limit 1
+
+     This was too much for Knex, so we partially construct these queries directly as strings;
+     */
+
+    const subsQrys = [];
     const cpgLists = await tx('campaign_lists').where('campaign', campaignId);
 
     for (const cpgList of cpgLists) {
         const addSegmentQuery = cpgList.segment ? await segments.getQueryGeneratorTx(tx, cpgList.list, cpgList.segment) : () => {};
         const subsTable = subscriptions.getSubscriptionTableName(cpgList.list);
 
-        subscriptionsQueries.push(function() {
-            this.from(subsTable)
-                .leftJoin('campaign_messages', 'campaign_messages.subscription', subsTable + '.id')
-                .where('campaign_messages.campaign', cpgList.campaign)
-                .where('campaign_messages.list', cpgList.list)
-                .where(subsTable + '.status', SubscriptionStatus.SUBSCRIBED)
-                .where(function() {
-                    addSegmentQuery(this);
-                })
-                .select([subsTable + '.email', knex.raw('? AS campaign_list_id', [cpgList.id]), knex.raw('campaign_messages.id IS NOT NULL AS sent')]);
-        });
+        const sqlQry = knex.from(subsTable)
+            .leftJoin('campaign_messages', 'campaign_messages.subscription', subsTable + '.id')
+            .where('campaign_messages.campaign', cpgList.campaign)
+            .where('campaign_messages.list', cpgList.list)
+            .where(subsTable + '.status', SubscriptionStatus.SUBSCRIBED)
+            .where(function() {
+                addSegmentQuery(this);
+            })
+            .select([subsTable + '.email', knex.raw('? AS campaign_list_id', [cpgList.id]), knex.raw('campaign_messages.id IS NOT NULL AS sent')])
+            .toSQL().toNative();
+
+        subsQrys.push(sqlQry);
     }
 
-    if (subscriptionsQueries.length > 0) {
+    if (subsQrys.length > 0) {
+        let subsQry;
+        const unsentWhere = onlyUnsent ? ' where `sent` = false' : '';
+
+        if (subsQrys.length === 1) {
+            const subsUnionSql = '(select `email`, `campaign_list_id`, `sent` from (' + subsQrys[0].sql + ') as `pending_subscriptions_all`' +  unsentWhere + ') as `pending_subscriptions`'
+            subsQry = knex.raw(subsUnionSql, subsQrys[0].bindings);
+
+        } else {
+            const subsUnionSql = '(select `email`, min(`campaign_list_id`) as `campaign_list_id`, max(`sent`) as `sent` from (' +
+                subsQrys.map(qry => '(' + qry.sql + ')').join(' UNION ALL ') +
+                ') as `pending_subscriptions_all`' +  unsentWhere + ' group by `email`) as `pending_subscriptions`';
+            const subsUnionBindings = Array.prototype.concat(...subsQrys.map(qry => qry.bindings));
+            subsQry = knex.raw(subsUnionSql, subsUnionBindings);
+        }
+
         return knx => knx.from('campaign_lists')
             .where('campaign_lists.campaign', campaignId)
-            .innerJoin(
-                function () {
-                    let ret;
-                    if (subscriptionsQueries.length > 1) {
-                        ret = this.unionAll(subscriptionsQueries, true)
-                            .groupBy('email')
-                            .select(['email']).min('campaign_list_id AS campaign_list_id')
-                            .select(['sent']).max('sent AS sent');
-                    } else {
-                        ret = this.from(function () { subscriptionsQueries[0].apply(this); this.as('pending_subscriptions'); })
-                            .select(['email', 'sent', 'campaign_list_id']);
-                    }
-
-                    ret = ret.where('sent', false)
-                        .as('pending_subscriptions');
-
-                    if (batchSize) {
-                        ret = ret.limit(retrieveBatchSize);
-                    }
-
-                    return ret;
-                },
-                'campaign_lists.id',
-                'pending_subscriptions.campaign_list_id'
-            );
+            .innerJoin(subsQry, 'campaign_lists.id', 'pending_subscriptions.campaign_list_id');
 
     } else {
         return null;
@@ -624,7 +657,7 @@ async function _changeStatus(context, campaignId, permittedCurrentStates, newSta
 
 
 async function start(context, campaignId, startAt) {
-    await _changeStatus(context, campaignId, [CampaignStatus.IDLE, CampaignStatus.PAUSED], CampaignStatus.SCHEDULED, 'Cannot start campaign until it is in IDLE or PAUSED state', startAt);
+    await _changeStatus(context, campaignId, [CampaignStatus.IDLE, CampaignStatus.PAUSED, CampaignStatus.FINISHED], CampaignStatus.SCHEDULED, 'Cannot start campaign until it is in IDLE or PAUSED state', startAt);
 }
 
 async function stop(context, campaignId) {
diff --git a/services/importer.js b/services/importer.js
index d47da5ad..6a1088d5 100644
--- a/services/importer.js
+++ b/services/importer.js
@@ -137,7 +137,6 @@ function prepareCsv(impt) {
 }
 
 async function _execImportRun(impt, handlers) {
-    // FIXME - handle STOPPING
     try {
         let imptRun;
 
diff --git a/setup/knex/migrations/20170506102634_v1_to_v2.js b/setup/knex/migrations/20170506102634_v1_to_v2.js
index 87a2355d..ed459b0b 100644
--- a/setup/knex/migrations/20170506102634_v1_to_v2.js
+++ b/setup/knex/migrations/20170506102634_v1_to_v2.js
@@ -1062,7 +1062,7 @@ async function migrateTriggers(knex) {
 
     for (const trigger of triggers) {
         const campaign = await knex('campaigns')
-            .innerJoin('campaign_lists', 'campaigns.id', 'campaign_lists.campaign')
+            .innerJoin('campaign_lists', 'campaigns.id', 'campaign_lists.campaign') // We assume here that every campaign has a list.
             .groupBy('campaigns.id')
             .select(
                 knex.raw(`GROUP_CONCAT(campaign_lists.list SEPARATOR \';\') as lists`)