Merge branch 'master' of github.com:Mailtrain-org/mailtrain into access

Conflicts:
	views/users/api.hbs

.dockerignore

@@ -0,0 +1 @@
+node_modules

.gitignore

@@ -27,3 +27,7 @@ public/grapejs/uploads/*
 public/grapejs/templates/*
 !public/grapejs/templates/demo
 !public/grapejs/templates/aves
+
+config/production.toml
+workers/reports/config/production.toml
+docker-compose.override.yml

Dockerfile

@@ -1,9 +1,13 @@
-FROM centos
-RUN curl --silent --location https://rpm.nodesource.com/setup_7.x | bash -
-RUN yum install -y git make gcc nodejs ImageMagick && yum clean all
-COPY . /app
+FROM node:8.1
+
+# First install dependencies
+COPY ./package.json ./app/
 WORKDIR /app/
 ENV NODE_ENV production
 RUN npm install --no-progress --production && npm install --no-progress passport-ldapjs
+# Later, copy the app files. That improves development speed as buiding the Docker image will not have 
+# to download and install all the NPM dependencies every time there's a change in the source code
+COPY . /app
 EXPOSE 3000
-CMD ["/usr/bin/node", "index.js"]
+ENTRYPOINT ["bash", "/app/docker-entrypoint.sh"]
+CMD ["node", "index.js"]

Gruntfile.js

@@ -5,7 +5,7 @@ module.exports = function (grunt) {
     // Project configuration.
     grunt.initConfig({
         eslint: {
-            all: ['lib/**/*.js', 'test/**/*.js', 'config/**/*.js', 'Gruntfile.js', 'app.js', 'index.js', 'routes/editorapi.js']
+            all: ['lib/**/*.js', 'test/**/*.js', 'config/**/*.js', 'services/**/*.js', 'Gruntfile.js', 'app.js', 'index.js', 'routes/editorapi.js']
         },
 
         nodeunit: {

README.md

@@ -120,77 +120,24 @@ If you are using the bundled ZoneMTA then you should make sure you are using a p
 With proper SPF, DKIM and PTR records (DMARC wouldn't hurt either) I got perfect 10/10 score out from [MailTester](https://www.mail-tester.com/) when sending a campaign message to a MailTester test address. I did not have VERP turned on, so the sender address matched return path address.
 
 ### Simple Install (Docker)
-##### Requirements:
-    * Docker
-    * docker-compose
+#### Requirements:
 
-  1. Download Mailtrain files using git: `git clone git://github.com/Mailtrain-org/mailtrain.git` (or download [zipped repo](https://github.com/Mailtrain-org/mailtrain/archive/master.zip)) and open Mailtrain folder `cd mailtrain`
-  2. Run `sudo docker build -t mailtrain-node:latest .`
-  3. Copy default.toml to production.toml. Run `sudo mkdir -p /etc/mailtrain && sudo cp config/default.toml /etc/mailtrain/production.toml`
-  4. Create `/etc/docker-compose.yml`. Example (dont forget change MYSQL_ROOT_PASS and MYSQL_USER_PASSWORD to your passwords):
-  ```
-  version: '2'
-  services:
-    mailtrain-mysql:
-        image: mysql:latest
-        ports:
-          - "3306:3306"
-        container_name: "mailtrain-mysql"
-        restart: always
-        environment:
-           MYSQL_ROOT_PASSWORD: "MYSQL_ROOT_PASS"
-           MYSQL_DATABASE: "mailtrain"
-           MYSQL_USER: "mailtrain"
-           MYSQL_PASSWORD: "MYSQL_USER_PASSWORD"
-        volumes:
-           - mailtrain-mysq-data:/var/lib/mysql
+  * [Docker](https://www.docker.com/)
+  * [Docker Compose](https://docs.docker.com/compose/)
 
-    mailtrain-redis:
-        image: redis:3.0
-        container_name: "mailtrain-redis"
-        volumes:
-           - mailtrain-redis-data:/data
+#### Install:
 
-    mailtrain-node:
-      image: mailtrain-node:latest
-      container_name: "mailtrain-node"
-      links:
-        - "mailtrain-mysql:mailtrain-mysql"
-        - "mailtrain-redis:mailtrain-redis"
-      ports:
-        - "3000:3000"
-      volumes:
-        - "/etc/mailtrain/production.toml:/app/config/production.toml"
-        - "mailtrain-node-data:/app/public/grapejs/uploads"
-        - "mailtrain-node-data:/app/public/mosaico/uploads"
-  volumes:
-    mailtrain-mysq-data: {}
-    mailtrain-redis-data: {}
-    mailtrain-node-data: {}
+* Download Mailtrain files using git: `git clone git://github.com/Mailtrain-org/mailtrain.git` (or download [zipped repo](https://github.com/Mailtrain-org/mailtrain/archive/master.zip)) and open Mailtrain folder `cd mailtrain`
+* **Note**: depending on how you have configured your system and Docker you may need to prepend the commands below with `sudo`.
+* Copy the file `docker-compose.override.yml.tmpl` to `docker-compose.override.yml.tmpl` and modify it if you need to.
+* Bring up the stack with: `docker-compose up -d`, by default it will use the included `docker-compose.yml` file and override some configurations taken from the `docker-compose.override.yml` file.
+* If you want to use only / copy the `docker-compose.yml` file (for example, if you were deploying with Rancher), you may need to first run `docker-compose build` to make sure your system has a Docker image `mailtrain:latest`.
+* Open [http://localhost:3000/](http://localhost:3000/) (change the host name `localhost` to the name of the host where you are deploying the system).
+* Authenticate as user `admin` with password `test`
+* Navigate to [http://localhost:3000/settings](http://localhost:3000/settings) and update service configuration.
+* Navigate to [http://localhost:3000/users/account](http://localhost:3000/users/account) and update user information and password.
 
-  ```
-  5. Update MySQL and Redis credintial in `/etc/mailtrain/production.toml` like this:
-  ```
-  [mysql]
-  host="mailtrain-mysql"
-  user="mailtrain"
-  password="MYSQL_USER_PASSWORD"
-  database="mailtrain"
-  port=3306
-  charset="utf8mb4"
-  timezone="UTC"
-
-  [redis]
-  enabled=true
-  host="mailtrain-redis"
-  port=6379
-  db=5
-  ```
-  6. Run docker container with command `sudo docker-compose -f /etc/docker-compose.yml up -d`
-  7. Open [http://localhost:3000/](http://localhost:3000/)
-  8. Authenticate as `admin`:`test`
-  9. Navigate to [http://localhost:3000/settings](http://localhost:3000/settings) and update service configuration
-  10. Navigate to [http://localhost:3000/users/account](http://localhost:3000/users/account) and update user information and password
+**Note**: If you need to add or modify custom configurations, copy the file `config/docker-production.toml.tmpl` to `config/production.toml` and modify as you need. By default, the Docker image will do just that, automatically, so you can bring up the stack and it will work with default configurations.
   
 
 ### Manual Install (any OS that supports Node.js)

client/src/account/API.js

@@ -179,6 +179,54 @@ export default class API extends Component {
                 <pre>curl -XPOST {serviceUrl}api/delete/B16uVTdW?access_token={accessToken} \
 --data 'EMAIL=test@example.com'</pre>
 
+                <h3>POST /api/field/:listId – {t('Add new custom field')}</h3>
+
+                <p>
+                    {t('This API call creates a new custom field for a list.')}
+                </p>
+
+                <p>
+                    <strong>GET</strong> {t('arguments')}
+                </p>
+                <ul>
+                    <li><strong>access_token</strong> – {t('your personal access token')}</li>
+                </ul>
+
+                <p>
+                    <strong>POST</strong> {t('arguments')}
+                </p>
+                <ul>
+                    <li><strong>NAME</strong> – {t('field name')} (<em>{t('required')}</em>)</li>
+                    <li><strong>TYPE</strong> – {t('one of the following types:')}
+                        <ul>
+                            <li><strong>text</strong> Text</li>
+                            <li><strong>website</strong> Website</li>
+                            <li><strong>longtext</strong> Multi-line text</li>
+                            <li><strong>gpg</strong> GPG Public Key</li>
+                            <li><strong>number</strong> Number</li>
+                            <li><strong>radio</strong> Radio Buttons</li>
+                            <li><strong>checkbox</strong> Checkboxes</li>
+                            <li><strong>dropdown</strong> Drop Down</li>
+                            <li><strong>date-us</strong> Date (MM/DD/YYY)</li>
+                            <li><strong>date-eur</strong> Date (DD/MM/YYYY)</li>
+                            <li><strong>birthday-us</strong> Birthday (MM/DD)</li>
+                            <li><strong>birthday-eur</strong> Birthday (DD/MM)</li>
+                            <li><strong>json</strong> JSON value for custom rendering</li>
+                            <li><strong>option</strong> Option</li>
+                        </ul>
+                    </li>
+                    <li><strong>GROUP</strong> – {t('If the type is 'option' then you also need to specify the parent element ID')}</li>
+                    <li><strong>GROUP_TEMPLATE</strong> – {t('Template for the group element. If not set, then values of the elements are joined with commas')}</li>
+                    <li><strong>VISIBLE</strong> – yes/no, {t('if not visible then the subscriber can not view or modify this value at the profile page')}</li>
+                </ul>
+
+                <p>
+                    <strong>{t('Example')}</strong>
+                </p>
+
+                <pre>curl -XPOST {{serviceUrl}}api/field/B16uVTdW?access_token={{accessToken}} \
+--data 'NAME=Birthday&amp;TYPE=birthday-us&amp;VISIBLE=yes'</pre>
+
                 <h3>GET /api/blacklist/get – {t('Get list of blacklisted emails')}</h3>
 
                 <p>

config/docker-production.toml.tmpl

@@ -0,0 +1,9 @@
+[mysql]
+host="mysql"
+
+[redis]
+enabled=true
+host="redis"
+
+[reports]
+enabled=true

docker-compose.override.yml.tmpl

@@ -0,0 +1,14 @@
+version: '2'
+services:
+  mysql:
+    restart: always
+  redis:
+    restart: always
+  mailtrain:
+    build: ./
+    # volumes:
+      # - ./:/app
+    ports:
+      - "3000:3000"
+    restart: always
+  

docker-compose.yml

@@ -0,0 +1,31 @@
+version: '2'
+services:
+  mysql:
+    image: mysql:5.7
+    environment:
+      - MYSQL_ROOT_PASSWORD=mailtrain
+      - MYSQL_DATABASE=mailtrain
+      - MYSQL_USER=mailtrain
+      - MYSQL_PASSWORD=mailtrain
+    volumes:
+      - mailtrain-mysq-data:/var/lib/mysql
+  redis:
+    image: redis:3.0
+    volumes:
+      - mailtrain-redis-data:/data
+  mailtrain:
+    image: mailtrain:latest
+    depends_on:
+      - mysql
+      - redis
+    volumes:
+      - mailtrain-node-config:/app/config
+      - mailtrain-node-data:/app/public/grapejs/uploads
+      - mailtrain-node-data:/app/public/mosaico/uploads
+      - mailtrain-node-reports:/app/protected/reports
+volumes:
+  mailtrain-mysq-data: {}
+  mailtrain-redis-data: {}
+  mailtrain-node-data: {}
+  mailtrain-node-config: {}
+  mailtrain-node-reports: {}

docker-entrypoint.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+if [ ! -f "/app/config/production.toml" ] ; then 
+    echo "No production.toml, copying from docker-production.toml.tmpl"
+    cp /app/config/docker-production.toml.tmpl /app/config/production.toml
+fi
+if [ ! -f "/app/workers/reports/config/production.toml" ] ; then 
+    echo "No production.toml for reports"
+    if [ -f "/app/config/production.toml" ] ; then 
+        echo "copying config/production.toml to reports config directory"
+        cp /app/config/production.toml /app/workers/reports/config/production.toml
+    else
+        echo "copying config/docker-production.toml.tmpl to reports config directory as production.toml"
+        cp /app/config/docker-production.toml.tmpl /app/workers/reports/config/production.toml
+    fi
+fi
+exec "$@"

lib/models/triggers.js

@@ -98,7 +98,10 @@ module.exports.getQuery = (id, callback) => {
         }
 
         let limit = 300;
-        let treshold = 3600 * 6; // time..NOW..time+6h, 6 hour window after trigger target to detect it
+
+        // time..NOW..time + 24h, 24 hour window after trigger target to detect it
+        //We need a 24 hour window for triggers as the format for dates added via the API are stored as 00:00:00
+        let treshold = 3600 * 24;
 
         let intervalQuery = (column, seconds, treshold) => column + ' <= NOW() - INTERVAL ' + seconds + ' SECOND AND ' + column + ' >= NOW() - INTERVAL ' + (treshold + seconds) + ' SECOND';
 

routes/editorapi.js

@@ -375,6 +375,7 @@ router.post('/upload', passport.csrfProtection, (req, res) => {
         };
 
         const dirName = getDirName();
+        const serviceUrlParts = url.parse(serviceUrl);
 
         if (dirName === false) {
             return res.status(500).send(_('Invalid resource type or ID'));
@@ -391,7 +392,8 @@ router.post('/upload', passport.csrfProtection, (req, res) => {
             uploadDir: path.join(__dirname, '..', 'public', req.query.editor, 'uploads', dirName),
             uploadUrl: '/' + req.query.editor + '/uploads/' + dirName, // must be root relative
             acceptFileTypes: /\.(gif|jpe?g|png)$/i,
-            hostname: url.parse(serviceUrl).host // include port
+            hostname: serviceUrlParts.host, // include port
+            ssl: serviceUrlParts.protocol === 'https:'
         };
 
         const mockres = httpMocks.createResponse({

services/executor.js

@@ -37,14 +37,14 @@ function spawnProcess(tid, executable, args, outFile, errFile, cwd, uid, gid) {
                 return;
             }
 
-            privilegeHelpers.ensureMailtrainOwner(outFile, (err) => {
+            privilegeHelpers.ensureMailtrainOwner(outFile, err => {
                 if (err) {
-                    log.warn('Executor', 'Cannot change owner of output file of process tid:%s.', tid)
+                    log.warn('Executor', 'Cannot change owner of output file of process tid:%s.', tid);
                 }
 
-                privilegeHelpers.ensureMailtrainOwner(errFile, (err) => {
+                privilegeHelpers.ensureMailtrainOwner(errFile, err => {
                     if (err) {
-                        log.warn('Executor', 'Cannot change owner of error output file of process tid:%s.', tid)
+                        log.warn('Executor', 'Cannot change owner of error output file of process tid:%s.', tid);
                     }
 
                     const options = {
@@ -79,12 +79,12 @@ function spawnProcess(tid, executable, args, outFile, errFile, cwd, uid, gid) {
                         delete processes[tid];
                         log.info('Executor', 'Process tid:%s pid:%s exited with code %s signal %s.', tid, pid, code, signal);
 
-                        fs.close(outFd, (err) => {
+                        fs.close(outFd, err => {
                             if (err) {
                                 log.error('Executor', err);
                             }
 
-                            fs.close(errFd, (err) => {
+                            fs.close(errFd, err => {
                                 if (err) {
                                     log.error('Executor', err);
                                 }

services/feedcheck.js

@@ -135,7 +135,7 @@ function checkEntries(parent, entries, callback) {
                 if (/\[RSS_ENTRY[\w]*\]/i.test(html)) {
                     html = html.replace(/\[RSS_ENTRY\]/, entry.content); //for backward compatibility
                     Object.keys(entry).forEach(key => {
-                        html = html.replace('\[RSS_ENTRY_'+key.toUpperCase()+'\]', entry[key])
+                        html = html.replace('[RSS_ENTRY_' + key.toUpperCase() + ']', entry[key]);
                     });
                 } else {
                     html = entry.content + html;

services/postfix-bounce-server.js

@@ -70,17 +70,40 @@ let server = net.createServer(socket => {
     });
 });
 
+module.exports = callback => {
+    if (!config.postfixbounce.enabled) {
+        return setImmediate(callback);
+    }
+
+    let started = false;
+
     server.on('error', err => {
-    log.error('POSTFIXBOUNCE', err && err.stack);
+        const port = config.postfixbounce.port;
+        const bind = typeof port === 'string' ? 'Pipe ' + port : 'Port ' + port;
+
+        switch (err.code) {
+            case 'EACCES':
+                log.error('POSTFIXBOUNCE', '%s requires elevated privileges.', bind);
+                break;
+            case 'EADDRINUSE':
+                log.error('POSTFIXBOUNCE', '%s is already in use', bind);
+                break;
+            default:
+                log.error('POSTFIXBOUNCE', err);
+        }
+
+        if (!started) {
+            started = true;
+            return callback(err);
+        }
     });
 
-module.exports = callback => {
-    if (config.postfixbounce.enabled) {
     server.listen(config.postfixbounce.port, config.postfixbounce.host, () => {
+        if (started) {
+            return server.close();
+        }
+        started = true;
         log.info('POSTFIXBOUNCE', 'Server listening on port %s', config.postfixbounce.port);
         setImmediate(callback);
     });
-    } else {
-        setImmediate(callback);
-    }
 };

services/test-server.js

@@ -164,6 +164,10 @@ let mailBoxServer = http.createServer((req, res) => {
     });
 });
 
+mailBoxServer.on('error', err => {
+    log.error('Test SMTP Mailbox Server', err);
+});
+
 module.exports = callback => {
     if (config.testserver.enabled) {
         server.listen(config.testserver.port, config.testserver.host, () => {

services/verp-server.js

@@ -29,7 +29,7 @@ let server = new SMTPServer({
             let user = address.address.split('@').shift();
             let host = address.address.split('@').pop();
 
-            if (host !== configItems.verpHostname || !/^[a-z0-9_\-]+\.[a-z0-9_\-]+\.[a-z0-9_\-]+$/i.test(user)) {
+            if (host !== configItems.verpHostname || !/^[a-z0-9_-]+\.[a-z0-9_-]+\.[a-z0-9_-]+$/i.test(user)) {
                 err = new Error('Unknown user ' + address.address);
                 err.responseCode = 510;
                 return callback(err);
@@ -99,15 +99,36 @@ let server = new SMTPServer({
     }
 });
 
-server.on('error', err => {
-    log.error('VERP', err);
-    server.close();
-});
-
 module.exports = callback => {
     if (!config.verp.enabled) {
         return setImmediate(callback);
     }
+
+    let started = false;
+
+    server.on('error', err => {
+        const port = config.verp.port;
+        const bind = typeof port === 'string' ? 'Pipe ' + port : 'Port ' + port;
+
+        switch (err.code) {
+            case 'EACCES':
+                log.error('VERP', '%s requires elevated privileges', bind);
+                break;
+            case 'EADDRINUSE':
+                log.error('VERP', '%s is already in use', bind);
+                break;
+            case 'ECONNRESET': // Usually happens when a client does not disconnect cleanly
+            case 'EPIPE': // Remote connection was closed before the server attempted to send data
+            default:
+                log.error('VERP', err);
+        }
+
+        if (!started) {
+            started = true;
+            return callback(err);
+        }
+    });
+
     let hosts;
     if (typeof config.verp.host === 'string' && config.verp.host) {
         hosts = config.verp.host.trim().split(',').map(host => host.trim()).filter(host => host.trim());
@@ -121,10 +142,14 @@ module.exports = callback => {
     let pos = 0;
     let startNextHost = () => {
         if (pos >= hosts.length) {
+            started = true;
             return setImmediate(callback);
         }
         let host = hosts[pos++];
         server.listen(config.verp.port, host, () => {
+            if (started) {
+                return server.close();
+            }
             log.info('VERP', 'Server listening on %s:%s', host || '*', config.verp.port);
             setImmediate(startNextHost);
         });

views/blacklist.hbs

@@ -8,10 +8,13 @@
 <form class="form-horizontal" method="post" action="/blacklist/ajax/add">
     <input type="hidden" name="_csrf" value="{{csrfToken}}">
     <input type="hidden" name="next" value="/blacklist">
-    <div class="col-sm-4">
+    <div class="form-group">
+        <div class="col-sm-4 col-xs-8">
             <input type="text" class="form-control input-md" name="email" id="add-email-input" value="" placeholder="{{#translate}}Add email to blacklist{{/translate}}">
         </div>
         <button type="submit" class="btn btn-success"><span class="glyphicon glyphicon-plus" aria-hidden="true"></span>{{#translate}}Add{{/translate}}</button>
+    </div>
+    <p class="help-block">A blacklisted email address will not receive messages from any campaign. The blacklist does not apply to transactional messages.</p>
 </form>
 
 <form class="form-horizontal" id="delete-email-form" method="post" action="/blacklist/ajax/delete">