Son Goku

- Rebased sql, test, web, defintions, languages, INSTALL, and libs folders.

libs/.gitignore

@@ -0,0 +1 @@
+customAutoLoad

libs/auth.js

@@ -35,6 +35,10 @@ module.exports = function(s,config,lang){
         //check IP address of connecting user
         var finish=function(user){
             if(s.api[params.auth].ip.indexOf('0.0.0.0')>-1||s.api[params.auth].ip.indexOf(params.ip)>-1){
+                if(!user.lang){
+                    var details = s.parseJSON(user.details).lang
+                    user.lang = s.getDefinitonFile(user.details.lang) || s.copySystemDefaultLanguage()
+                }
                 cb(user);
             }else{
                 failed();
@@ -43,7 +47,10 @@ module.exports = function(s,config,lang){
         //check if auth key is user's temporary session key
         if(s.group[params.ke]&&s.group[params.ke].users&&s.group[params.ke].users[params.auth]){
             s.group[params.ke].users[params.auth].permissions={};
-            cb(s.group[params.ke].users[params.auth]);
+            if(!s.group[params.ke].users[params.auth].lang){
+                s.group[params.ke].users[params.auth].lang = s.copySystemDefaultLanguage()
+            }
+            cb(s.group[params.ke].users[params.auth])
         }else{
             //check if key is already in memory to save query time
             if(s.api[params.auth]&&s.api[params.auth].details){
@@ -185,6 +192,10 @@ module.exports = function(s,config,lang){
         if(userFound === true){
             return true
         }else{
+            if(res)res.end(s.prettyPrint({
+                ok: false,
+                msg: lang['Not Authorized']
+            }))
             return false
         }
     }

libs/basic.js

@@ -12,7 +12,7 @@ module.exports = function(s,config){
         if(s.isWin===true){
             cmd = "Taskkill /IM ffmpeg.exe /F"
         }else{
-            cmd = "ps aux | grep -ie ffmpeg | awk '{print $2}' | xargs kill -9"
+            cmd = "pkill -9 ffmpeg"
         }
         exec(cmd,{detached: true})
     };
@@ -31,12 +31,14 @@ module.exports = function(s,config){
         }
     }
     s.parseJSON = function(string){
+        var parsed
         try{
-            string = JSON.parse(string)
+            parsed = JSON.parse(string)
         }catch(err){
 
         }
-        return string
+        if(!parsed)parsed = string
+        return parsed
     }
     s.stringJSON = function(json){
         try{
@@ -224,4 +226,30 @@ module.exports = function(s,config){
             break;
         }
     }
+    s.createTimeout = function(timeoutVar,timeoutLength,defaultLength,multiplier,callback){
+        var theTimeout
+        if(!multiplier)multiplier = 1000 * 60
+        if(!timeoutLength || timeoutLength === ''){
+            theTimeout = defaultLength
+        }else{
+            theTimeout = parseFloat(timeoutLength) * multiplier
+        }
+        clearTimeout(timeoutVar)
+        timeoutVar = setTimeout(function(){
+            clearTimeout(timeoutVar)
+            delete(timeoutVar)
+            if(callback)callback()
+        },theTimeout)
+    }
+    Object.defineProperty(Array.prototype, 'chunk', {
+        value: function(chunkSize){
+            var temporal = [];
+
+            for (var i = 0; i < this.length; i+= chunkSize){
+                temporal.push(this.slice(i,i+chunkSize));
+            }
+
+            return temporal;
+        }
+    });
 }

libs/cloudUploaders.js

@@ -2,7 +2,26 @@ var fs = require('fs');
 var exec = require('child_process').exec;
 var spawn = require('child_process').spawn;
 var webdav = require("webdav-fs");
+var ssh2SftpClient = require('node-ssh')
 module.exports = function(s,config,lang){
+    var addCloudUploader = function(opt){
+        s.loadGroupAppExtender(opt.loadGroupAppExtender)
+        s.unloadGroupAppExtender(opt.unloadGroupAppExtender)
+        s.insertCompletedVideoExtender(opt.insertCompletedVideoExtender)
+        s.deleteVideoFromCloudExtensions[opt.name] = opt.deleteVideoFromCloudExtensions
+        s.cloudDiskUseStartupExtensions[opt.name] = opt.cloudDiskUseStartupExtensions
+        s.beforeAccountSave(opt.beforeAccountSave)
+        s.onAccountSave(opt.onAccountSave)
+        s.cloudDisksLoader(opt.name)
+    }
+    var addSimpleUploader = function(opt){
+        s.loadGroupAppExtender(opt.loadGroupAppExtender)
+        s.unloadGroupAppExtender(opt.unloadGroupAppExtender)
+        s.insertCompletedVideoExtender(opt.insertCompletedVideoExtender)
+        s.beforeAccountSave(opt.beforeAccountSave)
+        s.onAccountSave(opt.onAccountSave)
+        s.onMonitorSave(opt.onMonitorSave)
+    }
     // WebDAV
     var beforeAccountSaveForWebDav = function(d){
         //d = save event
@@ -20,33 +39,33 @@ module.exports = function(s,config,lang){
     }
     var loadWebDavForUser = function(e){
         // e = user
-        var ar = JSON.parse(e.details);
-        if(ar.webdav_use_global === '1' && config.cloudUploaders && config.cloudUploaders.WebDAV){
+        var userDetails = JSON.parse(e.details);
+        if(userDetails.webdav_use_global === '1' && config.cloudUploaders && config.cloudUploaders.WebDAV){
             // {
             //     webdav_user: "",
             //     webdav_pass: "",
             //     webdav_url: "",
             //     webdav_dir: "",
             // }
-            ar = Object.assign(ar,config.cloudUploaders.WebDAV)
+            userDetails = Object.assign(userDetails,config.cloudUploaders.WebDAV)
         }
         //owncloud/webdav
         if(!s.group[e.ke].webdav &&
-           ar.webdav_user&&
-           ar.webdav_user!==''&&
-           ar.webdav_pass&&
-           ar.webdav_pass!==''&&
-           ar.webdav_url&&
-           ar.webdav_url!==''
+           userDetails.webdav_user&&
+           userDetails.webdav_user!==''&&
+           userDetails.webdav_pass&&
+           userDetails.webdav_pass!==''&&
+           userDetails.webdav_url&&
+           userDetails.webdav_url!==''
           ){
-            if(!ar.webdav_dir||ar.webdav_dir===''){
-                ar.webdav_dir='/'
+            if(!userDetails.webdav_dir||userDetails.webdav_dir===''){
+                userDetails.webdav_dir='/'
             }
-            ar.webdav_dir = s.checkCorrectPathEnding(ar.webdav_dir)
+            userDetails.webdav_dir = s.checkCorrectPathEnding(userDetails.webdav_dir)
             s.group[e.ke].webdav = webdav(
-                ar.webdav_url,
-                ar.webdav_user,
-                ar.webdav_pass
+                userDetails.webdav_url,
+                userDetails.webdav_user,
+                userDetails.webdav_pass
             )
         }
     }
@@ -174,8 +193,8 @@ module.exports = function(s,config,lang){
     }
     var loadAmazonS3ForUser = function(e){
         // e = user
-        var ar = JSON.parse(e.details)
-        if(ar.aws_use_global === '1' && config.cloudUploaders && config.cloudUploaders.AmazonS3){
+        var userDetails = JSON.parse(e.details)
+        if(userDetails.aws_use_global === '1' && config.cloudUploaders && config.cloudUploaders.AmazonS3){
             // {
             //     aws_accessKeyId: "",
             //     aws_secretAccessKey: "",
@@ -183,30 +202,30 @@ module.exports = function(s,config,lang){
             //     aws_s3_bucket: "",
             //     aws_s3_dir: "",
             // }
-            ar = Object.assign(ar,config.cloudUploaders.AmazonS3)
+            userDetails = Object.assign(userDetails,config.cloudUploaders.AmazonS3)
         }
         //Amazon S3
         if(!s.group[e.ke].aws &&
            !s.group[e.ke].aws_s3 &&
-           ar.aws_s3 !== '0' &&
-           ar.aws_accessKeyId !== ''&&
-           ar.aws_secretAccessKey &&
-           ar.aws_secretAccessKey !== ''&&
-           ar.aws_region &&
-           ar.aws_region !== ''&&
-           ar.aws_s3_bucket !== ''
+           userDetails.aws_s3 !== '0' &&
+           userDetails.aws_accessKeyId !== ''&&
+           userDetails.aws_secretAccessKey &&
+           userDetails.aws_secretAccessKey !== ''&&
+           userDetails.aws_region &&
+           userDetails.aws_region !== ''&&
+           userDetails.aws_s3_bucket !== ''
           ){
-            if(!ar.aws_s3_dir || ar.aws_s3_dir === '/'){
-                ar.aws_s3_dir = ''
+            if(!userDetails.aws_s3_dir || userDetails.aws_s3_dir === '/'){
+                userDetails.aws_s3_dir = ''
             }
-            if(ar.aws_s3_dir !== ''){
-                ar.aws_s3_dir = s.checkCorrectPathEnding(ar.aws_s3_dir)
+            if(userDetails.aws_s3_dir !== ''){
+                userDetails.aws_s3_dir = s.checkCorrectPathEnding(userDetails.aws_s3_dir)
             }
             s.group[e.ke].aws = new require("aws-sdk")
             s.group[e.ke].aws.config = new s.group[e.ke].aws.Config({
-                accessKeyId: ar.aws_accessKeyId,
-                secretAccessKey: ar.aws_secretAccessKey,
-                region: ar.aws_region
+                accessKeyId: userDetails.aws_accessKeyId,
+                secretAccessKey: userDetails.aws_secretAccessKey,
+                region: userDetails.aws_region
             })
             s.group[e.ke].aws_s3 = new s.group[e.ke].aws.S3();
         }
@@ -296,63 +315,67 @@ module.exports = function(s,config,lang){
         }
     }
     var loadBackblazeB2ForUser = function(e){
-        var ar = JSON.parse(e.details);
+        var userDetails = JSON.parse(e.details);
         try{
-            if(ar.b2_use_global === '1' && config.cloudUploaders && config.cloudUploaders.BackblazeB2){
+            if(userDetails.b2_use_global === '1' && config.cloudUploaders && config.cloudUploaders.BackblazeB2){
                 // {
                 //     bb_b2_accountId: "",
                 //     bb_b2_applicationKey: "",
                 //     bb_b2_bucket: "",
                 //     bb_b2_dir: "",
                 // }
-                ar = Object.assign(ar,config.cloudUploaders.BackblazeB2)
+                userDetails = Object.assign(userDetails,config.cloudUploaders.BackblazeB2)
             }
             if(!s.group[e.ke].bb_b2 &&
-               ar.bb_b2_accountId &&
-               ar.bb_b2_accountId !=='' &&
-               ar.bb_b2_applicationKey &&
-               ar.bb_b2_applicationKey !=='' &&
-               ar.bb_b2_bucket &&
-               ar.bb_b2_bucket !== ''
+               userDetails.bb_b2_accountId &&
+               userDetails.bb_b2_accountId !=='' &&
+               userDetails.bb_b2_applicationKey &&
+               userDetails.bb_b2_applicationKey !=='' &&
+               userDetails.bb_b2_bucket &&
+               userDetails.bb_b2_bucket !== ''
               ){
                 var B2 = require('backblaze-b2')
-                if(!ar.bb_b2_dir || ar.bb_b2_dir === '/'){
-                  ar.bb_b2_dir = ''
+                if(!userDetails.bb_b2_dir || userDetails.bb_b2_dir === '/'){
+                  userDetails.bb_b2_dir = ''
                 }
-                if(ar.bb_b2_dir !== ''){
-                  ar.bb_b2_dir = s.checkCorrectPathEnding(ar.bb_b2_dir)
+                if(userDetails.bb_b2_dir !== ''){
+                  userDetails.bb_b2_dir = s.checkCorrectPathEnding(userDetails.bb_b2_dir)
                 }
-                var b2 = new B2({
-                    accountId: ar.bb_b2_accountId,
-                    applicationKey: ar.bb_b2_applicationKey
-                })
-                s.group[e.ke].bb_b2 = b2
                 var backblazeErr = function(err){
                     // console.log(err)
-                    s.userLog({mid:'$USER',ke:e.ke},{type:lang['Backblaze Error'],msg:err.data})
+                    s.userLog({mid:'$USER',ke:e.ke},{type:lang['Backblaze Error'],msg:err.data || err})
                 }
-                b2.authorize().then(function(resp){
-                    s.group[e.ke].bb_b2_downloadUrl = resp.data.downloadUrl
-                    b2.listBuckets().then(function(resp){
-                        var buckets = resp.data.buckets
-                        var bucketN = -2
-                        buckets.forEach(function(item,n){
-                            if(item.bucketName === ar.bb_b2_bucket){
-                                bucketN = n
+                var createB2Connection = function(){
+                    var b2 = new B2({
+                        accountId: userDetails.bb_b2_accountId,
+                        applicationKey: userDetails.bb_b2_applicationKey
+                    })
+                    b2.authorize().then(function(resp){
+                        s.group[e.ke].bb_b2_downloadUrl = resp.data.downloadUrl
+                        b2.listBuckets().then(function(resp){
+                            var buckets = resp.data.buckets
+                            var bucketN = -2
+                            buckets.forEach(function(item,n){
+                                if(item.bucketName === userDetails.bb_b2_bucket){
+                                    bucketN = n
+                                }
+                            })
+                            if(bucketN > -1){
+                                s.group[e.ke].bb_b2_bucketId = buckets[bucketN].bucketId
+                            }else{
+                                b2.createBucket(
+                                    userDetails.bb_b2_bucket,
+                                    'allPublic'
+                                ).then(function(resp){
+                                    s.group[e.ke].bb_b2_bucketId = resp.data.bucketId
+                                }).catch(backblazeErr)
                             }
-                        })
-                        if(bucketN > -1){
-                            s.group[e.ke].bb_b2_bucketId = buckets[bucketN].bucketId
-                        }else{
-                            b2.createBucket(
-                                ar.bb_b2_bucket,
-                                'allPublic'
-                            ).then(function(resp){
-                                s.group[e.ke].bb_b2_bucketId = resp.data.bucketId
-                            }).catch(backblazeErr)
-                        }
+                        }).catch(backblazeErr)
                     }).catch(backblazeErr)
-                }).catch(backblazeErr)
+                    s.group[e.ke].bb_b2 = b2
+                }
+                createB2Connection()
+                s.group[e.ke].bb_b2_refreshTimer = setTimeout(createB2Connection,1000 * 60 * 60)
             }
         }catch(err){
             s.debugLog(err)
@@ -360,6 +383,7 @@ module.exports = function(s,config,lang){
     }
     var unloadBackblazeB2ForUser = function(user){
         s.group[user.ke].bb_b2 = null
+        clearTimeout(s.group[user.ke].bb_b2_refreshTimer)
     }
     var deleteVideoFromBackblazeB2 = function(e,video,callback){
         // e = user
@@ -431,156 +455,259 @@ module.exports = function(s,config,lang){
             })
         }
     }
+    //Wasabi Hot Cloud Storage
+    var beforeAccountSaveForWasabiHotCloudStorage = function(d){
+        //d = save event
+        d.form.details.whcs_use_global=d.d.whcs_use_global
+        d.form.details.use_whcs=d.d.use_whcs
+    }
+    var cloudDiskUseStartupForWasabiHotCloudStorage = function(group,userDetails){
+        group.cloudDiskUse['whcs'].name = 'Wasabi Hot Cloud Storage'
+        group.cloudDiskUse['whcs'].sizeLimitCheck = (userDetails.use_whcs_size_limit === '1')
+        if(!userDetails.whcs_size_limit || userDetails.whcs_size_limit === ''){
+            group.cloudDiskUse['whcs'].sizeLimit = 10000
+        }else{
+            group.cloudDiskUse['whcs'].sizeLimit = parseFloat(userDetails.whcs_size_limit)
+        }
+    }
+    var loadWasabiHotCloudStorageForUser = function(e){
+        // e = user
+        var userDetails = JSON.parse(e.details)
+        if(userDetails.whcs_use_global === '1' && config.cloudUploaders && config.cloudUploaders.WasabiHotCloudStorage){
+            // {
+            //     whcs_accessKeyId: "",
+            //     whcs_secretAccessKey: "",
+            //     whcs_region: "",
+            //     whcs_bucket: "",
+            //     whcs_dir: "",
+            // }
+            userDetails = Object.assign(userDetails,config.cloudUploaders.WasabiHotCloudStorage)
+        }
+        //Wasabi Hot Cloud Storage
+        if(!s.group[e.ke].whcs &&
+           userDetails.whcs !== '0' &&
+           userDetails.whcs_accessKeyId !== ''&&
+           userDetails.whcs_secretAccessKey &&
+           userDetails.whcs_secretAccessKey !== ''&&
+           userDetails.whcs_region &&
+           userDetails.whcs_region !== ''&&
+           userDetails.whcs_bucket !== ''
+          ){
+            if(!userDetails.whcs_dir || userDetails.whcs_dir === '/'){
+                userDetails.whcs_dir = ''
+            }
+            if(userDetails.whcs_dir !== ''){
+                userDetails.whcs_dir = s.checkCorrectPathEnding(userDetails.whcs_dir)
+            }
+            var AWS = new require("aws-sdk")
+            s.group[e.ke].whcs = AWS
+            var wasabiEndpoint = new AWS.Endpoint('s3.wasabisys.com')
+            s.group[e.ke].whcs.config = new s.group[e.ke].whcs.Config({
+                endpoint: wasabiEndpoint,
+                accessKeyId: userDetails.whcs_accessKeyId,
+                secretAccessKey: userDetails.whcs_secretAccessKey,
+                region: userDetails.whcs_region
+            })
+            s.group[e.ke].whcs = new s.group[e.ke].whcs.S3();
+        }
+    }
+    var unloadWasabiHotCloudStorageForUser = function(user){
+        s.group[user.ke].whcs = null
+    }
+    var deleteVideoFromWasabiHotCloudStorage = function(e,video,callback){
+        // e = user
+        try{
+            var videoDetails = JSON.parse(video.details)
+        }catch(err){
+            var videoDetails = video.details
+        }
+        if(!videoDetails.location){
+            videoDetails.location = video.href.split('wasabisys.com')[1]
+        }
+        s.group[e.ke].whcs.deleteObject({
+            Bucket: s.group[e.ke].init.whcs_bucket,
+            Key: videoDetails.location,
+        }, function(err, data) {
+            if (err) console.log(err);
+            callback()
+        });
+    }
+    var uploadVideoToWasabiHotCloudStorage = function(e,k){
+        //e = video object
+        //k = temporary values
+        if(!k)k={};
+        //cloud saver - Wasabi Hot Cloud Storage
+        if(s.group[e.ke].whcs && s.group[e.ke].init.use_whcs !== '0' && s.group[e.ke].init.whcs_save === '1'){
+            var ext = k.filename.split('.')
+            ext = ext[ext.length - 1]
+            var fileStream = fs.createReadStream(k.dir+k.filename);
+            fileStream.on('error', function (err) {
+                console.error(err)
+            })
+            var saveLocation = s.group[e.ke].init.whcs_dir+e.ke+'/'+e.mid+'/'+k.filename
+            s.group[e.ke].whcs.upload({
+                Bucket: s.group[e.ke].init.whcs_bucket,
+                Key: saveLocation,
+                Body:fileStream,
+                ACL:'public-read',
+                ContentType:'video/'+ext
+            },function(err,data){
+                if(err){
+                    s.userLog(e,{type:lang['Wasabi Hot Cloud Storage Upload Error'],msg:err})
+                }
+                if(s.group[e.ke].init.whcs_log === '1' && data && data.Location){
+                    var save = [
+                        e.mid,
+                        e.ke,
+                        k.startTime,
+                        1,
+                        s.s({
+                            type : 'whcs',
+                            location : saveLocation
+                        }),
+                        k.filesize,
+                        k.endTime,
+                        data.Location
+                    ]
+                    s.sqlQuery('INSERT INTO `Cloud Videos` (mid,ke,time,status,details,size,end,href) VALUES (?,?,?,?,?,?,?,?)',save)
+                    s.setCloudDiskUsedForGroup(e,{
+                        amount : k.filesizeMB,
+                        storageType : 'whcs'
+                    })
+                    s.purgeCloudDiskForGroup(e,'whcs')
+                }
+            })
+        }
+    }
     //SFTP
-    // var beforeAccountSaveForSftp = function(d){
-    //     //d = save event
-    //     d.form.details.use_sftp = d.d.use_sftp
-    // }
-    // var cloudDiskUseStartupForSftp = function(group,userDetails){
-    //     group.cloudDiskUse['sftp'].name = 'SFTP'
-    //     group.cloudDiskUse['sftp'].sizeLimitCheck = (userDetails.use_aws_s3_size_limit === '1')
-    //     if(!userDetails.aws_s3_size_limit || userDetails.aws_s3_size_limit === ''){
-    //         group.cloudDiskUse['sftp'].sizeLimit = 10000
-    //     }else{
-    //         group.cloudDiskUse['sftp'].sizeLimit = parseFloat(userDetails.aws_s3_size_limit)
-    //     }
-    // }
-    // var loadSftpForUser = function(e){
-    //     // e = user
-    //     var ar = JSON.parse(e.details);
-    //     //SFTP
-    //     if(!s.group[e.ke].sftp &&
-    //        !s.group[e.ke].sftp &&
-    //        ar.sftp !== '0' &&
-    //        ar.sftp_accessKeyId !== ''&&
-    //        ar.sftp_secretAccessKey &&
-    //        ar.sftp_secretAccessKey !== ''&&
-    //        ar.sftp_region &&
-    //        ar.sftp_region !== ''&&
-    //        ar.sftp_bucket !== ''
-    //       ){
-    //         if(!ar.sftp_dir || ar.sftp_dir === '/'){
-    //             ar.sftp_dir = ''
-    //         }
-    //         if(ar.sftp_dir !== ''){
-    //             ar.sftp_dir = s.checkCorrectPathEnding(ar.sftp_dir)
-    //         }
-    //         s.group[e.ke].sftp = new s.group[e.ke].sftp.S3();
-    //         s.group[e.ke].sftp = new require('ssh2-sftp-client')();
-    //         var connectionDetails = {
-    //             host: ar.sftp_host,
-    //             port: ar.sftp_port
-    //         }
-    //         if(!ar.sftp_port)ar.sftp_port = 22
-    //         if(ar.sftp_username)connectionDetails.username = ar.sftp_username
-    //         if(ar.sftp_password)connectionDetails.password = ar.sftp_password
-    //         if(ar.sftp_privateKey)connectionDetails.privateKey = ar.sftp_privateKey
-    //         sftp.connect(connectionDetails).then(() => {
-    //             return sftp.list('/pathname');
-    //         }).then((data) => {
-    //             console.log(data, 'the data info');
-    //         }).catch((err) => {
-    //             console.log(err, 'catch error');
-    //         });
-    //     }
-    // }
-    // var unloadSftpForUser = function(user){
-    //     s.group[user.ke].sftp = null
-    // }
-    // var deleteVideoFromSftp = function(e,video,callback){
-    //     // e = user
-    //     try{
-    //         var videoDetails = JSON.parse(video.details)
-    //     }catch(err){
-    //         var videoDetails = video.details
-    //     }
-    //     s.group[e.ke].sftp.deleteObject({
-    //         Bucket: s.group[e.ke].init.sftp_bucket,
-    //         Key: videoDetails.location,
-    //     }, function(err, data) {
-    //         if (err) console.log(err);
-    //         callback()
-    //     });
-    // }
-    // var uploadVideoToSftp = function(e,k){
-    //     //e = video object
-    //     //k = temporary values
-    //     if(!k)k={};
-    //     //cloud saver - SFTP
-    //     if(s.group[e.ke].sftp && s.group[e.ke].init.use_sftp !== '0' && s.group[e.ke].init.sftp_save === '1'){
-    //         var fileStream = fs.createReadStream(k.dir+k.filename);
-    //         fileStream.on('error', function (err) {
-    //             console.error(err)
-    //         })
-    //         var saveLocation = s.group[e.ke].init.sftp_dir+e.ke+'/'+e.mid+'/'+k.filename
-    //         s.group[e.ke].sftp.upload({
-    //             Bucket: s.group[e.ke].init.sftp_bucket,
-    //             Key: saveLocation,
-    //             Body:fileStream,
-    //             ACL:'public-read'
-    //         },function(err,data){
-    //             if(err){
-    //                 s.userLog(e,{type:lang['SFTP Upload Error'],msg:err})
-    //             }
-    //             if(s.group[e.ke].init.sftp_log === '1' && data && data.Location){
-    //                 var save = [
-    //                     e.mid,
-    //                     e.ke,
-    //                     k.startTime,
-    //                     1,
-    //                     s.s({
-    //                         type : 'sftp',
-    //                         location : saveLocation
-    //                     }),
-    //                     k.filesize,
-    //                     k.endTime,
-    //                     data.Location
-    //                 ]
-    //                 s.sqlQuery('INSERT INTO `Cloud Videos` (mid,ke,time,status,details,size,end,href) VALUES (?,?,?,?,?,?,?,?)',save)
-    //                 s.setCloudDiskUsedForGroup(e,{
-    //                     amount : k.filesizeMB,
-    //                     storageType : 'sftp'
-    //                 })
-    //                 s.purgeCloudDiskForGroup(e,'sftp')
-    //             }
-    //         })
-    //     }
-    // }
+    var sftpErr = function(err){
+        // console.log(err)
+        s.userLog({mid:'$USER',ke:e.ke},{type:lang['SFTP Error'],msg:err.data || err})
+    }
+    var beforeAccountSaveForSftp = function(d){
+        //d = save event
+        d.form.details.use_sftp = d.d.use_sftp
+    }
+    var loadSftpForUser = function(e){
+        // e = user
+        var userDetails = JSON.parse(e.details);
+        //SFTP
+        if(!s.group[e.ke].sftp &&
+            !s.group[e.ke].sftp &&
+            userDetails.sftp !== '0' &&
+            userDetails.sftp_host &&
+            userDetails.sftp_host !== ''&&
+            userDetails.sftp_port &&
+            userDetails.sftp_port !== ''
+          ){
+            if(!userDetails.sftp_dir || userDetails.sftp_dir === '/'){
+                userDetails.sftp_dir = ''
+            }
+            if(userDetails.sftp_dir !== ''){
+                userDetails.sftp_dir = s.checkCorrectPathEnding(userDetails.sftp_dir)
+            }
+            var sftp = new ssh2SftpClient()
+            var connectionDetails = {
+                host: userDetails.sftp_host,
+                port: userDetails.sftp_port
+            }
+            if(!userDetails.sftp_port)connectionDetails.port = 22
+            if(userDetails.sftp_username && userDetails.sftp_username !== '')connectionDetails.username = userDetails.sftp_username
+            if(userDetails.sftp_password && userDetails.sftp_password !== '')connectionDetails.password = userDetails.sftp_password
+            if(userDetails.sftp_privateKey && userDetails.sftp_privateKey !== '')connectionDetails.privateKey = userDetails.sftp_privateKey
+            sftp.connect(connectionDetails).catch(sftpErr)
+            s.group[e.ke].sftp = sftp
+        }
+    }
+    var unloadSftpForUser = function(user){
+        if(s.group[user.ke].sftp && s.group[user.ke].sftp.end)s.group[user.ke].sftp.end().then(function(){
+            s.group[user.ke].sftp = null
+        })
+    }
+    var uploadVideoToSftp = function(e,k){
+        //e = video object
+        //k = temporary values
+        if(!k)k={};
+        //cloud saver - SFTP
+        if(s.group[e.ke].sftp && s.group[e.ke].init.use_sftp !== '0' && s.group[e.ke].init.sftp_save === '1'){
+            var localPath = k.dir + k.filename
+            var saveLocation = s.group[e.ke].init.sftp_dir + e.ke + '/' + e.mid + '/' + k.filename
+            s.group[e.ke].sftp.putFile(localPath, saveLocation).catch(sftpErr)
+        }
+    }
+    var createSftpDirectory = function(monitorConfig){
+        var monitorSaveDirectory = s.group[monitorConfig.ke].init.sftp_dir + monitorConfig.ke + '/' + monitorConfig.mid
+        s.group[monitorConfig.ke].sftp.mkdir(monitorSaveDirectory, true).catch(function(err){
+            if(err.code !== 'ERR_ASSERTION'){
+                sftpErr(err)
+            }
+        })
+    }
+    var onMonitorSaveForSftp = function(monitorConfig){
+        if(s.group[monitorConfig.ke].sftp && s.group[monitorConfig.ke].init.use_sftp !== '0' && s.group[monitorConfig.ke].init.sftp_save === '1'){
+            createSftpDirectory(monitorConfig)
+        }
+    }
+    var onAccountSaveForSftp = function(group,userDetails,user){
+        if(s.group[user.ke] && s.group[user.ke].sftp && s.group[user.ke].init.use_sftp !== '0' && s.group[user.ke].init.sftp_save === '1'){
+            Object.keys(s.group[user.ke].mon_conf).forEach(function(monitorId){
+                createSftpDirectory(s.group[user.ke].mon_conf[monitorId])
+            })
+        }
+    }
     //add the extenders
     //webdav
-    s.loadGroupAppExtender(loadWebDavForUser)
-    s.unloadGroupAppExtender(unloadWebDavForUser)
-    s.insertCompletedVideoExtender(uploadVideoToWebDav)
-    s.deleteVideoFromCloudExtensions['webdav'] = deleteVideoFromWebDav
-    s.cloudDiskUseStartupExtensions['webdav'] = cloudDiskUseStartupForWebDav
-    s.beforeAccountSave(beforeAccountSaveForWebDav)
-    s.onAccountSave(cloudDiskUseStartupForWebDav)
-    s.cloudDisksLoader('webdav')
+    addCloudUploader({
+        name: 'webdav',
+        loadGroupAppExtender: loadWebDavForUser,
+        unloadGroupAppExtender: unloadWebDavForUser,
+        insertCompletedVideoExtender: uploadVideoToWebDav,
+        deleteVideoFromCloudExtensions: deleteVideoFromWebDav,
+        cloudDiskUseStartupExtensions: cloudDiskUseStartupForWebDav,
+        beforeAccountSave: beforeAccountSaveForWebDav,
+        onAccountSave: cloudDiskUseStartupForWebDav,
+    })
     //amazon s3
-    s.loadGroupAppExtender(loadAmazonS3ForUser)
-    s.unloadGroupAppExtender(unloadAmazonS3ForUser)
-    s.insertCompletedVideoExtender(uploadVideoToAmazonS3)
-    s.deleteVideoFromCloudExtensions['s3'] = deleteVideoFromAmazonS3
-    s.cloudDiskUseStartupExtensions['s3'] = cloudDiskUseStartupForAmazonS3
-    s.beforeAccountSave(beforeAccountSaveForAmazonS3)
-    s.onAccountSave(cloudDiskUseStartupForAmazonS3)
-    s.cloudDisksLoader('s3')
+    addCloudUploader({
+        name: 's3',
+        loadGroupAppExtender: loadAmazonS3ForUser,
+        unloadGroupAppExtender: unloadAmazonS3ForUser,
+        insertCompletedVideoExtender: uploadVideoToAmazonS3,
+        deleteVideoFromCloudExtensions: deleteVideoFromAmazonS3,
+        cloudDiskUseStartupExtensions: cloudDiskUseStartupForAmazonS3,
+        beforeAccountSave: beforeAccountSaveForAmazonS3,
+        onAccountSave: cloudDiskUseStartupForAmazonS3,
+    })
     //backblaze b2
-    s.loadGroupAppExtender(loadBackblazeB2ForUser)
-    s.unloadGroupAppExtender(unloadBackblazeB2ForUser)
-    s.insertCompletedVideoExtender(uploadVideoToBackblazeB2)
-    s.deleteVideoFromCloudExtensions['b2'] = deleteVideoFromBackblazeB2
-    s.cloudDiskUseStartupExtensions['b2'] = cloudDiskUseStartupForBackblazeB2
-    s.beforeAccountSave(beforeAccountSaveForBackblazeB2)
-    s.onAccountSave(cloudDiskUseStartupForBackblazeB2)
-    s.cloudDisksLoader('b2')
-    //SFTP
-    // s.loadGroupAppExtender(loadSftpForUser)
-    // s.unloadGroupAppExtender(unloadSftpForUser)
-    // s.insertCompletedVideoExtender(uploadVideoToSftp)
-    // s.deleteVideoFromCloudExtensions['sftp'] = deleteVideoFromSftp
-    // s.cloudDiskUseStartupExtensions['sftp'] = cloudDiskUseStartupForSftp
-    // s.beforeAccountSave(beforeAccountSaveForSftp)
-    // s.onAccountSave(cloudDiskUseStartupForSftp)
-    // s.cloudDisksLoader('sftp')
+    addCloudUploader({
+        name: 'b2',
+        loadGroupAppExtender: loadBackblazeB2ForUser,
+        unloadGroupAppExtender: unloadBackblazeB2ForUser,
+        insertCompletedVideoExtender: uploadVideoToBackblazeB2,
+        deleteVideoFromCloudExtensions: deleteVideoFromBackblazeB2,
+        cloudDiskUseStartupExtensions: cloudDiskUseStartupForBackblazeB2,
+        beforeAccountSave: beforeAccountSaveForBackblazeB2,
+        onAccountSave: cloudDiskUseStartupForBackblazeB2,
+    })
+    //wasabi
+    addCloudUploader({
+        name: 'whcs',
+        loadGroupAppExtender: loadWasabiHotCloudStorageForUser,
+        unloadGroupAppExtender: unloadWasabiHotCloudStorageForUser,
+        insertCompletedVideoExtender: uploadVideoToWasabiHotCloudStorage,
+        deleteVideoFromCloudExtensions: deleteVideoFromWasabiHotCloudStorage,
+        cloudDiskUseStartupExtensions: cloudDiskUseStartupForWasabiHotCloudStorage,
+        beforeAccountSave: beforeAccountSaveForWasabiHotCloudStorage,
+        onAccountSave: cloudDiskUseStartupForWasabiHotCloudStorage,
+    })
+    //SFTP (Simple Uploader)
+    addSimpleUploader({
+        name: 'sftp',
+        loadGroupAppExtender: loadSftpForUser,
+        unloadGroupAppExtender: unloadSftpForUser,
+        insertCompletedVideoExtender: uploadVideoToSftp,
+        beforeAccountSave: beforeAccountSaveForSftp,
+        onAccountSave: onAccountSaveForSftp,
+        onMonitorSave: onMonitorSaveForSftp,
+    })
 }

libs/codeTester.js

@@ -0,0 +1,67 @@
+var fs = require('fs');
+var execSync = require('child_process').execSync;
+module.exports = function(s,config,lang){
+    var onFFmpegLoaded = function(ffmpeg){
+        if(process.argv[2] && process.argv[2].indexOf('test') > -1){
+            config.testMode = true
+        }
+        if(config.testMode === true){
+            config.videosDir = s.mainDirectory + '/videosTest/'
+            config.port = 9999
+            if(config.childNodes && config.childNodes.enabled === true && config.childNodes.mode === 'master'){
+                config.childNodes.port = 9998
+            }
+            s.ffmpegFunctions = ffmpeg
+        }
+    }
+    var onBeforeDatabaseLoad = function(ffmpeg){
+        if(config.testMode === true){
+            try{
+                execSync('rm ' + s.mainDirectory + '/shinobi-test.sqlite')
+            }catch(err){
+
+            }
+            try{
+                require('sqlite3')
+            }catch(err){
+                execSync('npm install sqlite3 --unsafe-perm')
+            }
+            execSync('cp ' + s.mainDirectory + '/sql/shinobi.sample.sqlite ' + s.mainDirectory + '/shinobi-test.sqlite')
+            config.databaseType = 'sqlite3'
+            config.db = {
+                filename: s.mainDirectory + "/shinobi-test.sqlite"
+            }
+        }
+    }
+    var onProcessReady = function(){
+        if(config.testMode === true){
+            s.location.super =  s.mainDirectory + '/super-test.json'
+            fs.writeFileSync(s.location.super,s.s([
+                {
+                    "mail":"admin@shinobi.video",
+                    "pass":"21232f297a57a5a743894a0e4a801fc3",
+                    "tokens":[
+                        "111"
+                    ]
+                }
+            ],null,3))
+            setTimeout(function(){
+                require(s.mainDirectory + '/test/run.js')(s,config,lang,io)
+            },500)
+        }
+    }
+    var onProcessExit = function(){
+        if(config.testMode === true){
+            execSync('rm ' + s.mainDirectory + '/shinobi-test.sqlite')
+            execSync('rm ' + s.location.super)
+            execSync('rm -rf ' + config.videosDir)
+            console.log('---- Temporary Files Cleaned Up')
+            process.exit()
+        }
+    }
+    //attach event handlers
+    s.onFFmpegLoaded(onFFmpegLoaded)
+    s.onBeforeDatabaseLoad(onBeforeDatabaseLoad)
+    s.onProcessReady(onProcessReady)
+    s.onProcessExit(onProcessExit)
+}

libs/config.js

@@ -34,10 +34,11 @@ module.exports = function(s){
     if(config.databaseLogs === undefined){config.databaseLogs=false}
     if(config.useUTC === undefined){config.useUTC=false}
     if(config.iconURL === undefined){config.iconURL = "https://shinobi.video/libs/assets/icon/apple-touch-icon-152x152.png"}
-    if(config.pipeAddition === undefined){config.pipeAddition=7}else{config.pipeAddition=parseInt(config.pipeAddition)}
+    if(config.pipeAddition === undefined){config.pipeAddition=10}else{config.pipeAddition=parseInt(config.pipeAddition)}
     if(config.hideCloudSaveUrls === undefined){config.hideCloudSaveUrls = true}
     if(config.insertOrphans === undefined){config.insertOrphans = true}
     if(config.orphanedVideoCheckMax === undefined){config.orphanedVideoCheckMax = 20}
+    if(config.detectorMergePamRegionTriggers === undefined){config.detectorMergePamRegionTriggers = false}
     //Child Nodes
     if(config.childNodes === undefined)config.childNodes = {};
         //enabled
@@ -50,6 +51,12 @@ module.exports = function(s){
         if(config.childNodes.key === undefined)config.childNodes.key = [
             '3123asdasdf1dtj1hjk23sdfaasd12asdasddfdbtnkkfgvesra3asdsd3123afdsfqw345'
         ];
-
+    if(config.cron.key === 'change_this_to_something_very_random__just_anything_other_than_this'){
+        console.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
+        console.error('!! Change your cron key in your conf.json.                     !!')
+        console.error(`!! If you're running Shinobi remotely you should do this now.  !!`)
+        console.error('!! You can do this in the Super User panel or from terminal.   !!')
+        console.error('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!')
+    }
     return config
 }

libs/customAutoLoad.js

@@ -0,0 +1,134 @@
+var fs = require('fs')
+var express = require('express')
+module.exports = function(s,config,lang,app,io){
+    s.customAutoLoadModules = {}
+    s.customAutoLoadTree = {
+        pages: [],
+        PageBlocks: [],
+        LibsJs: [],
+        LibsCss: [],
+        adminPageBlocks: [],
+        adminLibsJs: [],
+        adminLibsCss: [],
+        superPageBlocks: [],
+        superLibsJs: [],
+        superLibsCss: []
+    }
+    var folderPath = __dirname + '/customAutoLoad'
+    var search = function(searchFor,searchIn){return searchIn.indexOf(searchFor) > -1}
+    fs.readdir(folderPath,function(err,folderContents){
+        if(!err && folderContents){
+            folderContents.forEach(function(filename){
+                s.customAutoLoadModules[filename] = {}
+                var customModulePath = folderPath + '/' + filename
+                if(filename.indexOf('.js') > -1){
+                    s.customAutoLoadModules[filename].type = 'file'
+                    try{
+                        require(customModulePath)(s,config,lang,app,io)
+                    }catch(err){
+                        console.log('Failed to Load Module : ' + filename)
+                        console.log(err)
+                    }
+                }else{
+                    if(fs.lstatSync(customModulePath).isDirectory()){
+                        s.customAutoLoadModules[filename].type = 'folder'
+                        try{
+                            require(customModulePath)(s,config,lang,app,io)
+                            fs.readdir(customModulePath,function(err,folderContents){
+                                folderContents.forEach(function(name){
+                                    switch(name){
+                                        case'web':
+                                            var webFolder = s.checkCorrectPathEnding(customModulePath) + 'web/'
+                                            fs.readdir(webFolder,function(err,webFolderContents){
+                                                webFolderContents.forEach(function(name){
+                                                    switch(name){
+                                                        case'libs':
+                                                        case'pages':
+                                                            if(name === 'libs'){
+                                                                if(config.webPaths.home !== '/'){
+                                                                    app.use('/libs',express.static(webFolder + '/libs'))
+                                                                }
+                                                                app.use(s.checkCorrectPathEnding(config.webPaths.home)+'libs',express.static(webFolder + '/libs'))
+                                                                app.use(s.checkCorrectPathEnding(config.webPaths.admin)+'libs',express.static(webFolder + '/libs'))
+                                                                app.use(s.checkCorrectPathEnding(config.webPaths.super)+'libs',express.static(webFolder + '/libs'))
+                                                            }
+                                                            var libFolder = webFolder + name + '/'
+                                                            fs.readdir(libFolder,function(err,webFolderContents){
+                                                                webFolderContents.forEach(function(libName){
+                                                                    var thirdLevelName = libFolder + libName
+                                                                    switch(libName){
+                                                                        case'js':
+                                                                        case'css':
+                                                                        case'blocks':
+                                                                            fs.readdir(thirdLevelName,function(err,webFolderContents){
+                                                                                webFolderContents.forEach(function(filename){
+                                                                                    var fullPath = thirdLevelName + '/' + filename
+                                                                                    var blockPrefix = ''
+                                                                                    switch(true){
+                                                                                        case search('super.',filename):
+                                                                                            blockPrefix = 'super'
+                                                                                        break;
+                                                                                        case search('admin.',filename):
+                                                                                            blockPrefix = 'admin'
+                                                                                        break;
+                                                                                    }
+                                                                                    switch(libName){
+                                                                                        case'js':
+                                                                                            s.customAutoLoadTree[blockPrefix + 'LibsJs'].push(filename)
+                                                                                        break;
+                                                                                        case'css':
+                                                                                            s.customAutoLoadTree[blockPrefix + 'LibsCss'].push(filename)
+                                                                                        break;
+                                                                                        case'blocks':
+                                                                                            s.customAutoLoadTree[blockPrefix + 'PageBlocks'].push(fullPath)
+                                                                                        break;
+                                                                                    }
+                                                                                })
+                                                                            })
+                                                                        break;
+                                                                        default:
+                                                                            if(libName.indexOf('.ejs') > -1){
+                                                                                s.customAutoLoadTree.pages.push(thirdLevelName)
+                                                                            }
+                                                                        break;
+                                                                    }
+                                                                })
+                                                            })
+                                                        break;
+                                                    }
+                                                })
+                                            })
+                                        break;
+                                        case'languages':
+                                            var languagesFolder = s.checkCorrectPathEnding(customModulePath) + 'languages/'
+                                            fs.readdir(languagesFolder,function(err,files){
+                                                if(err)return console.log(err);
+                                                files.forEach(function(filename){
+                                                    var fileData = require(languagesFolder + filename)
+                                                    var rule = filename.replace('.json','')
+                                                    if(config.language === rule){
+                                                        lang = Object.assign(lang,fileData)
+                                                    }
+                                                    if(s.loadedLanguages[rule]){
+                                                        s.loadedLanguages[rule] = Object.assign(s.loadedLanguages[rule],fileData)
+                                                    }else{
+                                                        s.loadedLanguages[rule] = Object.assign(s.copySystemDefaultLanguage(),fileData)
+                                                    }
+                                                })
+                                            })
+                                        break;
+                                    }
+                                })
+                            })
+                        }catch(err){
+                            console.log('Failed to Load Module : ' + filename)
+                            console.log(err)
+                        }
+                    }
+                }
+            })
+        }else{
+            fs.mkdirSync(folderPath)
+        }
+    })
+}

libs/detector.js

@@ -1,6 +1,11 @@
-var P2P = require('pipe2pam');
+// Matrix In Region Libs >
+var SAT = require('sat')
+var V = SAT.Vector;
+var P = SAT.Polygon;
+// Matrix In Region Libs />
+var P2P = require('pipe2pam')
 // pamDiff is based on https://www.npmjs.com/package/pam-diff
-var PamDiff = require('./detectorPamDiff.js');
+var PamDiff = require('pam-diff')
 module.exports = function(s,config){
     s.createPamDiffEngine = function(e){
         var width,
@@ -46,61 +51,146 @@ module.exports = function(s,config){
                     [width,height],
                     [width,0]
                 ]
-            };
+            }
         }
 
         e.triggerTimer = {}
 
         var regions = s.createPamDiffRegionArray(regionJson,globalColorThreshold,globalSensitivity,fullFrame)
-
-        s.group[e.ke].mon[e.id].pamDiff = new PamDiff({
+        var pamDiffOptions = {
             grayscale: 'luminosity',
-            regions : regions.forPam,
-            drawMatrix : e.details.detector_show_matrix
-        });
-        s.group[e.ke].mon[e.id].p2p = new P2P();
-        var sendTrigger = function(trigger){
-            var detectorObject = {
-                f:'trigger',
-                id:e.id,
-                ke:e.ke,
-                name:trigger.name,
-                details:{
-                    plug:'built-in',
-                    name:trigger.name,
-                    reason:'motion',
-                    confidence:trigger.percent
-                },
-                plates:[],
-                imgHeight:e.details.detector_scale_y,
-                imgWidth:e.details.detector_scale_x
-            }
-            if(trigger.matrix)detectorObject.details.matrices = [trigger.matrix]
-            var region = Object.values(regionJson).find(x => x.name == detectorObject.name)
-            s.checkMaximumSensitivity(e, region, detectorObject, function() {
-                s.checkTriggerThreshold(e, region, detectorObject, function() {
-                    detectorObject.doObjectDetection = (s.ocv && e.details.detector_use_detect_object === '1')
-                    s.triggerEvent(detectorObject)
-                })
-            })
+            regions : regions.forPam
         }
-        if(e.details.detector_noise_filter==='1'){
-            if(!s.group[e.ke].mon[e.id].noiseFilterArray)s.group[e.ke].mon[e.id].noiseFilterArray = {}
-            var noiseFilterArray = s.group[e.ke].mon[e.id].noiseFilterArray
-            Object.keys(regions.notForPam).forEach(function(name){
-                if(!noiseFilterArray[name])noiseFilterArray[name]=[];
-            })
-            s.group[e.ke].mon[e.id].pamDiff.on('diff', (data) => {
-                data.trigger.forEach(function(trigger){
-                    s.filterTheNoise(e,noiseFilterArray,regions,trigger,function(){
-                        sendTrigger(trigger)
+        if(e.details.detector_show_matrix==='1'){
+            pamDiffOptions.response = 'bounds'
+        }
+        s.group[e.ke].mon[e.id].pamDiff = new PamDiff(pamDiffOptions);
+        s.group[e.ke].mon[e.id].p2p = new P2P()
+        var regionArray = Object.values(regionJson)
+        if(config.detectorMergePamRegionTriggers === true){
+            // merge pam triggers for performance boost
+            var buildTriggerEvent = function(trigger){
+                var detectorObject = {
+                    f:'trigger',
+                    id:e.id,
+                    ke:e.ke,
+                    name:trigger.name,
+                    details:{
+                        plug:'built-in',
+                        name:trigger.name,
+                        reason:'motion',
+                        confidence:trigger.percent
+                    },
+                    plates:[],
+                    imgHeight:e.details.detector_scale_y,
+                    imgWidth:e.details.detector_scale_x
+                }
+                if(trigger.merged){
+                    if(trigger.matrices)detectorObject.details.matrices = trigger.matrices
+                    var filteredCount = 0
+                    var filteredCountSuccess = 0
+                    trigger.merged.forEach(function(triggerPiece){
+                        var region = regionArray.find(x => x.name == triggerPiece.name)
+                        s.checkMaximumSensitivity(e, region, detectorObject, function(err1) {
+                            s.checkTriggerThreshold(e, region, detectorObject, function(err2) {
+                                ++filteredCount
+                                if(!err1 && !err2)++filteredCountSuccess
+                                if(filteredCount === trigger.merged.length && filteredCountSuccess > 0){
+                                    detectorObject.doObjectDetection = (s.isAtleatOneDetectorPluginConnected && e.details.detector_use_detect_object === '1')
+                                    s.triggerEvent(detectorObject)
+                                }
+                            })
+                        })
+                    })
+                }else{
+                    if(trigger.matrix)detectorObject.details.matrices = [trigger.matrix]
+                    var region = regionArray.find(x => x.name == detectorObject.name)
+                    s.checkMaximumSensitivity(e, region, detectorObject, function(err1) {
+                        s.checkTriggerThreshold(e, region, detectorObject, function(err2) {
+                            if(!err1 && !err2){
+                                detectorObject.doObjectDetection = (s.isAtleatOneDetectorPluginConnected && e.details.detector_use_detect_object === '1')
+                                s.triggerEvent(detectorObject)
+                            }
+                        })
+                    })
+                }
+            }
+            if(e.details.detector_noise_filter==='1'){
+                if(!s.group[e.ke].mon[e.id].noiseFilterArray)s.group[e.ke].mon[e.id].noiseFilterArray = {}
+                var noiseFilterArray = s.group[e.ke].mon[e.id].noiseFilterArray
+                Object.keys(regions.notForPam).forEach(function(name){
+                    if(!noiseFilterArray[name])noiseFilterArray[name]=[];
+                })
+                s.group[e.ke].mon[e.id].pamDiff.on('diff', (data) => {
+                    var filteredCount = 0
+                    var filteredCountSuccess = 0
+                    data.trigger.forEach(function(trigger){
+                        s.filterTheNoise(e,noiseFilterArray,regions,trigger,function(err){
+                            ++filteredCount
+                            if(!err)++filteredCountSuccess
+                            if(filteredCount === data.trigger.length && filteredCountSuccess > 0){
+                                buildTriggerEvent(s.mergePamTriggers(data))
+                            }
+                        })
                     })
                 })
-            })
+            }else{
+                s.group[e.ke].mon[e.id].pamDiff.on('diff', (data) => {
+                    buildTriggerEvent(s.mergePamTriggers(data))
+                })
+            }
         }else{
-            s.group[e.ke].mon[e.id].pamDiff.on('diff', (data) => {
-                data.trigger.forEach(sendTrigger)
-            })
+            //config.detectorMergePamRegionTriggers NOT true
+            //original behaviour, all regions have their own event.
+            var buildTriggerEvent = function(trigger){
+                var detectorObject = {
+                    f:'trigger',
+                    id:e.id,
+                    ke:e.ke,
+                    name:trigger.name,
+                    details:{
+                        plug:'built-in',
+                        name:trigger.name,
+                        reason:'motion',
+                        confidence:trigger.percent
+                    },
+                    plates:[],
+                    imgHeight:e.details.detector_scale_y,
+                    imgWidth:e.details.detector_scale_x
+                }
+                if(trigger.matrix)detectorObject.details.matrices = [trigger.matrix]
+                var region = Object.values(regionJson).find(x => x.name == detectorObject.name)
+                s.checkMaximumSensitivity(e, region, detectorObject, function(err1) {
+                    s.checkTriggerThreshold(e, region, detectorObject, function(err2) {
+                        if(!err1 && ! err2){
+                            detectorObject.doObjectDetection = (s.isAtleatOneDetectorPluginConnected && e.details.detector_use_detect_object === '1')
+                            s.triggerEvent(detectorObject)
+                        }
+                    })
+                })
+            }
+            if(e.details.detector_noise_filter==='1'){
+                if(!s.group[e.ke].mon[e.id].noiseFilterArray)s.group[e.ke].mon[e.id].noiseFilterArray = {}
+                var noiseFilterArray = s.group[e.ke].mon[e.id].noiseFilterArray
+                Object.keys(regions.notForPam).forEach(function(name){
+                    if(!noiseFilterArray[name])noiseFilterArray[name]=[];
+                })
+                s.group[e.ke].mon[e.id].pamDiff.on('diff', (data) => {
+                    data.trigger.forEach(function(trigger){
+                        s.filterTheNoise(e,noiseFilterArray,regions,trigger,function(){
+                            s.createMatrixFromPamTrigger(trigger)
+                            buildTriggerEvent(trigger)
+                        })
+                    })
+                })
+            }else{
+                s.group[e.ke].mon[e.id].pamDiff.on('diff', (data) => {
+                    data.trigger.forEach(function(trigger){
+                        s.createMatrixFromPamTrigger(trigger)
+                        buildTriggerEvent(trigger)
+                    })
+                })
+            }
         }
     }
 
@@ -164,17 +254,20 @@ module.exports = function(s,config){
         theNoise = theNoise / noiseFilterArray[trigger.name].length;
         var triggerPercentWithoutNoise = trigger.percent - theNoise;
         if(triggerPercentWithoutNoise > regions.notForPam[trigger.name].sensitivity){
-            callback(trigger)
+            callback(null,trigger)
+        }else{
+            callback(true)
         }
     }
 
-    s.checkMaximumSensitivity = function(monitor, region, detectorObject, success) {
+    s.checkMaximumSensitivity = function(monitor, region, detectorObject, callback) {
         var logName = detectorObject.id + ':' + detectorObject.name
         var globalMaxSensitivity = parseInt(monitor.details.detector_max_sensitivity) || undefined
         var maxSensitivity = parseInt(region.max_sensitivity) || globalMaxSensitivity
         if (maxSensitivity === undefined || detectorObject.details.confidence <= maxSensitivity) {
-            success()
+            callback(null)
         } else {
+            callback(true)
             if (monitor.triggerTimer[detectorObject.name] !== undefined) {
                 clearTimeout(monitor.triggerTimer[detectorObject.name].timeout)
                 monitor.triggerTimer[detectorObject.name] = undefined
@@ -182,10 +275,10 @@ module.exports = function(s,config){
         }
     }
 
-    s.checkTriggerThreshold = function(monitor, region, detectorObject, success){
+    s.checkTriggerThreshold = function(monitor, region, detectorObject, callback){
         var threshold = parseInt(region.threshold) || globalThreshold
         if (threshold <= 1) {
-            success()
+            callback(null)
         } else {
             if (monitor.triggerTimer[detectorObject.name] === undefined) {
                 monitor.triggerTimer[detectorObject.name] = {
@@ -194,10 +287,11 @@ module.exports = function(s,config){
                 }
             }
             if (--monitor.triggerTimer[detectorObject.name].count == 0) {
-                success()
+                callback(null)
                 clearTimeout(monitor.triggerTimer[detectorObject.name].timeout)
                 monitor.triggerTimer[detectorObject.name] = undefined
             } else {
+                callback(true)
                 var fps = parseFloat(monitor.details.detector_fps) || 2
                 if (monitor.triggerTimer[detectorObject.name].timeout !== null)
                     clearTimeout(monitor.triggerTimer[detectorObject.name].timeout)
@@ -207,4 +301,92 @@ module.exports = function(s,config){
             }
         }
     }
+    s.mergePamTriggers = function(data){
+        if(data.trigger.length > 1){
+            var n = 0
+            var sum = 0
+            var name = []
+            var matrices = []
+            data.trigger.forEach(function(trigger){
+                name.push(trigger.name + ' ('+trigger.percent+'%)')
+                ++n
+                sum += trigger.percent
+                s.createMatrixFromPamTrigger(trigger)
+                if(trigger.matrix)matrices.push(trigger.matrix)
+            })
+            var average = sum / n
+            name = name.join(', ')
+            if(matrices.length === 0)matrices = null
+            var trigger = {
+                name: name,
+                percent: parseInt(average),
+                matrices: matrices,
+                merged: data.trigger
+            }
+        }else{
+            var trigger = data.trigger[0]
+            s.createMatrixFromPamTrigger(trigger)
+            trigger.matrices = [trigger.matrix]
+        }
+        return trigger
+    }
+    s.isAtleastOneMatrixInRegion = function(regions,matrices,callback){
+        var regionPolys = []
+        var matrixPoints = []
+        regions.forEach(function(region,n){
+            var polyPoints = []
+            region.points.forEach(function(point){
+                polyPoints.push(new V(parseInt(point[0]),parseInt(point[1])))
+            })
+            regionPolys[n] = new P(new V(0,0), polyPoints)
+        })
+        var collisions = []
+        var foundInRegion = false
+        matrices.forEach(function(matrix){
+            var matrixPoints = [
+                new V(matrix.x,matrix.y),
+                new V(matrix.width,matrix.y),
+                new V(matrix.width,matrix.height),
+                new V(matrix.x,matrix.height)
+            ]
+            var matrixPoly = new P(new V(0,0), matrixPoints)
+            regionPolys.forEach(function(region,n){
+                var response = new SAT.Response()
+                var collided = SAT.testPolygonPolygon(matrixPoly, region, response)
+                if(collided === true){
+                    collisions.push({
+                        matrix: matrix,
+                        region: regions[n]
+                    })
+                    foundInRegion = true
+                }
+            })
+        })
+        if(callback)callback(foundInRegion,collisions)
+        return foundInRegion
+    }
+    s.createMatrixFromPamTrigger = function(trigger){
+        if(
+            trigger.minX &&
+            trigger.maxX &&
+            trigger.minY &&
+            trigger.maxY
+        ){
+            var coordinates = [
+                {"x" : trigger.minX, "y" : trigger.minY},
+                {"x" : trigger.maxX, "y" : trigger.minY},
+                {"x" : trigger.maxX, "y" : trigger.maxY}
+            ]
+            var width = Math.sqrt( Math.pow(coordinates[1].x - coordinates[0].x, 2) + Math.pow(coordinates[1].y - coordinates[0].y, 2));
+            var height = Math.sqrt( Math.pow(coordinates[2].x - coordinates[1].x, 2) + Math.pow(coordinates[2].y - coordinates[1].y, 2))
+            trigger.matrix = {
+                x: coordinates[0].x,
+                y: coordinates[0].y,
+                width: width,
+                height: height,
+                tag: trigger.name
+            }
+        }
+        return trigger
+    }
 }

libs/dropInEvents.js

@@ -0,0 +1,208 @@
+var fs = require('fs')
+var execSync = require('child_process').execSync
+module.exports = function(s,config,lang,app,io){
+    if(config.dropInEventServer === true){
+        if(config.dropInEventDeleteFileAfterTrigger === undefined)config.dropInEventDeleteFileAfterTrigger = true
+        var authenticateUser = function(username,password,callback){
+            var splitUsername = username.split('@')
+            if(splitUsername[1] !== 'Shinobi' && splitUsername[1] !== 'shinobi'){
+                s.sqlQuery('SELECT ke,uid FROM Users WHERE mail=? AND (pass=? OR pass=?)',[
+                    username,
+                    password,
+                    s.createHash(password)
+                ],function(err,r){
+                    var user
+                    if(r && r[0]){
+                        user = r[0]
+                    }
+                    callback(err,user)
+                })
+            }else{
+                s.sqlQuery('SELECT ke,uid FROM API WHERE code=? AND ke=?',[
+                    splitUsername[0], //code
+                    password //ke
+                ],function(err,r){
+                    var apiKey
+                    if(r && r[0]){
+                        apiKey = r[0]
+                    }
+                    callback(err,apiKey)
+                })
+            }
+        }
+        var beforeMonitorsLoadedOnStartup = function(){
+            if(!config.dropInEventsDir){
+                config.dropInEventsDir = s.dir.streams + 'dropInEvents/'
+            }
+            s.dir.dropInEvents = s.checkCorrectPathEnding(config.dropInEventsDir)
+            //dropInEvents dir
+            if(!fs.existsSync(s.dir.dropInEvents)){
+                fs.mkdirSync(s.dir.dropInEvents)
+            }
+        }
+        var getDropInEventDir = function(monitorConfig){
+            var ke = monitorConfig.ke
+            var mid = monitorConfig.mid
+            var groupEventDropDir = s.dir.dropInEvents + ke
+            var monitorEventDropDir = groupEventDropDir + '/' + mid + '/'
+            return monitorEventDropDir
+        }
+        var onMonitorStop = function(monitorConfig){
+            var ke = monitorConfig.ke
+            var mid = monitorConfig.mid
+            if(s.group[monitorConfig.ke].mon[monitorConfig.mid].dropInEventWatcher){
+                s.group[monitorConfig.ke].mon[monitorConfig.mid].dropInEventWatcher.close()
+                delete(s.group[monitorConfig.ke].mon[monitorConfig.mid].dropInEventWatcher)
+            }
+            var monitorEventDropDir = getDropInEventDir(monitorConfig)
+            if(fs.existsSync(monitorEventDropDir))execSync('rm -rf ' + monitorEventDropDir)
+        }
+        var onMonitorInit = function(monitorConfig){
+            onMonitorStop(monitorConfig)
+            var ke = monitorConfig.ke
+            var mid = monitorConfig.mid
+            var monitorEventDropDir = getDropInEventDir(monitorConfig)
+            var groupEventDropDir = s.dir.dropInEvents + ke
+            if(!fs.existsSync(groupEventDropDir)){
+                fs.mkdirSync(groupEventDropDir)
+            }
+            var monitorEventDropDir = groupEventDropDir + '/' + mid + '/'
+            if(!fs.existsSync(monitorEventDropDir)){
+                fs.mkdirSync(monitorEventDropDir)
+            }
+            var fileQueue = {}
+            s.group[monitorConfig.ke].mon[monitorConfig.mid].dropInEventFileQueue = fileQueue
+            var eventTrigger = function(eventType,filename){
+                var filePath = monitorEventDropDir + filename
+                if(filename.indexOf('.jpg') > -1 || filename.indexOf('.jpeg') > -1){
+                    var snapPath = s.dir.streams + ke + '/' + mid + '/s.jpg'
+                    fs.unlink(snapPath,function(err){
+                        fs.createReadStream(filePath).pipe(fs.createWriteStream(snapPath))
+                        s.triggerEvent({
+                            id: mid,
+                            ke: ke,
+                            details: {
+                                confidence: 100,
+                                name: filename,
+                                plug: "dropInEvent",
+                                reason: "dropInEvent"
+                            }
+                        })
+                    })
+                }else{
+                    s.triggerEvent({
+                        id: mid,
+                        ke: ke,
+                        details: {
+                            confidence: 100,
+                            name: filename,
+                            plug: "dropInEvent",
+                            reason: "ftpServer"
+                        }
+                    })
+                }
+                if(config.dropInEventDeleteFileAfterTrigger){
+                    setTimeout(function(){
+                        fs.unlink(filePath,function(err){
+
+                        })
+                    },1000 * 60 * 5)
+                }
+            }
+            var directoryWatch = fs.watch(monitorEventDropDir,function(eventType,filename){
+                if(fs.existsSync(monitorEventDropDir + filename)){
+                    clearTimeout(fileQueue[filename])
+                    fileQueue[filename] = setTimeout(function(){
+                        eventTrigger(eventType,filename)
+                    },1200)
+                }
+            })
+            s.group[monitorConfig.ke].mon[monitorConfig.mid].dropInEventWatcher = directoryWatch
+        }
+        // FTP Server
+        if(config.ftpServer === true){
+            if(!config.ftpServerPort)config.ftpServerPort = 21
+            if(!config.ftpServerUrl)config.ftpServerUrl = `ftp://0.0.0.0:${config.ftpServerPort}`
+            config.ftpServerUrl = config.ftpServerUrl.replace('{{PORT}}',config.ftpServerPort)
+            const FtpSrv = require('ftp-srv')
+            const ftpServer = new FtpSrv({
+                url: config.ftpServerUrl,
+                // log:{trace:function(){},error:function(){},child:function(){},info:function(){},warn:function(){}
+            })
+
+            ftpServer.on('login', (data, resolve, reject) => {
+                var username = data.username
+                var password = data.password
+                authenticateUser(username,password,function(err,user){
+                    if(user){
+                        resolve({root: s.dir.dropInEvents + user.ke})
+                    }else{
+                        // reject(new Error('Failed Authorization'))
+                    }
+                })
+            })
+
+            ftpServer.listen().then(() => {
+                s.systemLog(`FTP Server running on port ${config.ftpServerPort}...`)
+            }).catch(function(err){
+                s.systemLog(err)
+            })
+        }
+        //add extensions
+        s.beforeMonitorsLoadedOnStartup(beforeMonitorsLoadedOnStartup)
+        s.onMonitorInit(onMonitorInit)
+        s.onMonitorStop(onMonitorStop)
+    }
+    // SMTP Server
+    // allow starting SMTP server without dropInEventServer
+    if(config.smtpServer === true){
+        var SMTPServer = require("smtp-server").SMTPServer;
+        if(!config.smtpServerPort && (config.smtpServerSsl && config.smtpServerSsl.enabled !== false || config.ssl)){config.smtpServerPort = 465}else if(!config.smtpServerPort){config.smtpServerPort = 25}
+        var smtpOptions = {
+            onAuth(auth, session, callback) {
+                var username = auth.username
+                var password = auth.password
+                authenticateUser(username,password,function(err,user){
+                    if(user){
+                        callback(null, {user: user.ke})
+                    }else{
+                        callback(new Error(lang.failedLoginText2))
+                    }
+                })
+            },
+            onRcptTo(address, session, callback) {
+                var split = address.address.split('@')
+                var monitorId = split[0]
+                var ke = session.user
+                if(s.group[ke].mon_conf[monitorId] && s.group[ke].mon[monitorId].isStarted === true){
+                    s.triggerEvent({
+                        id: monitorId,
+                        ke: ke,
+                        details: {
+                            confidence: 100,
+                            name: address.address,
+                            plug: "dropInEvent",
+                            reason: "smtpServer"
+                        }
+                    })
+                }else{
+                    return callback(new Error(lang['No Monitor Exists with this ID.']))
+                }
+                callback()
+            }
+        }
+        if(config.smtpServerSsl && config.smtpServerSsl.enabled !== false || config.ssl && config.ssl.cert && config.ssl.key){
+            var key = config.ssl.key || fs.readFileSync(config.smtpServerSsl.key)
+            var cert = config.ssl.cert || fs.readFileSync(config.smtpServerSsl.cert)
+            smtpOptions = Object.assign(smtpOptions,{
+                secure: true,
+                key: config.ssl.key,
+                cert: config.ssl.cert
+            })
+        }
+        var server = new SMTPServer(smtpOptions)
+        server.listen(config.smtpServerPort,function(){
+            s.systemLog(`SMTP Server running on port ${config.smtpServerPort}...`)
+        })
+    }
+}

libs/events.js

@@ -4,6 +4,25 @@ var exec = require('child_process').exec;
 var spawn = require('child_process').spawn;
 var request = require('request');
 module.exports = function(s,config,lang){
+    var addEventDetailsToString = function(eventData,string,addOps){
+        //d = event data
+        if(!addOps)addOps = {}
+        var newString = string + ''
+        var d = Object.assign(eventData,addOps)
+        var detailString = s.stringJSON(d.details)
+        newString = newString
+            .replace(/{{TIME}}/g,d.currentTimestamp)
+            .replace(/{{REGION_NAME}}/g,d.details.name)
+            .replace(/{{SNAP_PATH}}/g,s.dir.streams+'/'+d.ke+'/'+d.id+'/s.jpg')
+            .replace(/{{MONITOR_ID}}/g,d.id)
+            .replace(/{{GROUP_KEY}}/g,d.ke)
+            .replace(/{{DETAILS}}/g,detailString)
+        if(d.details.confidence){
+            newString = newString
+            .replace(/{{CONFIDENCE}}/g,d.details.confidence)
+        }
+        return newString
+    }
     s.filterEvents = function(x,d){
         switch(x){
             case'archive':
@@ -36,15 +55,13 @@ module.exports = function(s,config,lang){
         s.onEventTriggerBeforeFilterExtensions.forEach(function(extender){
             extender(d,filter)
         })
-        if(s.group[d.ke].mon[d.id].open){
-            d.details.videoTime = s.group[d.ke].mon[d.id].open;
-        }
         var detailString = JSON.stringify(d.details);
         if(!s.group[d.ke]||!s.group[d.ke].mon[d.id]){
             return s.systemLog(lang['No Monitor Found, Ignoring Request'])
         }
         d.mon=s.group[d.ke].mon_conf[d.id];
         var currentConfig = s.group[d.ke].mon[d.id].details
+        var hasMatrices = (d.details.matrices && d.details.matrices.length > 0)
         //read filters
         if(
             currentConfig.use_detector_filters === '1' &&
@@ -160,7 +177,7 @@ module.exports = function(s,config,lang){
             })
             if(d.details.matrices && d.details.matrices.length === 0 || filter.halt === true){
                 return
-            }else if(d.details.matrices && d.details.matrices.length > 0){
+            }else if(hasMatrices){
                 var reviewedMatrix = []
                 d.details.matrices.forEach(function(matrix){
                     if(matrix)reviewedMatrix.push(matrix)
@@ -193,6 +210,16 @@ module.exports = function(s,config,lang){
                 return
             }
         }
+        // check if object should be in region
+        if(hasMatrices && currentConfig.detector_obj_region === '1'){
+            var regions = s.group[d.ke].mon[d.id].parsedObjects.cords
+            var isMatrixInRegions = s.isAtleastOneMatrixInRegion(regions,d.details.matrices)
+            if(isMatrixInRegions){
+                s.debugLog('Matrix in region!')
+            }else{
+                return
+            }
+        }
         // check modified indifference
         if(filter.indifference !== false && d.details.confidence < parseFloat(filter.indifference)){
             // fails indifference check for modified indifference
@@ -209,13 +236,29 @@ module.exports = function(s,config,lang){
                 frame : s.group[d.ke].mon[d.id].lastJpegDetectorFrame
             })
         }else{
+            if(currentConfig.detector_multi_trigger === '1'){
+                s.getCamerasForMultiTrigger(d.mon).forEach(function(monitor){
+                    if(monitor.mid !== d.id){
+                        s.triggerEvent({
+                            id: monitor.mid,
+                            ke: monitor.ke,
+                            details: {
+                                confidence: 100,
+                                name: "multiTrigger",
+                                plug: d.details.plug,
+                                reason: d.details.reason
+                            }
+                        })
+                    }
+                })
+            }
             //save this detection result in SQL, only coords. not image.
-            if(filter.save && currentConfig.detector_save==='1'){
-                s.sqlQuery('INSERT INTO Events (ke,mid,details) VALUES (?,?,?)',[d.ke,d.id,detailString])
+            if(filter.save && currentConfig.detector_save === '1'){
+                s.sqlQuery('INSERT INTO Events (ke,mid,details,time) VALUES (?,?,?,?)',[d.ke,d.id,detailString,new Date()])
             }
             if(currentConfig.detector_notrigger === '1'){
                 var detector_notrigger_timeout
-                if(!currentConfig.detector_notrigger_timeout||currentConfig.detector_notrigger_timeout===''){
+                if(!currentConfig.detector_notrigger_timeout||currentConfig.detector_notrigger_timeout === ''){
                     detector_notrigger_timeout = 10
                 }
                 detector_notrigger_timeout = parseFloat(currentConfig.detector_notrigger_timeout)*1000*60;
@@ -270,17 +313,7 @@ module.exports = function(s,config,lang){
             })
 
             if(filter.webhook && currentConfig.detector_webhook === '1'){
-                var detector_webhook_url = currentConfig.detector_webhook_url
-                    .replace(/{{TIME}}/g,d.currentTimestamp)
-                    .replace(/{{REGION_NAME}}/g,d.details.name)
-                    .replace(/{{SNAP_PATH}}/g,s.dir.streams+'/'+d.ke+'/'+d.id+'/s.jpg')
-                    .replace(/{{MONITOR_ID}}/g,d.id)
-                    .replace(/{{GROUP_KEY}}/g,d.ke)
-                    .replace(/{{DETAILS}}/g,detailString)
-                    if(d.details.confidence){
-                        detector_webhook_url = detector_webhook_url
-                        .replace(/{{CONFIDENCE}}/g,d.details.confidence)
-                    }
+                var detector_webhook_url = addEventDetailsToString(d,currentConfig.detector_webhook_url)
                 request({url:detector_webhook_url,method:'GET',encoding:null},function(err,data){
                     if(err){
                         s.userLog(d,{type:lang["Event Webhook Error"],msg:{error:err,data:data}})
@@ -289,28 +322,8 @@ module.exports = function(s,config,lang){
             }
 
             if(filter.command && currentConfig.detector_command_enable === '1' && !s.group[d.ke].mon[d.id].detector_command){
-                var detector_command_timeout
-                if(!currentConfig.detector_command_timeout||currentConfig.detector_command_timeout===''){
-                    detector_command_timeout = 1000*60*10;
-                }else{
-                    detector_command_timeout = parseFloat(currentConfig.detector_command_timeout)*1000*60;
-                }
-                s.group[d.ke].mon[d.id].detector_command=setTimeout(function(){
-                    clearTimeout(s.group[d.ke].mon[d.id].detector_command);
-                    delete(s.group[d.ke].mon[d.id].detector_command);
-
-                },detector_command_timeout);
-                var detector_command = currentConfig.detector_command
-                    .replace(/{{TIME}}/g,d.currentTimestamp)
-                    .replace(/{{REGION_NAME}}/g,d.details.name)
-                    .replace(/{{SNAP_PATH}}/g,s.dir.streams+'/'+d.ke+'/'+d.id+'/s.jpg')
-                    .replace(/{{MONITOR_ID}}/g,d.id)
-                    .replace(/{{GROUP_KEY}}/g,d.ke)
-                    .replace(/{{DETAILS}}/g,detailString)
-                    if(d.details.confidence){
-                        detector_command = detector_command
-                        .replace(/{{CONFIDENCE}}/g,d.details.confidence)
-                    }
+                s.createTimeout(s.group[d.ke].mon[d.id].detector_command,currentConfig.detector_command_timeout,10)
+                var detector_command = addEventDetailsToString(d,currentConfig.detector_command)
                 exec(detector_command,{detached: true})
             }
         }
@@ -319,14 +332,18 @@ module.exports = function(s,config,lang){
         s.tx(d.cx,'DETECTOR_'+d.ke+d.id);
     }
     s.createEventBasedRecording = function(d){
+        d.mon = s.group[d.ke].mon_conf[d.id]
         var currentConfig = s.group[d.ke].mon[d.id].details
+        if(currentConfig.detector !== '1'){
+            return
+        }
         var detector_timeout
         if(!currentConfig.detector_timeout||currentConfig.detector_timeout===''){
             detector_timeout = 10
         }else{
             detector_timeout = parseFloat(currentConfig.detector_timeout)
         }
-        if(currentConfig.watchdog_reset !== '1' || !s.group[d.ke].mon[d.id].eventBasedRecording.timeout){
+        if(currentConfig.watchdog_reset === '1' || !s.group[d.ke].mon[d.id].eventBasedRecording.timeout){
             clearTimeout(s.group[d.ke].mon[d.id].eventBasedRecording.timeout)
             s.group[d.ke].mon[d.id].eventBasedRecording.timeout = setTimeout(function(){
                 s.group[d.ke].mon[d.id].eventBasedRecording.allowEnd = true
@@ -336,40 +353,28 @@ module.exports = function(s,config,lang){
             },detector_timeout * 1000 * 60)
         }
         if(!s.group[d.ke].mon[d.id].eventBasedRecording.process){
-            if(!d.auth){
-                d.auth = s.gid(60)
-            }
-            if(!s.api[d.auth]){
-                s.api[d.auth] = {
-                    system: 1,
-                    ip: '0.0.0.0',
-                    details: {},
-                    lang: lang
-                }
-            }
             s.group[d.ke].mon[d.id].eventBasedRecording.allowEnd = false;
             var runRecord = function(){
                 var filename = s.formattedTime()+'.mp4'
-                s.userLog(d,{type:"Traditional Recording",msg:"Started"})
+                s.userLog(d,{type:lang["Traditional Recording"],msg:lang["Started"]})
                 //-t 00:'+s.timeObject(new Date(detector_timeout * 1000 * 60)).format('mm:ss')+'
-                s.group[d.ke].mon[d.id].eventBasedRecording.process = spawn(config.ffmpegDir,s.splitForFFPMEG(('-loglevel warning -analyzeduration 1000000 -probesize 1000000 -re -i http://'+config.ip+':'+config.port+'/'+d.auth+'/hls/'+d.ke+'/'+d.id+'/detectorStream.m3u8 -c:v copy -strftime 1 "'+s.getVideoDirectory(d.mon) + filename + '"')))
+                s.group[d.ke].mon[d.id].eventBasedRecording.process = spawn(config.ffmpegDir,s.splitForFFPMEG(('-loglevel warning -analyzeduration 1000000 -probesize 1000000 -re -i "'+s.dir.streams+'/'+d.ke+'/'+d.id+'/detectorStream.m3u8" -c:v copy -strftime 1 "'+s.getVideoDirectory(d.mon) + filename + '"')))
                 var ffmpegError='';
                 var error
                 s.group[d.ke].mon[d.id].eventBasedRecording.process.stderr.on('data',function(data){
-                    s.userLog(d,{type:"Traditional Recording",msg:data.toString()})
+                    s.userLog(d,{type:lang["Traditional Recording"],msg:data.toString()})
                 })
                 s.group[d.ke].mon[d.id].eventBasedRecording.process.on('close',function(){
                     if(!s.group[d.ke].mon[d.id].eventBasedRecording.allowEnd){
-                        s.userLog(d,{type:"Traditional Recording",msg:"Detector Recording Process Exited Prematurely. Restarting."})
+                        s.userLog(d,{type:lang["Traditional Recording"],msg:lang["Detector Recording Process Exited Prematurely. Restarting."]})
                         runRecord()
                         return
                     }
                     s.insertCompletedVideo(d.mon,{
                         file : filename
                     })
-                    s.userLog(d,{type:"Traditional Recording",msg:"Detector Recording Complete"})
-                    delete(s.api[d.auth])
-                    s.userLog(d,{type:"Traditional Recording",msg:'Clear Recorder Process'})
+                    s.userLog(d,{type:lang["Traditional Recording"],msg:lang["Detector Recording Complete"]})
+                    s.userLog(d,{type:lang["Traditional Recording"],msg:lang["Clear Recorder Process"]})
                     delete(s.group[d.ke].mon[d.id].eventBasedRecording.process)
                     clearTimeout(s.group[d.ke].mon[d.id].eventBasedRecording.timeout)
                     delete(s.group[d.ke].mon[d.id].eventBasedRecording.timeout)
@@ -385,5 +390,12 @@ module.exports = function(s,config,lang){
             s.group[e.ke].mon[e.id].eventBasedRecording.allowEnd = true;
             s.group[e.ke].mon[e.id].eventBasedRecording.process.kill('SIGTERM');
         }
+        // var stackedProcesses = s.group[e.ke].mon[e.id].eventBasedRecording.stackable
+        // Object.keys(stackedProcesses).forEach(function(key){
+        //     var item = stackedProcesses[key]
+        //     clearTimeout(item.timeout)
+        //     item.allowEnd = true;
+        //     item.process.kill('SIGTERM');
+        // })
     }
 }

libs/extenders.js

@@ -1,5 +1,15 @@
 module.exports = function(s,config){
     ////// USER //////
+    s.onSocketAuthenticationExtensions = []
+    s.onSocketAuthentication = function(callback){
+        s.onSocketAuthenticationExtensions.push(callback)
+    }
+    //
+    s.loadGroupExtensions = []
+    s.loadGroupExtender = function(callback){
+        s.loadGroupExtensions.push(callback)
+    }
+    //
     s.loadGroupAppExtensions = []
     s.loadGroupAppExtender = function(callback){
         s.loadGroupAppExtensions.push(callback)
@@ -30,6 +40,11 @@ module.exports = function(s,config){
         s.onTwoFactorAuthCodeNotificationExtensions.push(callback)
     }
     //
+    s.onStalePurgeLockExtensions = []
+    s.onStalePurgeLock = function(callback){
+        s.onStalePurgeLockExtensions.push(callback)
+    }
+    //
     s.cloudDiskUseStartupExtensions = {}
 
     ////// EVENTS //////
@@ -51,8 +66,81 @@ module.exports = function(s,config){
     s.onMonitorInit = function(callback){
         s.onMonitorInitExtensions.push(callback)
     }
+    //
+    s.onMonitorStartExtensions = []
+    s.onMonitorStart = function(callback){
+        s.onMonitorStartExtensions.push(callback)
+    }
+    //
+    s.onMonitorStopExtensions = []
+    s.onMonitorStop = function(callback){
+        s.onMonitorStopExtensions.push(callback)
+    }
+    //
+    s.onMonitorSaveExtensions = []
+    s.onMonitorSave = function(callback){
+        s.onMonitorSaveExtensions.push(callback)
+    }
+    //
+    s.onMonitorUnexpectedExitExtensions = []
+    s.onMonitorUnexpectedExit = function(callback){
+        s.onMonitorUnexpectedExitExtensions.push(callback)
+    }
+    //
     s.onDetectorNoTriggerTimeoutExtensions = []
     s.onDetectorNoTriggerTimeout = function(callback){
         s.onDetectorNoTriggerTimeoutExtensions.push(callback)
     }
+    //
+    s.onFfmpegCameraStringCreationExtensions = []
+    s.onFfmpegCameraStringCreation = function(callback){
+        s.onFfmpegCameraStringCreationExtensions.push(callback)
+    }
+    //
+    s.onMonitorPingFailedExtensions = []
+    s.onMonitorPingFailed = function(callback){
+        s.onMonitorPingFailedExtensions.push(callback)
+    }
+    //
+    s.onMonitorDiedExtensions = []
+    s.onMonitorDied = function(callback){
+        s.onMonitorDiedExtensions.push(callback)
+    }
+
+    ///////// SYSTEM ////////
+    s.onProcessReadyExtensions = []
+    s.onProcessReady = function(callback){
+        s.onProcessReadyExtensions.push(callback)
+    }
+    //
+    s.onProcessExitExtensions = []
+    s.onProcessExit = function(callback){
+        s.onProcessExitExtensions.push(callback)
+    }
+    //
+    s.onBeforeDatabaseLoadExtensions = []
+    s.onBeforeDatabaseLoad = function(callback){
+        s.onBeforeDatabaseLoadExtensions.push(callback)
+    }
+    //
+    s.onFFmpegLoadedExtensions = []
+    s.onFFmpegLoaded = function(callback){
+        s.onFFmpegLoadedExtensions.push(callback)
+    }
+    //
+    s.beforeMonitorsLoadedOnStartupExtensions = []
+    s.beforeMonitorsLoadedOnStartup = function(callback){
+        s.beforeMonitorsLoadedOnStartupExtensions.push(callback)
+    }
+    //
+    s.onWebSocketConnectionExtensions = []
+    s.onWebSocketConnection = function(callback){
+        s.onWebSocketConnectionExtensions.push(callback)
+    }
+    //
+    s.onWebSocketDisconnectionExtensions = []
+    s.onWebSocketDisconnection = function(callback){
+        s.onWebSocketDisconnectionExtensions.push(callback)
+    }
+    //
 }

libs/ffmpeg.js

@@ -104,6 +104,9 @@ module.exports = function(s,config,onFinish){
     ffmpeg.completeCheck = function(){
         ffmpeg.checkVersion(function(){
             ffmpeg.checkHwAccelMethods(function(){
+                s.onFFmpegLoadedExtensions.forEach(function(extender){
+                    extender(ffmpeg)
+                })
                 onFinish(ffmpeg)
             })
         })
@@ -132,7 +135,9 @@ module.exports = function(s,config,onFinish){
                     string += ' -map '+v.map
                 })
             }else{
-                string += ' -map 0:0'
+                var primaryMap = '0:0'
+                if(e.details.primary_input && e.details.primary_input !== '')primaryMap = e.details.primary_input
+                string += ' -map ' + primaryMap
             }
         }
         return string;
@@ -379,7 +384,16 @@ module.exports = function(s,config,onFinish){
         //x = temporary values
         //check if CUDA is enabled
         e.isStreamer = (e.type === 'dashcam'|| e.type === 'socket')
-        if(e.details.accelerator === '1' && e.details.hwaccel === 'cuvid' && e.details.hwaccel_vcodec === ('h264_cuvid' || 'hevc_cuvid' || 'mjpeg_cuvid' || 'mpeg4_cuvid')){
+        e.coProcessor = false
+        if(
+            e.details.use_coprocessor === '1' &&
+            e.details.accelerator === '1' &&
+            e.isStreamer === false &&
+            (!e.details.input_maps || e.details.input_maps.length === 0) &&
+            (e.details.snap === '1' || e.details.stream_type === 'mjpeg' || e.details.stream_type === 'b64' || e.details.detector === '1')
+          ){
+            e.coProcessor = true
+        }else if(e.details.accelerator === '1' && e.details.hwaccel === 'cuvid' && e.details.hwaccel_vcodec === ('h264_cuvid' || 'hevc_cuvid' || 'mjpeg_cuvid' || 'mpeg4_cuvid')){
             e.cudaEnabled = true
         }
         //
@@ -560,9 +574,11 @@ module.exports = function(s,config,onFinish){
                 x.pipe+=x.preset_stream+x.stream_acodec+x.stream_vcodec+' -f hls'+x.cust_stream+' -hls_time '+x.hls_time+' -hls_list_size '+x.hls_list_size+' -start_number 0 -hls_allow_cache 0 -hls_flags +delete_segments+omit_endlist "'+e.sdir+'s.m3u8"';
             break;
             case'mjpeg':
+                if(e.coProcessor === false){
                     if(e.details.stream_quality && e.details.stream_quality !== '')x.cust_stream+=' -q:v '+e.details.stream_quality;
                     if(x.dimensions && x.cust_stream.indexOf('-s ')===-1){x.cust_stream+=' -s '+x.dimensions}
                     x.pipe+=' -an -c:v mjpeg -f mpjpeg -boundary_tag shinobi'+x.cust_stream+x.stream_video_filters+' pipe:1';
+                }
             break;
             case'h265':
                 x.cust_stream+=' -movflags +frag_keyframe+empty_moov+default_base_moof -metadata title="Shinobi H.265 Stream" -reset_timestamps 1'
@@ -575,9 +591,11 @@ module.exports = function(s,config,onFinish){
                 x.pipe+=' -f hevc'+x.stream_acodec+x.stream_vcodec+x.cust_stream+' pipe:1';
             break;
             case'b64':case'':case undefined:case null://base64
-                if(e.details.stream_quality && e.details.stream_quality !== '')x.cust_stream+=' -q:v '+e.details.stream_quality;
-                if(x.dimensions && x.cust_stream.indexOf('-s ')===-1){x.cust_stream+=' -s '+x.dimensions}
-                x.pipe+=' -an -c:v mjpeg -f image2pipe'+x.cust_stream+x.stream_video_filters+' pipe:1';
+                if(e.coProcessor === false){
+                    if(e.details.stream_quality && e.details.stream_quality !== '')x.cust_stream+=' -q:v '+e.details.stream_quality;
+                    if(x.dimensions && x.cust_stream.indexOf('-s ')===-1){x.cust_stream+=' -s '+x.dimensions}
+                    x.pipe+=' -an -c:v mjpeg -f image2pipe'+x.cust_stream+x.stream_video_filters+' pipe:1';
+                }
             break;
             default:
                 x.pipe=''
@@ -585,11 +603,12 @@ module.exports = function(s,config,onFinish){
         }
         if(e.details.stream_channels){
             e.details.stream_channels.forEach(function(v,n){
+                // if(v.stream_type === 'mjpeg')e.coProcessor = true;
                 x.pipe += s.createStreamChannel(e,n+config.pipeAddition,v)
             })
         }
         //api - snapshot bin/ cgi.bin (JPEG Mode)
-        if(e.details.snap === '1'){
+        if(e.details.snap === '1' && e.coProcessor === false){
             if(e.details.input_map_choices&&e.details.input_map_choices.snap){
                 //add input feed map
                 x.pipe += s.createFFmpegMap(e,e.details.input_map_choices.snap)
@@ -628,7 +647,7 @@ module.exports = function(s,config,onFinish){
             x.dimensions = e.details.stream_scale_x+'x'+e.details.stream_scale_y;
         }
         //record - segmenting
-        x.segment=' -f segment -segment_atclocktime 1 -reset_timestamps 1 -strftime 1 -segment_list pipe:2 -segment_time '+(60*e.cutoff)+' "'+e.dir+'%Y-%m-%dT%H-%M-%S.'+e.ext+'"';
+        x.segment=' -f segment -segment_format_options movflags=faststart+frag_keyframe+empty_moov -segment_atclocktime 1 -reset_timestamps 1 -strftime 1 -segment_list pipe:2 -segment_time '+(60*e.cutoff)+' "'+e.dir+'%Y-%m-%dT%H-%M-%S.'+e.ext+'"';
         //record - set defaults for extension, video quality
         switch(e.ext){
             case'mp4':
@@ -732,40 +751,67 @@ module.exports = function(s,config,onFinish){
             x.record_string+=x.vcodec+x.record_fps+x.record_video_filters+x.record_dimensions+x.segment;
         }
     }
+    ffmpeg.buildAudioDetector = function(e,x){
+        if(e.details.detector_audio === '1'){
+            if(e.details.input_map_choices&&e.details.input_map_choices.detector_audio){
+                //add input feed map
+                x.pipe += s.createFFmpegMap(e,e.details.input_map_choices.detector_audio)
+            }else{
+                x.pipe += ' -map 0:a'
+            }
+            x.pipe += ' -acodec pcm_s16le -f s16le -ac 1 -ar 16000 pipe:6'
+        }
+    }
     ffmpeg.buildMainDetector = function(e,x){
         //e = monitor object
         //x = temporary values
         x.cust_detect = ' '
         //detector - plugins, motion
-        if(e.details.detector === '1' && e.details.detector_send_frames === '1'){
-            if(e.details.input_map_choices&&e.details.input_map_choices.detector){
+        var sendFramesGlobally = (e.details.detector_send_frames === '1')
+        var sendFramesToObjectDetector = (e.details.detector_send_frames_object !== '0' && e.details.detector_use_detect_object === '1')
+        if(e.details.detector === '1' && (sendFramesGlobally || sendFramesToObjectDetector) && e.coProcessor === false){
+            if(sendFramesGlobally && e.details.input_map_choices && e.details.input_map_choices.detector){
                 //add input feed map
                 x.pipe += s.createFFmpegMap(e,e.details.input_map_choices.detector)
             }
-            if(!e.details.detector_fps||e.details.detector_fps===''){e.details.detector_fps=2}
-            if(e.details.detector_scale_x&&e.details.detector_scale_x!==''&&e.details.detector_scale_y&&e.details.detector_scale_y!==''){x.dratio=' -s '+e.details.detector_scale_x+'x'+e.details.detector_scale_y}else{x.dratio=' -s 320x240'}
+            if(!e.details.detector_fps || e.details.detector_fps === ''){x.detector_fps = 2}else{x.detector_fps = parseInt(e.details.detector_fps)}
+            if(e.details.detector_scale_x && e.details.detector_scale_x !== '' && e.details.detector_scale_y && e.details.detector_scale_y !== ''){x.dratio=' -s '+e.details.detector_scale_x+'x'+e.details.detector_scale_y}else{x.dratio=' -s 320x240'}
             if(e.details.cust_detect&&e.details.cust_detect!==''){x.cust_detect+=e.details.cust_detect;}
-            x.detector_vf = ['fps='+e.details.detector_fps]
+            if(sendFramesGlobally)x.pipe += ' -r ' + x.detector_fps + x.dratio + x.cust_detect
+            x.detector_vf = []
             if(e.cudaEnabled){
                 x.detector_vf.push('hwdownload,format=nv12')
             }
-            x.detector_vf = '-vf "'+x.detector_vf.join(',')+'"'
-            if(e.details.detector_pam==='1'){
-                if(e.cudaEnabled){
+            if(sendFramesGlobally && x.detector_vf.length > 0)x.pipe += ' -vf "'+x.detector_vf.join(',')+'"'
+
+            var h264Output = ' -q:v 1 -an -c:v libx264 -f hls -tune zerolatency -g 1 -hls_time 2 -hls_list_size 3 -start_number 0 -live_start_index 3 -hls_allow_cache 0 -hls_flags +delete_segments+omit_endlist "'+e.sdir+'detectorStreamX.m3u8"'
+            if(e.details.detector_pam === '1'){
+                if(sendFramesGlobally && e.cudaEnabled){
                     x.pipe += ' -vf "hwdownload,format=nv12"'
                 }
-                x.pipe+=' -an -c:v pam -pix_fmt gray -f image2pipe -r '+e.details.detector_fps+x.cust_detect+x.dratio+' pipe:3'
+                if(sendFramesGlobally)x.pipe += ' -an -c:v pam -pix_fmt gray -f image2pipe pipe:3'
                 if(e.details.detector_use_detect_object === '1'){
                     //for object detection
                     x.pipe += s.createFFmpegMap(e,e.details.input_map_choices.detector)
-                    x.pipe += ' -f singlejpeg '+x.detector_vf+x.cust_detect+x.dratio+' pipe:4';
+                    if(e.details.detector_scale_x_object&&e.details.detector_scale_x_object!==''&&e.details.detector_scale_y_object&&e.details.detector_scale_y_object!==''){x.dobjratio=' -s '+e.details.detector_scale_x_object+'x'+e.details.detector_scale_y_object}else{x.dobjratio=x.dratio}
+                    x.pipe += ' -r ' + x.detector_fps + x.dobjratio + x.cust_detect
+                    if(e.details.detector_h264 === '1'){
+                        x.pipe += h264Output
+                    }else{
+                        x.pipe += ' -an -f singlejpeg pipe:4'
+                    }
+                }
+            }else if(sendFramesGlobally){
+                if(e.details.detector_h264 === '1'){
+                    x.pipe += h264Output
+                }else{
+                    x.pipe += ' -an -f singlejpeg pipe:3'
                 }
-            }else{
-                x.pipe+=' -f image2pipe '+x.detector_vf+x.cust_detect+x.dratio+' pipe:3';
             }
         }
         //Traditional Recording Buffer
         if(e.details.detector=='1'&&e.details.detector_trigger=='1'&&e.details.detector_record_method==='sip'){
+            if(e.details.cust_sip_record && e.details.cust_sip_record !== ''){x.pipe += ' ' + e.details.cust_sip_record}
             if(e.details.input_map_choices&&e.details.input_map_choices.detector_sip_buffer){
                 //add input feed map
                 x.pipe += s.createFFmpegMap(e,e.details.input_map_choices.detector_sip_buffer)
@@ -809,7 +855,6 @@ module.exports = function(s,config,onFinish){
             if(!e.details.detector_buffer_hls_list_size||e.details.detector_buffer_hls_list_size===''){e.details.detector_buffer_hls_list_size='4'}
             if(!e.details.detector_buffer_start_number||e.details.detector_buffer_start_number===''){e.details.detector_buffer_start_number='0'}
             if(!e.details.detector_buffer_live_start_index||e.details.detector_buffer_live_start_index===''){e.details.detector_buffer_live_start_index='-3'}
-
             if(e.details.detector_buffer_vcodec.indexOf('_vaapi')>-1){
                 if(x.hwaccel.indexOf('-vaapi_device')>-1){
                     x.detector_buffer_filters.push('format=nv12')
@@ -830,7 +875,13 @@ module.exports = function(s,config,onFinish){
             if(x.detector_buffer_filters.length>0){
                 x.pipe+=' -vf '+x.detector_buffer_filters.join(',')
             }
-            x.pipe+=x.detector_buffer_fps+x.detector_buffer_acodec+' -c:v '+e.details.detector_buffer_vcodec+' -f hls -tune '+e.details.detector_buffer_tune+' -g '+e.details.detector_buffer_g+' -hls_time '+e.details.detector_buffer_hls_time+' -hls_list_size '+e.details.detector_buffer_hls_list_size+' -start_number '+e.details.detector_buffer_start_number+' -live_start_index '+e.details.detector_buffer_live_start_index+' -hls_allow_cache 0 -hls_flags +delete_segments+omit_endlist "'+e.sdir+'detectorStream.m3u8"'
+            x.pipe += x.detector_buffer_fps+x.detector_buffer_acodec+' -c:v '+e.details.detector_buffer_vcodec+' -f hls -tune '+e.details.detector_buffer_tune+' -g '+e.details.detector_buffer_g+' -hls_time '+e.details.detector_buffer_hls_time+' -hls_list_size '+e.details.detector_buffer_hls_list_size+' -start_number '+e.details.detector_buffer_start_number+' -live_start_index '+e.details.detector_buffer_live_start_index+' -hls_allow_cache 0 -hls_flags +delete_segments+omit_endlist "'+e.sdir+'detectorStream.m3u8"'
+        }
+    }
+    ffmpeg.buildCoProcessorFeed = function(e,x){
+        if(e.coProcessor === true){
+            // the coProcessor ffmpeg consumes this HLS stream (no audio, frames only)
+            x.pipe += ' -q:v 1 -an -c:v copy -f hls -tune zerolatency -g 1 -hls_time 2 -hls_list_size 3 -start_number 0 -live_start_index 3 -hls_allow_cache 0 -hls_flags +delete_segments+omit_endlist "'+e.sdir+'coProcessor.m3u8"'
         }
     }
     ffmpeg.assembleMainPieces = function(e,x){
@@ -849,6 +900,10 @@ module.exports = function(s,config,onFinish){
             case'mjpeg':
                 x.ffmpegCommandString += ' -reconnect 1 -f mjpeg'+x.cust_input+x.hwaccel+' -i "'+e.url+'"';
             break;
+            case'rtmp':
+                if(!e.details.rtmp_key)e.details.rtmp_key = ''
+                x.ffmpegCommandString += x.cust_input+x.hwaccel+` -i "rtmp://127.0.0.1:1935/${e.ke + '_' + e.mid + '_' + e.details.rtmp_key}"`;
+            break;
             case'h264':case'hls':case'mp4':
                 x.ffmpegCommandString += x.cust_input+x.hwaccel+' -i "'+e.url+'"';
             break;
@@ -883,7 +938,12 @@ module.exports = function(s,config,onFinish){
         ffmpeg.buildMainInput(e,x)
         ffmpeg.buildMainStream(e,x)
         ffmpeg.buildMainRecording(e,x)
+        ffmpeg.buildAudioDetector(e,x)
         ffmpeg.buildMainDetector(e,x)
+        ffmpeg.buildCoProcessorFeed(e,x)
+        s.onFfmpegCameraStringCreationExtensions.forEach(function(extender){
+            extender(e,x)
+        })
         ffmpeg.assembleMainPieces(e,x)
         ffmpeg.createPipeArray(e,x)
         //hold ffmpeg command for log stream
@@ -895,9 +955,9 @@ module.exports = function(s,config,onFinish){
     }
     if(!config.ffmpegDir){
         ffmpeg.checkForWindows(function(){
-            ffmpeg.checkForUnix(function(){
-                ffmpeg.checkForFfbinary(function(){
-                    ffmpeg.checkForNpmStatic(function(){
+            ffmpeg.checkForFfbinary(function(){
+                ffmpeg.checkForNpmStatic(function(){
+                    ffmpeg.checkForUnix(function(){
                         console.log('No FFmpeg found.')
                     })
                 })

libs/ffmpegCoProcessor.js

@@ -0,0 +1,210 @@
+var spawn = require('child_process').spawn;
+module.exports = function(s,config,lang,ffmpeg){
+    ffmpeg.buildCoProcessorInput = function(e,x){
+        if(e.details.userLoglevel&&e.details.userLoglevel!==''){x.loglevel='-loglevel '+e.details.userLoglevel;}else{x.loglevel='-loglevel error'}
+        x.input = x.loglevel+' -re -i '+e.sdir+'coProcessor.m3u8'
+    }
+    ffmpeg.buildCoProcessorStream = function(e,x){
+        x.stream_video_filters = []
+        //stream - timestamp
+        if(e.details.stream_timestamp&&e.details.stream_timestamp=="1"&&e.details.vcodec!=='copy'){
+            //font
+            if(e.details.stream_timestamp_font&&e.details.stream_timestamp_font!==''){x.stream_timestamp_font=e.details.stream_timestamp_font}else{x.stream_timestamp_font='/usr/share/fonts/truetype/freefont/FreeSans.ttf'}
+            //position x
+            if(e.details.stream_timestamp_x&&e.details.stream_timestamp_x!==''){x.stream_timestamp_x=e.details.stream_timestamp_x}else{x.stream_timestamp_x='(w-tw)/2'}
+            //position y
+            if(e.details.stream_timestamp_y&&e.details.stream_timestamp_y!==''){x.stream_timestamp_y=e.details.stream_timestamp_y}else{x.stream_timestamp_y='0'}
+            //text color
+            if(e.details.stream_timestamp_color&&e.details.stream_timestamp_color!==''){x.stream_timestamp_color=e.details.stream_timestamp_color}else{x.stream_timestamp_color='white'}
+            //box color
+            if(e.details.stream_timestamp_box_color&&e.details.stream_timestamp_box_color!==''){x.stream_timestamp_box_color=e.details.stream_timestamp_box_color}else{x.stream_timestamp_box_color='0x00000000@1'}
+            //text size
+            if(e.details.stream_timestamp_font_size&&e.details.stream_timestamp_font_size!==''){x.stream_timestamp_font_size=e.details.stream_timestamp_font_size}else{x.stream_timestamp_font_size='10'}
+
+            x.stream_video_filters.push('drawtext=fontfile='+x.stream_timestamp_font+':text=\'%{localtime}\':x='+x.stream_timestamp_x+':y='+x.stream_timestamp_y+':fontcolor='+x.stream_timestamp_color+':box=1:boxcolor='+x.stream_timestamp_box_color+':fontsize='+x.stream_timestamp_font_size);
+        }
+        //stream - watermark for -vf
+        if(e.details.stream_watermark&&e.details.stream_watermark=="1"&&e.details.stream_watermark_location&&e.details.stream_watermark_location!==''){
+            switch(e.details.stream_watermark_position){
+                case'tl'://top left
+                    x.stream_watermark_position='10:10'
+                break;
+                case'tr'://top right
+                    x.stream_watermark_position='main_w-overlay_w-10:10'
+                break;
+                case'bl'://bottom left
+                    x.stream_watermark_position='10:main_h-overlay_h-10'
+                break;
+                default://bottom right
+                    x.stream_watermark_position='(main_w-overlay_w-10)/2:(main_h-overlay_h-10)/2'
+                break;
+            }
+            x.stream_video_filters.push('movie='+e.details.stream_watermark_location+'[watermark],[in][watermark]overlay='+x.stream_watermark_position+'[out]');
+        }
+        //stream - rotation
+        if(e.details.rotate_stream&&e.details.rotate_stream!==""&&e.details.rotate_stream!=="no"&&e.details.stream_vcodec!=='copy'){
+            x.stream_video_filters.push('transpose='+e.details.rotate_stream);
+        }
+        if(e.details.svf&&e.details.svf!==''){
+            x.stream_video_filters.push(e.details.svf)
+        }
+        if(x.stream_video_filters.length>0){
+            x.stream_video_filters=' -vf '+x.stream_video_filters.join(',')
+        }else{
+            x.stream_video_filters=''
+        }
+        if(e.details.cust_stream&&e.details.cust_stream!==''){x.cust_stream=' '+e.details.cust_stream}else{x.cust_stream=''}
+        if(e.details.stream_fps&&e.details.stream_fps!==''){x.stream_fps=' -r '+e.details.stream_fps}else{x.stream_fps=''}
+        if(e.details.stream_vcodec !== 'copy' || e.details.stream_type === 'mjpeg' || e.details.stream_type === 'b64'){
+            x.cust_stream += x.stream_fps
+        }
+        switch(e.details.stream_type){
+            case'mjpeg':
+                if(e.details.stream_quality && e.details.stream_quality !== '')x.cust_stream+=' -q:v '+e.details.stream_quality;
+                if(x.dimensions && x.cust_stream.indexOf('-s ')===-1){x.cust_stream+=' -s '+x.dimensions}
+                x.pipe += ' -an -c:v mjpeg -f mpjpeg -boundary_tag shinobi'+x.cust_stream+x.stream_video_filters+' pipe:1';
+            break;
+            case'b64':case'':case undefined:case null://base64
+                if(e.details.stream_quality && e.details.stream_quality !== '')x.cust_stream+=' -q:v '+e.details.stream_quality;
+                if(x.dimensions && x.cust_stream.indexOf('-s ')===-1){x.cust_stream+=' -s '+x.dimensions}
+                x.pipe += ' -an -c:v mjpeg -f image2pipe'+x.cust_stream+x.stream_video_filters+' pipe:1';
+            break;
+        }
+    }
+    ffmpeg.buildCoProcessorDetector = function(e,x){
+        //detector frames
+        x.cust_detect=' '
+        if(e.details.detector === '1'){
+            if(e.details.detector_fps && e.details.detector_fps !== ''){
+                x.detector_fps = e.details.detector_fps
+            }else{
+                x.detector_fps = '2'
+            }
+            if(e.details.detector_scale_x && e.details.detector_scale_x !== '' && e.details.detector_scale_y && e.details.detector_scale_y !== ''){
+                x.dratio=' -s '+e.details.detector_scale_x+'x'+e.details.detector_scale_y
+            }else{
+                x.dratio=' -s 320x240'
+            }
+
+            if(e.details.cust_detect&&e.details.cust_detect!==''){x.cust_detect+=e.details.cust_detect;}
+            if(e.details.detector_pam==='1'){
+                x.pipe += ' -an -c:v pam -pix_fmt gray -f image2pipe -r '+x.detector_fps+x.cust_detect+x.dratio+' pipe:3'
+                if(e.details.detector_use_detect_object === '1'){
+                    if(e.details.detector_use_motion === '1'){
+                        if(e.details.detector_scale_x_object && e.details.detector_scale_x_object !== '' && e.details.detector_scale_y_object && e.details.detector_scale_y_object !== ''){
+                            x.dratio=' -s '+e.details.detector_scale_x_object+'x'+e.details.detector_scale_y_object
+                        }
+                        if(e.details.detector_fps_object && e.details.detector_fps_object !== ''){
+                            x.detector_fps = e.details.detector_fps_object
+                        }
+                    }
+                    //for object detection
+                    x.pipe += s.createFFmpegMap(e,e.details.input_map_choices.detector)
+                    x.pipe += ' -f singlejpeg -vf fps='+x.detector_fps+x.cust_detect+x.dratio+' pipe:4';
+                }
+            }else{
+                x.pipe+=' -f singlejpeg -vf fps='+x.detector_fps+x.cust_detect+x.dratio+' pipe:3';
+            }
+        }
+    }
+    ffmpeg.buildCoProcessorJpegApi = function(e,x){
+        //snapshot frames
+        if(e.details.snap === '1'){
+            if(!e.details.snap_fps || e.details.snap_fps === ''){e.details.snap_fps = 1}
+            if(e.details.snap_vf && e.details.snap_vf !== ''){x.snap_vf=' -vf '+e.details.snap_vf}else{x.snap_vf=''}
+            if(e.details.snap_scale_x && e.details.snap_scale_x !== '' && e.details.snap_scale_y && e.details.snap_scale_y !== ''){x.snap_ratio = ' -s '+e.details.snap_scale_x+'x'+e.details.snap_scale_y}else{x.snap_ratio=''}
+            if(e.details.cust_snap && e.details.cust_snap !== ''){x.cust_snap = ' '+e.details.cust_snap}else{x.cust_snap=''}
+            x.pipe += ' -update 1 -r '+e.details.snap_fps+x.cust_snap+x.snap_ratio+x.snap_vf+' "'+e.sdir+'s.jpg" -y';
+        }
+    }
+    ffmpeg.buildCoProcessorPipeArray = function(e,x){
+        x.stdioPipes = [];
+        var times = config.pipeAddition;
+        if(e.details.stream_channels){
+            times+=e.details.stream_channels.length
+        }
+        for(var i=0; i < times; i++){
+            x.stdioPipes.push('pipe')
+        }
+    }
+    s.ffmpegCoProcessor = function(e){
+        if(e.coProcessor === false)return;
+        var x = {}
+        x.pipe = ''
+        ffmpeg.buildCoProcessorInput(e,x)
+        ffmpeg.buildCoProcessorStream(e,x)
+        ffmpeg.buildCoProcessorDetector(e,x)
+        ffmpeg.buildCoProcessorJpegApi(e,x)
+        ffmpeg.buildCoProcessorPipeArray(e,x)
+        var commandString = x.input + x.pipe
+        if(commandString === x.input){
+            return false
+        }
+        s.group[e.ke].mon[e.mid].coProcessorCmd = commandString
+        return spawn(config.ffmpegDir,s.splitForFFPMEG((commandString).replace(/\s+/g,' ').trim()),{detached: true,stdio:x.stdioPipes})
+    }
+    s.coSpawnLauncher = function(e){
+        if(s.group[e.ke].mon[e.id].isStarted === true && e.coProcessor === true){
+            s.coSpawnClose(e)
+            s.group[e.ke].mon[e.id].coSpawnProcessor = s.ffmpegCoProcessor(e)
+            if(s.group[e.ke].mon[e.id].coSpawnProcessor === false){
+                return
+            }
+            s.userLog(e,{type:lang['coProcessor Started'],msg:{msg:lang.coProcessorTextStarted,cmd:s.group[e.ke].mon[e.id].coProcessorCmd}});
+            s.group[e.ke].mon[e.id].coSpawnProcessorExit = function(){
+                s.userLog(e,{type:lang['coProcess Unexpected Exit'],msg:{msg:lang['coProcess Crashed for Monitor']+' : '+e.id,cmd:s.group[e.ke].mon[e.id].coProcessorCmd}});
+                setTimeout(function(){
+                    s.coSpawnLauncher(e)
+                },2000)
+            }
+            s.group[e.ke].mon[e.id].coSpawnProcessor.on('end',s.group[e.ke].mon[e.id].coSpawnProcessorExit)
+            s.group[e.ke].mon[e.id].coSpawnProcessor.on('exit',s.group[e.ke].mon[e.id].coSpawnProcessorExit)
+            var checkLog = function(d,x){return d.indexOf(x)>-1;}
+            s.group[e.ke].mon[e.id].coSpawnProcessor.stderr.on('data',function(d){
+                d=d.toString();
+                switch(true){
+                    case checkLog(d,'deprecated pixel format used'):
+                    case checkLog(d,'[hls @'):
+                    case checkLog(d,'Past duration'):
+                    case checkLog(d,'Last message repeated'):
+                    case checkLog(d,'pkt->duration = 0'):
+                    case checkLog(d,'Non-monotonous DTS'):
+                    case checkLog(d,'NULL @'):
+                        return
+                    break;
+                }
+                s.userLog(e,{type:lang.coProcessor,msg:d});
+            })
+            if(e.frame_to_stream){
+                s.group[e.ke].mon[e.id].coSpawnProcessor.stdout.on('data',e.frame_to_stream)
+            }
+            if(e.details.detector === '1'){
+                s.ocvTx({f:'init_monitor',id:e.id,ke:e.ke})
+                //frames from motion detect
+                if(e.details.detector_pam === '1'){
+                   s.createPamDiffEngine(e)
+                   s.group[e.ke].mon[e.id].coSpawnProcessor.stdio[3].pipe(s.group[e.ke].mon[e.id].p2p).pipe(s.group[e.ke].mon[e.id].pamDiff)
+                    if(e.details.detector_use_detect_object === '1'){
+                        s.group[e.ke].mon[e.id].coSpawnProcessor.stdio[4].on('data',function(d){
+                            s.group[e.ke].mon[e.id].lastJpegDetectorFrame = d
+                        })
+                    }
+                }else if(s.isAtleatOneDetectorPluginConnected){
+                    s.group[e.ke].mon[e.id].coSpawnProcessor.stdio[3].on('data',function(d){
+                        s.ocvTx({f:'frame',mon:s.group[e.ke].mon_conf[e.id].details,ke:e.ke,id:e.id,time:s.formattedTime(),frame:d});
+                    })
+                }
+            }
+        }
+    }
+    s.coSpawnClose = function(e){
+        if(s.group[e.ke].mon[e.id].coSpawnProcessor){
+            s.group[e.ke].mon[e.id].coSpawnProcessor.removeListener('end',s.group[e.ke].mon[e.id].coSpawnProcessorExit);
+            s.group[e.ke].mon[e.id].coSpawnProcessor.removeListener('exit',s.group[e.ke].mon[e.id].coSpawnProcessorExit);
+            s.group[e.ke].mon[e.id].coSpawnProcessor.stdin.pause()
+            s.group[e.ke].mon[e.id].coSpawnProcessor.kill()
+            delete(s.group[e.ke].mon[e.id].coSpawnProcessor)
+            s.userLog(e,{type:lang['coProcessor Stopped'],msg:{msg:lang.coProcessorTextStopped+' : '+e.id}});
+        }
+    }
+}

libs/language.js

@@ -18,42 +18,50 @@ module.exports = function(s,config){
         var definitions = require(s.location.definitions+'/en_CA.json');
     }
     //load languages dynamically
+    s.copySystemDefaultLanguage = function(){
+        //en_CA
+        return Object.assign(lang,{})
+    }
     s.loadedLanguages={}
-    s.loadedLanguages[config.language]=lang;
+    s.loadedLanguages[config.language] = s.copySystemDefaultLanguage()
     s.getLanguageFile = function(rule){
         if(rule && rule !== ''){
             var file = s.loadedLanguages[file]
             if(!file){
                 try{
                     s.loadedLanguages[rule] = require(s.location.languages+'/'+rule+'.json')
-                    s.loadedLanguages[rule] = Object.assign(lang,s.loadedLanguages[rule])
+                    s.loadedLanguages[rule] = Object.assign(s.copySystemDefaultLanguage(),s.loadedLanguages[rule])
                     file = s.loadedLanguages[rule]
                 }catch(err){
-                    file = lang
+                    file = s.copySystemDefaultLanguage()
                 }
             }
         }else{
-            file = lang
+            file = s.copySystemDefaultLanguage()
         }
         return file
     }
     //load defintions dynamically
+    s.copySystemDefaultDefinitions = function(){
+        //en_CA
+        return Object.assign(definitions,{})
+    }
     s.loadedDefinitons={}
-    s.loadedDefinitons[config.language]=definitions;
+    s.loadedDefinitons[config.language] = s.copySystemDefaultDefinitions()
     s.getDefinitonFile = function(rule){
         if(rule && rule !== ''){
             var file = s.loadedDefinitons[file]
             if(!file){
                 try{
                     s.loadedDefinitons[rule] = require(s.location.definitions+'/'+rule+'.json')
-                    s.loadedDefinitons[rule] = Object.assign(definitions,s.loadedDefinitons[rule])
+                    s.loadedDefinitons[rule] = Object.assign(s.copySystemDefaultDefinitions(),s.loadedDefinitons[rule])
                     file = s.loadedDefinitons[rule]
                 }catch(err){
-                    file = definitions
+                    file = s.copySystemDefaultDefinitions()
                 }
             }
         }else{
-            file = definitions
+            file = s.copySystemDefaultDefinitions()
         }
         return file
     }

libs/monitor.js

@@ -6,6 +6,7 @@ var Mp4Frag = require('mp4frag');
 var onvif = require('node-onvif');
 var request = require('request');
 var connectionTester = require('connection-tester')
+var SoundDetection = require('shinobi-sound-detection')
 var URL = require('url')
 module.exports = function(s,config,lang){
     s.initiateMonitorObject = function(e){
@@ -21,6 +22,7 @@ module.exports = function(s,config,lang){
         if(!s.group[e.ke].mon[e.mid].eventBasedRecording){s.group[e.ke].mon[e.mid].eventBasedRecording={}};
         if(!s.group[e.ke].mon[e.mid].watch){s.group[e.ke].mon[e.mid].watch={}};
         if(!s.group[e.ke].mon[e.mid].fixingVideos){s.group[e.ke].mon[e.mid].fixingVideos={}};
+        if(!s.group[e.ke].mon[e.mid].parsedObjects){s.group[e.ke].mon[e.mid].parsedObjects={}};
         if(!s.group[e.ke].mon[e.mid].isStarted){s.group[e.ke].mon[e.mid].isStarted = false};
         if(s.group[e.ke].mon[e.mid].delete){clearTimeout(s.group[e.ke].mon[e.mid].delete)}
         if(!s.group[e.ke].mon_conf){s.group[e.ke].mon_conf={}}
@@ -103,7 +105,7 @@ module.exports = function(s,config,lang){
                 var snapBuffer = []
                 var snapProcess = spawn(config.ffmpegDir,('-loglevel quiet -re -i '+url+options+' -frames:v 1 -f image2pipe pipe:1').split(' '),{detached: true})
                 snapProcess.stdout.on('data',function(data){
-                    snapBuffer.push(data)
+                    if(snapBuffer)snapBuffer.push(data)
                 })
                 snapProcess.stderr.on('data',function(data){
                     console.log(data.toString())
@@ -163,6 +165,9 @@ module.exports = function(s,config,lang){
         var streamDirItems = fs.readdirSync(pathDir)
         var items = []
         var copiedItems = []
+        var videoLength = s.group[monitor.ke].mon_conf[monitor.id].details.detector_send_video_length
+        if(!videoLength || videoLength === '')videoLength = '10'
+        if(videoLength.length === 1)videoLength = '0' + videoLength
         var createMerged = function(copiedItems){
             var allts = pathDir+items.join('_')
             fs.stat(allts,function(err,stats){
@@ -170,7 +175,7 @@ module.exports = function(s,config,lang){
                     //not exist
                     var cat = 'cat '+copiedItems.join(' ')+' > '+allts
                     exec(cat,function(){
-                        var merger = spawn(config.ffmpegDir,s.splitForFFPMEG(('-re -i '+allts+' -acodec copy -vcodec copy '+pathDir+mergedFile)))
+                        var merger = spawn(config.ffmpegDir,s.splitForFFPMEG(('-re -i '+allts+' -acodec copy -vcodec copy -t 00:00:' + videoLength + ' '+pathDir+mergedFile)))
                         merger.stderr.on('data',function(data){
                             s.userLog(monitor,{type:"Buffer Merge",msg:data.toString()})
                         })
@@ -198,7 +203,7 @@ module.exports = function(s,config,lang){
             }
         })
         items.sort()
-        items = items.slice(items.length - 5,items.length)
+        // items = items.slice(items.length - 5,items.length)
         items.forEach(function(filename){
             try{
                 var tempFilename = filename.split('.')
@@ -218,6 +223,64 @@ module.exports = function(s,config,lang){
         })
         return items
     }
+    s.mergeRecordedVideos = function(videoRows,groupKey,callback){
+        var tempDir = s.dir.streams + groupKey + '/'
+        var pathDir = s.dir.fileBin + groupKey + '/'
+        var streamDirItems = fs.readdirSync(pathDir)
+        var items = []
+        var mergedFile = []
+        videoRows.forEach(function(video){
+            var filepath = s.getVideoDirectory(video) + s.formattedTime(video.time) + '.' + video.ext
+            if(
+                filepath.indexOf('.mp4') > -1
+                // || filename.indexOf('.webm') > -1
+            ){
+                mergedFile.push(s.formattedTime(video.time))
+                items.push(filepath)
+            }
+        })
+        mergedFile.sort()
+        mergedFile = mergedFile.join('_') + '.mp4'
+        var mergedFilepath = pathDir + mergedFile
+        var mergedRawFilepath = pathDir + 'raw_' + mergedFile
+        items.sort()
+        fs.stat(mergedFilepath,function(err,stats){
+            if(err){
+                //not exist
+                var tempScriptPath = tempDir + s.gid(5) + '.sh'
+                var cat = 'cat '+items.join(' ')+' > '+mergedRawFilepath
+                fs.writeFileSync(tempScriptPath,cat,'utf8')
+                exec('sh ' + tempScriptPath,function(){
+                    s.userLog({
+                        ke: groupKey,
+                        mid: '$USER'
+                    },{type:lang['Videos Merge'],msg:mergedFile})
+                    var merger = spawn(config.ffmpegDir,s.splitForFFPMEG(('-re -loglevel warning -i ' + mergedRawFilepath + ' -acodec copy -vcodec copy ' + mergedFilepath)))
+                    merger.stderr.on('data',function(data){
+                        s.userLog({
+                            ke: groupKey,
+                            mid: '$USER'
+                        },{type:lang['Videos Merge'],msg:data.toString()})
+                    })
+                    merger.on('close',function(){
+                        s.file('delete',mergedRawFilepath)
+                        s.file('delete',tempScriptPath)
+                        setTimeout(function(){
+                            fs.stat(mergedFilepath,function(err,stats){
+                                if(!err)s.file('delete',mergedFilepath)
+                            })
+                        },1000 * 60 * 60 * 24)
+                        delete(merger)
+                        callback(mergedFilepath,mergedFile)
+                    })
+                })
+            }else{
+                //file exist
+                callback(mergedFilepath,mergedFile)
+            }
+        })
+        return items
+    }
 
     s.cameraDestroy = function(x,e,p){
         if(s.group[e.ke]&&s.group[e.ke].mon[e.id]&&s.group[e.ke].mon[e.id].spawn !== undefined){
@@ -266,6 +329,7 @@ module.exports = function(s,config,lang){
             if(s.group[e.ke].mon[e.id].childNode){
                 s.cx({f:'kill',d:s.cleanMonitorObject(e)},s.group[e.ke].mon[e.id].childNodeId)
             }else{
+                s.coSpawnClose(e)
                 if(!x||x===1){return};
                 p=x.pid;
                 if(s.group[e.ke].mon_conf[e.id].type===('dashcam'||'socket'||'jpeg'||'pipe')){
@@ -282,12 +346,20 @@ module.exports = function(s,config,lang){
     s.cameraCheckObjectsInDetails = function(e){
         //parse Objects
         (['detector_cascades','cords','detector_filters','input_map_choices']).forEach(function(v){
-            if(e.details&&e.details[v]&&(e.details[v] instanceof Object)===false){
+            if(e.details && e.details[v]){
                 try{
-                    if(e.details[v] === '') e.details[v] = '{}'
-                    e.details[v]=JSON.parse(e.details[v]);
-                    if(!e.details[v])e.details[v]={};
-                    s.group[e.ke].mon[e.id].details = e.details;
+                    if(!e.details[v] || e.details[v] === '')e.details[v] = '{}'
+                    e.details[v] = s.parseJSON(e.details[v])
+                    if(!e.details[v])e.details[v] = {}
+                    s.group[e.ke].mon[e.id].details = e.details
+                    switch(v){
+                        case'cords':
+                            s.group[e.ke].mon[e.id].parsedObjects[v] = Object.values(s.parseJSON(e.details[v]))
+                        break;
+                        default:
+                            s.group[e.ke].mon[e.id].parsedObjects[v] = s.parseJSON(e.details[v])
+                        break;
+                    }
                 }catch(err){
 
                 }
@@ -595,6 +667,14 @@ module.exports = function(s,config,lang){
         // exec('chmod -R 777 '+e.sdir,function(err){
         //
         // })
+        var binDir = s.dir.fileBin + e.ke + '/'
+        if (!fs.existsSync(binDir)){
+            fs.mkdirSync(binDir)
+        }
+        binDir = s.dir.fileBin + e.ke + '/' + e.id + '/'
+        if (!fs.existsSync(binDir)){
+            fs.mkdirSync(binDir)
+        }
         return setStreamDir
     }
     s.stripAuthFromHost = function(e){
@@ -741,6 +821,9 @@ module.exports = function(s,config,lang){
                 }
                 s.fatalCameraError(e,'Process Unexpected Exit');
                 s.orphanedVideoCheck(e,2,null,true)
+                s.onMonitorUnexpectedExitExtensions.forEach(function(extender){
+                    extender(Object.assign(s.group[e.ke].mon_conf[e.id],{}),e)
+                })
             }
         }
         s.group[e.ke].mon[e.id].spawn.on('end',s.group[e.ke].mon[e.id].spawn_exit)
@@ -773,7 +856,47 @@ module.exports = function(s,config,lang){
         if(e.type==='jpeg'){
             s.cameraPullJpegStream(e)
         }
-        if(e.details.detector === '1'){
+        if(e.details.detector_audio === '1'){
+            var triggerLevel
+            var triggerLevelMax
+            if(e.details.detector_audio_min_db && e.details.detector_audio_min_db !== ''){
+                triggerLevel = parseInt(e.details.detector_audio_min_db)
+            }else{
+                triggerLevel = 5
+            }
+            if(e.details.detector_audio_max_db && e.details.detector_audio_max_db !== ''){
+                triggerLevelMax = parseInt(e.details.detector_audio_max_db)
+            }
+            var audioDetector = new SoundDetection({
+                format: {
+                    bitDepth: 16,
+                    numberOfChannels: 1,
+                    signed: true
+                },
+                triggerLevel: triggerLevel,
+                triggerLevelMax: triggerLevelMax
+            },function(dB) {
+                s.triggerEvent({
+                    f:'trigger',
+                    id:e.id,
+                    ke:e.ke,
+                    name: 'db',
+                    details:{
+                        plug:'audio',
+                        name:'db',
+                        reason:'soundChange',
+                        confidence:dB
+                    },
+                    plates:[],
+                    imgHeight:e.details.detector_scale_y,
+                    imgWidth:e.details.detector_scale_x
+                })
+            })
+            s.group[e.ke].mon[e.id].audioDetector = audioDetector
+            audioDetector.start()
+            s.group[e.ke].mon[e.id].spawn.stdio[6].pipe(audioDetector.streamDecoder)
+        }
+        if(e.details.detector === '1' && e.coProcessor === false){
             s.ocvTx({f:'init_monitor',id:e.id,ke:e.ke})
             //frames from motion detect
             if(e.details.detector_pam === '1'){
@@ -784,34 +907,10 @@ module.exports = function(s,config,lang){
                         s.group[e.ke].mon[e.id].lastJpegDetectorFrame = d
                     })
                 }
-            }else if(s.ocv){
-                if(s.ocv.connectionType !== 'ram'){
-                    s.group[e.ke].mon[e.id].spawn.stdio[3].on('data',function(d){
-                        s.ocvTx({f:'frame',mon:s.group[e.ke].mon_conf[e.id].details,ke:e.ke,id:e.id,time:s.formattedTime(),frame:d});
-                    })
-                }else{
-                    s.group[e.ke].mon[e.id].spawn.stdio[3].on('data',function(d){
-                        if(!s.group[e.ke].mon[e.id].detectorFrameSaveBuffer){
-                            s.group[e.ke].mon[e.id].detectorFrameSaveBuffer=[d]
-                        }else{
-                            s.group[e.ke].mon[e.id].detectorFrameSaveBuffer.push(d)
-                        }
-                        if(d[d.length-2] === 0xFF && d[d.length-1] === 0xD9){
-                            var buffer = Buffer.concat(s.group[e.ke].mon[e.id].detectorFrameSaveBuffer);
-                            var frameLocation = s.dir.streams + e.ke + '/' + e.id + '/' + s.gid(5) + '.jpg'
-                            if(s.ocv){
-                                fs.writeFile(frameLocation,buffer,function(err){
-                                    if(err){
-                                        s.debugLog(err)
-                                    }else{
-                                        s.ocvTx({f:'frameFromRam',mon:s.group[e.ke].mon_conf[e.id].details,ke:e.ke,id:e.id,time:s.formattedTime(),frameLocation:frameLocation})
-                                    }
-                                })
-                            }
-                            s.group[e.ke].mon[e.id].detectorFrameSaveBuffer = null;
-                        }
-                    })
-                }
+            }else if(s.isAtleatOneDetectorPluginConnected){
+                s.group[e.ke].mon[e.id].spawn.stdio[3].on('data',function(d){
+                    s.ocvTx({f:'frame',mon:s.group[e.ke].mon_conf[e.id].details,ke:e.ke,id:e.id,time:s.formattedTime(),frame:d});
+                })
             }
         }
         //frames to stream
@@ -862,7 +961,11 @@ module.exports = function(s,config,lang){
            break;
         }
         if(e.frameToStream){
-            s.group[e.ke].mon[e.id].spawn.stdout.on('data',e.frameToStream)
+            if(e.coProcessor === true && e.details.stream_type === ('b64'||'mjpeg')){
+
+            }else{
+                s.group[e.ke].mon[e.id].spawn.stdout.on('data',e.frameToStream)
+            }
         }
         if(e.details.stream_channels && e.details.stream_channels !== ''){
             var createStreamEmitter = function(channel,number){
@@ -908,6 +1011,13 @@ module.exports = function(s,config,lang){
         s.group[e.ke].mon[e.id].spawn.stderr.on('data',function(d){
             d=d.toString();
             switch(true){
+                case checkLog(d,'No space left on device'):
+                    s.checkUserPurgeLock(e.ke)
+                    s.purgeDiskForGroup(e)
+                break;
+                case checkLog(d,'error while decoding'):
+                    s.userLog(e,{type:lang['Error While Decoding'],msg:lang.ErrorWhileDecodingText});
+                break;
                 case checkLog(d,'[hls @'):
                 case checkLog(d,'Past duration'):
                 case checkLog(d,'Last message repeated'):
@@ -935,10 +1045,11 @@ module.exports = function(s,config,lang){
                 case checkLog(d,'mjpeg_decode_dc'):
                 case checkLog(d,'bad vlc'):
                 case checkLog(d,'error dc'):
+                case checkLog(d,'No route to host'):
                     s.launchMonitorProcesses(e)
                 break;
                 case /T[0-9][0-9]-[0-9][0-9]-[0-9][0-9]./.test(d):
-                    var filename = d.split('.')[0]+'.'+e.ext
+                    var filename = d.split('.')[0].split(' [')[0].trim()+'.'+e.ext
                     s.insertCompletedVideo(e,{
                         file : filename
                     },function(err){
@@ -996,7 +1107,7 @@ module.exports = function(s,config,lang){
                 e.detector_notrigger_timeout = parseFloat(e.details.detector_notrigger_timeout)*1000*60;
                 s.group[e.ke].mon[e.id].detector_notrigger_timeout_function = function(){
                     s.onDetectorNoTriggerTimeoutExtensions.forEach(function(extender){
-                        extender(r,e)
+                        extender(e)
                     })
                 }
                 clearInterval(s.group[e.ke].mon[e.id].detector_notrigger_timeout)
@@ -1009,7 +1120,11 @@ module.exports = function(s,config,lang){
                         if(s.group[e.ke].mon[e.id].isStarted === true){
                             fs.stat(e.sdir+'s.jpg',function(err,snap){
                                 var notStreaming = function(){
-                                    s.launchMonitorProcesses(e)
+                                    if(e.coProcessor === true){
+                                        s.coSpawnLauncher(e)
+                                    }else{
+                                        s.launchMonitorProcesses(e)
+                                    }
                                     s.userLog(e,{type:lang['Camera is not streaming'],msg:{msg:lang['Restarting Process']}})
                                     s.orphanedVideoCheck(e,2,null,true)
                                 }
@@ -1034,13 +1149,6 @@ module.exports = function(s,config,lang){
                 //check if ffmpeg is recording
                 s.group[e.ke].mon[e.id].fswatch = fs.watch(e.dir, {encoding : 'utf8'}, (event, filename) => {
                     switch(event){
-                        case'rename':
-                        try{
-                            s.group[e.ke].mon[e.id].open = filename.split('.')[0]
-                        }catch(err){
-                            s.debugLog('Failed to split filename : ',filename)
-                        }
-                        break;
                         case'change':
                             s.resetRecordingCheck(e)
                         break;
@@ -1094,11 +1202,22 @@ module.exports = function(s,config,lang){
                         ){
                             s.cameraFilterFfmpegLog(e)
                         }
+                        if(e.coProcessor === true){
+                            setTimeout(function(){
+                                s.coSpawnLauncher(e)
+                            },6000)
+                        }
+                        s.onMonitorStartExtensions.forEach(function(extender){
+                            extender(Object.assign(s.group[e.ke].mon_conf[e.id],{}),e)
+                        })
                       }else{
+                          s.onMonitorPingFailedExtensions.forEach(function(extender){
+                              extender(Object.assign(s.group[e.ke].mon_conf[e.id],{}),e)
+                          })
                           s.userLog(e,{type:lang["Ping Failed"],msg:lang.skipPingText1});
                           s.fatalCameraError(e,"Ping Failed");return;
-                    }
-                }
+                      }
+                  }
                 if(
                     e.type !== 'socket' &&
                     e.type !== 'dashcam' &&
@@ -1136,6 +1255,9 @@ module.exports = function(s,config,lang){
                     if(o.success === true){
                         startVideoProcessor()
                     }else{
+                        s.onMonitorPingFailedExtensions.forEach(function(extender){
+                            extender(Object.assign(s.group[e.ke].mon_conf[e.id],{}),e)
+                        })
                         s.userLog(e,{type:lang["Ping Failed"],msg:lang.skipPingText1});
                         s.fatalCameraError(e,"Ping Failed");return;
                     }
@@ -1200,7 +1322,10 @@ module.exports = function(s,config,lang){
         }else{
             s.cameraDestroy(s.group[e.ke].mon[e.id].spawn,e)
         }
-        s.sendMonitorStatus({id:e.id,ke:e.ke,status:lang.Died});
+        s.sendMonitorStatus({id:e.id,ke:e.ke,status:lang.Died})
+        s.onMonitorDiedExtensions.forEach(function(extender){
+            extender(Object.assign(s.group[e.ke].mon_conf[e.id],{}),e)
+        })
     }
     s.isWatchCountable = function(d){
         try{
@@ -1307,6 +1432,9 @@ module.exports = function(s,config,lang){
             }
             s.tx(txData,'GRP_'+form.ke)
             callback(!endData.ok,endData)
+            s.onMonitorSaveExtensions.forEach(function(extender){
+                extender(Object.assign(s.group[form.ke].mon_conf[form.mid],{}),form,endData)
+            })
         })
     }
     s.camera = function(x,e,cn){
@@ -1399,6 +1527,9 @@ module.exports = function(s,config,lang){
                     var wantedStatus = lang.Idle
                 }
                 s.sendMonitorStatus({id:e.id,ke:e.ke,status:wantedStatus})
+                s.onMonitorStopExtensions.forEach(function(extender){
+                    extender(Object.assign(s.group[e.ke].mon_conf[e.id],{}),e)
+                })
             break;
             case'start':case'record'://watch or record monitor url
                 s.initiateMonitorObject({ke:e.ke,mid:e.id})
@@ -1420,8 +1551,8 @@ module.exports = function(s,config,lang){
                     s.group[e.ke].mon[e.mid].isRecording = false
                 }
                 //set up fatal error handler
-                if(e.details.fatal_max===''){
-                    e.details.fatal_max = 10
+                if(e.details.fatal_max === ''){
+                    e.details.fatal_max = 0
                 }else{
                     e.details.fatal_max = parseFloat(e.details.fatal_max)
                 }
@@ -1439,4 +1570,78 @@ module.exports = function(s,config,lang){
         }
         if(typeof cn === 'function'){setTimeout(function(){cn()},1000)}
     }
+    //
+    s.activateMonitorStates = function(groupKey,stateName,user,callback){
+        var endData = {
+            ok: false
+        }
+        s.findPreset([groupKey,'monitorStates',stateName],function(notFound,preset){
+            if(notFound === false){
+                var sqlQuery = 'SELECT * FROM Monitors WHERE ke=? AND '
+                var monitorQuery = []
+                var sqlQueryValues = [groupKey]
+                var monitorPresets = {}
+                preset.details.monitors.forEach(function(monitor){
+                    monitorQuery.push('mid=?')
+                    sqlQueryValues.push(monitor.mid)
+                    monitorPresets[monitor.mid] = monitor
+                })
+                sqlQuery += '('+monitorQuery.join(' OR ')+')'
+                s.sqlQuery(sqlQuery,sqlQueryValues,function(err,monitors){
+                    if(monitors && monitors[0]){
+                        monitors.forEach(function(monitor){
+                            s.checkDetails(monitor)
+                            s.checkDetails(monitorPresets[monitor.mid])
+                            var monitorPreset = monitorPresets[monitor.mid]
+                            monitorPreset.details = Object.assign(monitor.details,monitorPreset.details)
+                            monitor = s.cleanMonitorObjectForDatabase(Object.assign(monitor,monitorPreset))
+                            monitor.details = JSON.stringify(monitor.details)
+                            s.addOrEditMonitor(Object.assign(monitor,{}),function(err,endData){
+
+                            },user)
+                        })
+                        endData.ok = true
+                        s.tx({f:'change_group_state',ke:groupKey,name:stateName},'GRP_'+groupKey)
+                        callback(endData)
+                    }else{
+                        endData.msg = user.lang['State Configuration has no monitors associated']
+                        callback(endData)
+                    }
+                })
+            }else{
+                endData.msg = user.lang['State Configuration Not Found']
+                callback(endData)
+            }
+        })
+    }
+    s.getCamerasForMultiTrigger = function(monitor){
+        var list={}
+        var cameras=[]
+        var group
+        try{
+            group=JSON.parse(monitor.details.group_detector_multi)
+            if(!group){group=[]}
+        }catch(err){
+            group=[]
+        }
+        group.forEach(function(b){
+            Object.keys(s.group[monitor.ke].mon_conf).forEach(function(v){
+                try{
+                    var groups = JSON.parse(s.group[monitor.ke].mon_conf[v].details.groups)
+                    if(!groups){
+                        groups=[]
+                    }
+                }catch(err){
+                    groups=[]
+                }
+                if(!list[v]&&groups.indexOf(b)>-1){
+                    list[v]={}
+                    if(s.group[monitor.ke].mon_conf[v].mode !== 'stop'){
+                        cameras.push(Object.assign({},s.group[monitor.ke].mon_conf[v]))
+                    }
+                }
+            })
+        })
+        return cameras
+    }
 }

libs/notification.js

@@ -1,3 +1,4 @@
+var fs = require("fs")
 var Discord = require("discord.js")
 module.exports = function(s,config,lang){
     //discord bot
@@ -21,16 +22,27 @@ module.exports = function(s,config,lang){
                       text: "Shinobi Systems"
                     }
                 },data)
-                bot.channels.get(s.group[groupKey].init.discordbot_channel).send({
-                    embed: sendBody,
-                    files: files
-                }).catch(err => {
-                    if(err){
-                        s.userLog({ke:groupKey,mid:'$USER'},{type:lang.DiscordErrorText,msg:err})
-                        s.group[groupKey].discordBot = null
-                        s.loadGroupApps({ke:groupKey})
-                    }
-                })
+                var discordChannel = bot.channels.get(s.group[groupKey].init.discordbot_channel)
+                if(discordChannel && discordChannel.send){
+                    discordChannel.send({
+                        embed: sendBody,
+                        files: files
+                    }).catch(err => {
+                        if(err){
+                            s.userLog({ke:groupKey,mid:'$USER'},{type:lang.DiscordErrorText,msg:err})
+                            s.group[groupKey].discordBot = null
+                            s.loadGroupApps({ke:groupKey})
+                        }
+                    })
+                }else{
+                    s.userLog({
+                        ke: groupKey,
+                        mid: '$USER'
+                    },{
+                        type: lang.DiscordErrorText,
+                        msg: 'Check the Channel ID'
+                    })
+                }
             }
             var onEventTriggerBeforeFilterForDiscord = function(d,filter){
                 filter.discord = true
@@ -131,7 +143,13 @@ module.exports = function(s,config,lang){
                   ){
                     s.group[user.ke].discordBot = new Discord.Client()
                     s.group[user.ke].discordBot.on('ready', () => {
-                        console.log(`${user.mail} : Discord Bot Logged in as ${s.group[user.ke].discordBot.user.tag}!`)
+                        s.userLog({
+                            ke: user.ke,
+                            mid: '$USER'
+                        },{
+                            type: lang.DiscordLoggedIn,
+                            msg: s.group[user.ke].discordBot.user.tag
+                        })
                     })
                     s.group[user.ke].discordBot.login(ar.discordbot_token)
                 }

libs/plugins.js

@@ -17,28 +17,76 @@ module.exports = function(s,config,lang){
             break;
         }
     }
-    //multi plugin connections
-    s.connectedPlugins={}
+    s.connectedPlugins = {}
+    s.connectedDetectorPlugins = {}
+    s.detectorPluginArray = []
+    s.isAtleatOneDetectorPluginConnected = false
+    s.addDetectorPlugin = function(name,d){
+        s.connectedDetectorPlugins[d.plug] = {
+            started: s.timeObject(),
+            id: d.id,
+            plug: d.plug,
+            notice: d.notice,
+            connectionType: d.connectionType
+        }
+        s.resetDetectorPluginArray()
+    }
+    s.removeDetectorPlugin = function(name){
+        delete(s.connectedDetectorPlugins[name])
+        s.resetDetectorPluginArray(name)
+    }
+    s.resetDetectorPluginArray = function(){
+        pluginArray = []
+        Object.keys(s.connectedPlugins).forEach(function(name){
+            var plugin = s.connectedPlugins[name]
+            if(plugin.plugged === true && plugin.type === 'detector'){
+                pluginArray.push(name)
+            }
+        })
+        if(pluginArray.length > 0)s.isAtleatOneDetectorPluginConnected = true
+        s.detectorPluginArray = pluginArray
+    }
+    s.sendToAllDetectors = function(data){
+        s.detectorPluginArray.forEach(function(name){
+            s.connectedPlugins[name].tx(data)
+        })
+    }
+    s.sendDetectorInfoToClient = function(data,txFunction){
+        s.detectorPluginArray.forEach(function(name){
+            var detectorData = Object.assign(data,{
+                notice: s.connectedDetectorPlugins[name].notice,
+                plug: name
+            })
+            txFunction(detectorData)
+        })
+    }
+    // s.sendToDetectorsInChain = function(){
+    //
+    // }
     s.pluginInitiatorSuccess=function(mode,d,cn){
         s.systemLog('pluginInitiatorSuccess',d)
-        if(mode==='client'){
-            //is in client mode (camera.js is client)
-            cn.pluginEngine=d.plug
-            if(!s.connectedPlugins[d.plug]){
-                s.connectedPlugins[d.plug]={plug:d.plug}
+        if(!s.connectedPlugins[d.plug]){
+            s.connectedPlugins[d.plug]={
+                plug: d.plug,
+                type: d.type
             }
+        }
+        s.connectedPlugins[d.plug].plugged = true
+        if(mode==='client'){
+            s.connectedPlugins[d.plug].tx = function(x){return cn.emit('f',x)}
+            //is in client mode (camera.js is client)
+            cn.pluginEngine = d.plug
             s.systemLog('Connected to plugin : Detector - '+d.plug+' - '+d.type)
             switch(d.type){
                 default:case'detector':
-                    s.ocv = {
-                        started: s.timeObject(),
+                    cn.detectorPlugin = d.plug
+                    s.addDetectorPlugin(d.plug,{
                         id: cn.id,
                         plug: d.plug,
                         notice: d.notice,
                         isClientPlugin: true,
                         connectionType: d.connectionType
-                    };
-                    cn.ocv = 1;
+                    })
                     s.tx({f:'detector_plugged',plug:d.plug,notice:d.notice},'CPU')
                 break;
             }
@@ -46,21 +94,18 @@ module.exports = function(s,config,lang){
             //is in host mode (camera.js is client)
             switch(d.type){
                 default:case'detector':
-                    s.ocv = {
-                        started:s.timeObject(),
+                    s.addDetectorPlugin(d.plug,{
                         id:"host",
                         plug:d.plug,
                         notice:d.notice,
                         isHostPlugin:true,
                         connectionType: d.connectionType
-                    };
+                    })
+                    s.tx({f:'detector_plugged',plug:d.plug,notice:d.notice},'CPU')
                 break;
             }
         }
-        s.connectedPlugins[d.plug].plugged=true
         s.tx({f:'readPlugins',ke:d.ke},'CPU')
-        s.ocvTx({f:'api_key',key:d.plug})
-        s.api[d.plug]={pluginEngine:d.plug,permissions:{},details:{},ip:'0.0.0.0'};
     }
     s.pluginInitiatorFail=function(mode,d,cn){
         if(s.connectedPlugins[d.plug])s.connectedPlugins[d.plug].plugged=false
@@ -73,7 +118,10 @@ module.exports = function(s,config,lang){
     }
     if(config.plugins&&config.plugins.length>0){
         config.plugins.forEach(function(v){
-            s.connectedPlugins[v.id]={plug:v.id}
+            s.connectedPlugins[v.id]={
+                plug: v.id,
+                type: v.type
+            }
             if(v.enabled===false){return}
             if(v.mode==='host'){
                 //is in host mode (camera.js is client)
@@ -102,7 +150,13 @@ module.exports = function(s,config,lang){
                 socket.on('ocv',s.pluginEventController);
                 socket.on('disconnect', function(){
                     s.connectedPlugins[v.id].plugged=false
-                    delete(s.api[v.id])
+                    if(v.type === 'detector'){
+                        s.tx({f:'detector_unplugged',plug:v.id},'CPU')
+                        s.removeDetectorPlugin(v.id)
+                        s.sendDetectorInfoToClient({f:'detector_plugged'},function(data){
+                            s.tx(data,'CPU')
+                        })
+                    }
                     s.systemLog('Plugin Disconnected : '+v.id)
                     s.connectedPlugins[v.id].reconnector = setInterval(function(){
                         if(socket.connected===true){

libs/process.js

@@ -7,6 +7,9 @@ module.exports = function(process,__dirname){
     });
     // [CTRL] + [C] = exit
     process.on('SIGINT', function() {
+        s.onProcessExitExtensions.forEach(function(extender){
+            extender()
+        })
         console.log('Shinobi is Exiting...')
         process.exit();
     });

libs/rtmpserver.js

@@ -0,0 +1,23 @@
+module.exports = function(s,config,lang){
+    if(config.rtmpServer){
+        var defaultRtmpServerConfig = {
+            port: 1935,
+            chunk_size: 60000,
+            gop_cache: true,
+            ping: 60,
+            ping_timeout: 30
+        }
+        var runningRtmpServerConfig
+        if(config.rtmpServer instanceof Object === 'false'){
+            runningRtmpServerConfig = defaultRtmpServerConfig
+        }else{
+            runningRtmpServerConfig = Object.assign(defaultRtmpServerConfig,config.rtmpServer)
+        }
+        s.systemLog(`RTMP Server Running on port ${runningRtmpServerConfig.port}...`)
+        var NodeRtmpServer = require('./rtmpserver/node_rtmp_server')
+        var nmcs = new NodeRtmpServer({
+            rtmp: runningRtmpServerConfig
+        })
+        nmcs.run()
+    }
+}

libs/rtmpserver/node_core_av.js

@@ -0,0 +1,501 @@
+//
+//  Created by Mingliang Chen on 17/12/21.
+//  illuspas[a]gmail.com
+//  Copyright (c) 2018 Nodemedia. All rights reserved.
+//
+
+const Bitop = require('./node_core_bitop');
+const AAC_SAMPLE_RATE = [
+  96000, 88200, 64000, 48000,
+  44100, 32000, 24000, 22050,
+  16000, 12000, 11025, 8000,
+  7350, 0, 0, 0
+];
+
+const AAC_CHANNELS = [
+  0, 1, 2, 3, 4, 5, 6, 8
+];
+
+const AUDIO_CODEC_NAME = [
+  '',
+  'ADPCM',
+  'MP3',
+  'LinearLE',
+  'Nellymoser16',
+  'Nellymoser8',
+  'Nellymoser',
+  'G711A',
+  'G711U',
+  '',
+  'AAC',
+  'Speex',
+  '',
+  '',
+  'MP3-8K',
+  'DeviceSpecific',
+  'Uncompressed'
+];
+
+const AUDIO_SOUND_RATE = [
+  5512, 11025, 22050, 44100
+];
+
+const VIDEO_CODEC_NAME = [
+  '',
+  'Jpeg',
+  'Sorenson-H263',
+  'ScreenVideo',
+  'On2-VP6',
+  'On2-VP6-Alpha',
+  'ScreenVideo2',
+  'H264',
+  '',
+  '',
+  '',
+  '',
+  'H265'
+];
+
+function getObjectType(bitop) {
+  let audioObjectType = bitop.read(5);
+  if (audioObjectType === 31) {
+    audioObjectType = bitop.read(6) + 32;
+  }
+  return audioObjectType;
+}
+
+function getSampleRate(bitop, info) {
+  info.sampling_index = bitop.read(4);
+  return info.sampling_index == 0x0f ? bitop.read(24) : AAC_SAMPLE_RATE[info.sampling_index];
+}
+
+function readAACSpecificConfig(aacSequenceHeader) {
+  let info = {};
+  let bitop = new Bitop(aacSequenceHeader);
+  bitop.read(16);
+  info.object_type = getObjectType(bitop);
+  info.sample_rate = getSampleRate(bitop, info);
+  info.chan_config = bitop.read(4);
+  if (info.chan_config < AAC_CHANNELS.length) {
+    info.channels = AAC_CHANNELS[info.chan_config];
+  }
+  info.sbr = -1;
+  info.ps = -1;
+  if (info.object_type == 5 || info.object_type == 29) {
+    if (info.object_type == 29) {
+      info.ps = 1;
+    }
+    info.ext_object_type = 5;
+    info.sbr = 1;
+    info.sample_rate = getSampleRate(bitop, info);
+    info.object_type = getObjectType(bitop);
+  }
+
+  return info;
+}
+
+function getAACProfileName(info) {
+  switch (info.object_type) {
+    case 1:
+      return 'Main';
+    case 2:
+      if (info.ps > 0) {
+        return 'HEv2';
+      }
+      if (info.sbr > 0) {
+        return 'HE';
+      }
+      return 'LC';
+    case 3:
+      return 'SSR';
+    case 4:
+      return 'LTP';
+    case 5:
+      return 'SBR';
+    default:
+      return '';
+  }
+}
+
+function readH264SpecificConfig(avcSequenceHeader) {
+  let info = {};
+  let profile_idc, width, height, crop_left, crop_right,
+    crop_top, crop_bottom, frame_mbs_only, n, cf_idc,
+    num_ref_frames;
+  let bitop = new Bitop(avcSequenceHeader);
+  bitop.read(48);
+  info.width = 0;
+  info.height = 0;
+
+  do {
+    info.profile = bitop.read(8);
+    info.compat = bitop.read(8);
+    info.level = bitop.read(8);
+    info.nalu = (bitop.read(8) & 0x03) + 1;
+    info.nb_sps = bitop.read(8) & 0x1F;
+    if (info.nb_sps == 0) {
+      break;
+    }
+    /* nal size */
+    bitop.read(16);
+
+    /* nal type */
+    if (bitop.read(8) != 0x67) {
+      break;
+    }
+    /* SPS */
+    profile_idc = bitop.read(8);
+
+    /* flags */
+    bitop.read(8);
+
+    /* level idc */
+    bitop.read(8);
+
+    /* SPS id */
+    bitop.read_golomb();
+
+    if (profile_idc == 100 || profile_idc == 110 ||
+      profile_idc == 122 || profile_idc == 244 || profile_idc == 44 ||
+      profile_idc == 83 || profile_idc == 86 || profile_idc == 118) {
+      /* chroma format idc */
+      cf_idc = bitop.read_golomb();
+
+      if (cf_idc == 3) {
+
+        /* separate color plane */
+        bitop.read(1);
+      }
+
+      /* bit depth luma - 8 */
+      bitop.read_golomb();
+
+      /* bit depth chroma - 8 */
+      bitop.read_golomb();
+
+      /* qpprime y zero transform bypass */
+      bitop.read(1);
+
+      /* seq scaling matrix present */
+      if (bitop.read(1)) {
+
+        for (n = 0; n < (cf_idc != 3 ? 8 : 12); n++) {
+
+          /* seq scaling list present */
+          if (bitop.read(1)) {
+
+            /* TODO: scaling_list()
+            if (n < 6) {
+            } else {
+            }
+            */
+          }
+        }
+      }
+    }
+
+    /* log2 max frame num */
+    bitop.read_golomb();
+
+    /* pic order cnt type */
+    switch (bitop.read_golomb()) {
+      case 0:
+
+        /* max pic order cnt */
+        bitop.read_golomb();
+        break;
+
+      case 1:
+
+        /* delta pic order alwys zero */
+        bitop.read(1);
+
+        /* offset for non-ref pic */
+        bitop.read_golomb();
+
+        /* offset for top to bottom field */
+        bitop.read_golomb();
+
+        /* num ref frames in pic order */
+        num_ref_frames = bitop.read_golomb();
+
+        for (n = 0; n < num_ref_frames; n++) {
+
+          /* offset for ref frame */
+          bitop.read_golomb();
+        }
+    }
+
+    /* num ref frames */
+    info.avc_ref_frames = bitop.read_golomb();
+
+    /* gaps in frame num allowed */
+    bitop.read(1);
+
+    /* pic width in mbs - 1 */
+    width = bitop.read_golomb();
+
+    /* pic height in map units - 1 */
+    height = bitop.read_golomb();
+
+    /* frame mbs only flag */
+    frame_mbs_only = bitop.read(1);
+
+    if (!frame_mbs_only) {
+
+      /* mbs adaprive frame field */
+      bitop.read(1);
+    }
+
+    /* direct 8x8 inference flag */
+    bitop.read(1);
+
+    /* frame cropping */
+    if (bitop.read(1)) {
+
+      crop_left = bitop.read_golomb();
+      crop_right = bitop.read_golomb();
+      crop_top = bitop.read_golomb();
+      crop_bottom = bitop.read_golomb();
+
+    } else {
+      crop_left = 0;
+      crop_right = 0;
+      crop_top = 0;
+      crop_bottom = 0;
+    }
+    info.level = info.level / 10.0;
+    info.width = (width + 1) * 16 - (crop_left + crop_right) * 2;
+    info.height = (2 - frame_mbs_only) * (height + 1) * 16 - (crop_top + crop_bottom) * 2;
+
+  } while (0);
+
+  return info;
+}
+
+function HEVCParsePtl(bitop, hevc, max_sub_layers_minus1) {
+  let general_ptl = {};
+
+  general_ptl.profile_space = bitop.read(2);
+  general_ptl.tier_flag = bitop.read(1);
+  general_ptl.profile_idc = bitop.read(5);
+  general_ptl.profile_compatibility_flags = bitop.read(32);
+  general_ptl.general_progressive_source_flag = bitop.read(1);
+  general_ptl.general_interlaced_source_flag = bitop.read(1);
+  general_ptl.general_non_packed_constraint_flag = bitop.read(1);
+  general_ptl.general_frame_only_constraint_flag = bitop.read(1);
+  bitop.read(32);
+  bitop.read(12);
+  general_ptl.level_idc = bitop.read(8);
+
+  general_ptl.sub_layer_profile_present_flag = [];
+  general_ptl.sub_layer_level_present_flag = [];
+
+  for (let i = 0; i < max_sub_layers_minus1; i++) {
+    general_ptl.sub_layer_profile_present_flag[i] = bitop.read(1);
+    general_ptl.sub_layer_level_present_flag[i] = bitop.read(1);
+  }
+
+  if (max_sub_layers_minus1 > 0) {
+    for (let i = max_sub_layers_minus1; i < 8; i++) {
+      bitop.read(2)
+    }
+  }
+
+  general_ptl.sub_layer_profile_space = [];
+  general_ptl.sub_layer_tier_flag = [];
+  general_ptl.sub_layer_profile_idc = [];
+  general_ptl.sub_layer_profile_compatibility_flag = [];
+  general_ptl.sub_layer_progressive_source_flag = [];
+  general_ptl.sub_layer_interlaced_source_flag = [];
+  general_ptl.sub_layer_non_packed_constraint_flag = [];
+  general_ptl.sub_layer_frame_only_constraint_flag = [];
+  general_ptl.sub_layer_level_idc = [];
+
+  for (let i = 0; i < max_sub_layers_minus1; i++) {
+    if (general_ptl.sub_layer_profile_present_flag[i]) {
+      general_ptl.sub_layer_profile_space[i] = bitop.read(2);
+      general_ptl.sub_layer_tier_flag[i] = bitop.read(1);
+      general_ptl.sub_layer_profile_idc[i] = bitop.read(5);
+      general_ptl.sub_layer_profile_compatibility_flag[i] = bitop.read(32);
+      general_ptl.sub_layer_progressive_source_flag[i] = bitop.read(1);
+      general_ptl.sub_layer_interlaced_source_flag[i] = bitop.read(1);
+      general_ptl.sub_layer_non_packed_constraint_flag[i] = bitop.read(1);
+      general_ptl.sub_layer_frame_only_constraint_flag[i] = bitop.read(1);
+      bitop.read(32);
+      bitop.read(12);
+    }
+    if (general_ptl.sub_layer_level_present_flag[i]) {
+      general_ptl.sub_layer_level_idc[i] = bitop.read(8);
+    }
+    else {
+      general_ptl.sub_layer_level_idc[i] = 1;
+    }
+  }
+  return general_ptl;
+}
+
+function HEVCParseSPS(SPS, hevc) {
+  let psps = {};
+  let NumBytesInNALunit = SPS.length;
+  let NumBytesInRBSP = 0;
+  let rbsp_array = [];
+  let bitop = new Bitop(SPS);
+
+  bitop.read(1);//forbidden_zero_bit
+  bitop.read(6);//nal_unit_type
+  bitop.read(6);//nuh_reserved_zero_6bits
+  bitop.read(3);//nuh_temporal_id_plus1
+
+  for (let i = 2; i < NumBytesInNALunit; i++) {
+    if (i + 2 < NumBytesInNALunit && bitop.look(24) == 0x000003) {
+      rbsp_array.push(bitop.read(8));
+      rbsp_array.push(bitop.read(8));
+      i += 2;
+      let emulation_prevention_three_byte = bitop.read(8); /* equal to 0x03 */
+    } else {
+      rbsp_array.push(bitop.read(8));
+    }
+  }
+  let rbsp = Buffer.from(rbsp_array);
+  let rbspBitop = new Bitop(rbsp);
+  psps.sps_video_parameter_set_id = rbspBitop.read(4);
+  psps.sps_max_sub_layers_minus1 = rbspBitop.read(3);
+  psps.sps_temporal_id_nesting_flag = rbspBitop.read(1);
+  psps.profile_tier_level = HEVCParsePtl(rbspBitop, hevc, psps.sps_max_sub_layers_minus1);
+  psps.sps_seq_parameter_set_id = rbspBitop.read_golomb();
+  psps.chroma_format_idc = rbspBitop.read_golomb();
+  if (psps.chroma_format_idc == 3) {
+    psps.separate_colour_plane_flag = rbspBitop.read(1);
+  } else {
+    psps.separate_colour_plane_flag = 0;
+  }
+  psps.pic_width_in_luma_samples = rbspBitop.read_golomb();
+  psps.pic_height_in_luma_samples = rbspBitop.read_golomb();
+  psps.conformance_window_flag = rbspBitop.read(1);
+  if (psps.conformance_window_flag) {
+    let vert_mult = 1 + (psps.chroma_format_idc < 2);
+    let horiz_mult = 1 + (psps.chroma_format_idc < 3);
+    psps.conf_win_left_offset = rbspBitop.read_golomb() * horiz_mult;
+    psps.conf_win_right_offset = rbspBitop.read_golomb() * horiz_mult;
+    psps.conf_win_top_offset = rbspBitop.read_golomb() * vert_mult;
+    psps.conf_win_bottom_offset = rbspBitop.read_golomb() * vert_mult;
+  }
+  // Logger.debug(psps);
+  return psps;
+}
+
+function readHEVCSpecificConfig(hevcSequenceHeader) {
+  let info = {};
+  info.width = 0;
+  info.height = 0;
+  info.profile = 0;
+  info.level = 0;
+  // let bitop = new Bitop(hevcSequenceHeader);
+  // bitop.read(48);
+  hevcSequenceHeader = hevcSequenceHeader.slice(5);
+
+  do {
+    let hevc = {};
+    if (hevcSequenceHeader.length < 23) {
+      break;
+    }
+
+    hevc.configurationVersion = hevcSequenceHeader[0];
+    if (hevc.configurationVersion != 1) {
+      break;
+    }
+    hevc.general_profile_space = (hevcSequenceHeader[1] >> 6) & 0x03;
+    hevc.general_tier_flag = (hevcSequenceHeader[1] >> 5) & 0x01;
+    hevc.general_profile_idc = hevcSequenceHeader[1] & 0x1F;
+    hevc.general_profile_compatibility_flags = (hevcSequenceHeader[2] << 24) | (hevcSequenceHeader[3] << 16) | (hevcSequenceHeader[4] << 8) | hevcSequenceHeader[5];
+    hevc.general_constraint_indicator_flags = ((hevcSequenceHeader[6] << 24) | (hevcSequenceHeader[7] << 16) | (hevcSequenceHeader[8] << 8) | hevcSequenceHeader[9]);
+    hevc.general_constraint_indicator_flags = (hevc.general_constraint_indicator_flags << 16) | (hevcSequenceHeader[10] << 8) | hevcSequenceHeader[11];
+    hevc.general_level_idc = hevcSequenceHeader[12];
+    hevc.min_spatial_segmentation_idc = ((hevcSequenceHeader[13] & 0x0F) << 8) | hevcSequenceHeader[14];
+    hevc.parallelismType = hevcSequenceHeader[15] & 0x03;
+    hevc.chromaFormat = hevcSequenceHeader[16] & 0x03;
+    hevc.bitDepthLumaMinus8 = hevcSequenceHeader[17] & 0x07;
+    hevc.bitDepthChromaMinus8 = hevcSequenceHeader[18] & 0x07;
+    hevc.avgFrameRate = (hevcSequenceHeader[19] << 8) | hevcSequenceHeader[20];
+    hevc.constantFrameRate = (hevcSequenceHeader[21] >> 6) & 0x03;
+    hevc.numTemporalLayers = (hevcSequenceHeader[21] >> 3) & 0x07;
+    hevc.temporalIdNested = (hevcSequenceHeader[21] >> 2) & 0x01;
+    hevc.lengthSizeMinusOne = hevcSequenceHeader[21] & 0x03;
+    let numOfArrays = hevcSequenceHeader[22];
+    let p = hevcSequenceHeader.slice(23);
+    for (let i = 0; i < numOfArrays; i++) {
+      if (p.length < 3) {
+        brak;
+      }
+      let nalutype = p[0];
+      let n = (p[1]) << 8 | p[2];
+      // Logger.debug(nalutype, n);
+      p = p.slice(3);
+      for (let j = 0; j < n; j++) {
+        if (p.length < 2) {
+          break;
+        }
+        k = (p[0] << 8) | p[1];
+        // Logger.debug('k', k);
+        if (p.length < 2 + k) {
+          break;
+        }
+        p = p.slice(2);
+        if (nalutype == 33) {
+          //SPS
+          let sps = Buffer.alloc(k);
+          p.copy(sps, 0, 0, k);
+          // Logger.debug(sps, sps.length);
+          hevc.psps = HEVCParseSPS(sps, hevc);
+          info.profile = hevc.general_profile_idc;
+          info.level = hevc.general_level_idc / 30.0;
+          info.width = hevc.psps.pic_width_in_luma_samples - (hevc.psps.conf_win_left_offset + hevc.psps.conf_win_right_offset);
+          info.height = hevc.psps.pic_height_in_luma_samples - (hevc.psps.conf_win_top_offset + hevc.psps.conf_win_bottom_offset);
+        }
+        p = p.slice(k);
+      }
+    }
+  } while (0);
+
+  return info;
+}
+
+function readAVCSpecificConfig(avcSequenceHeader) {
+  let codec_id = avcSequenceHeader[0] & 0x0f;
+  if (codec_id == 7) {
+    return readH264SpecificConfig(avcSequenceHeader);
+  } else if (codec_id == 12) {
+    return readHEVCSpecificConfig(avcSequenceHeader);
+  }
+}
+
+
+function getAVCProfileName(info) {
+  switch (info.profile) {
+    case 1:
+      return 'Main';
+    case 2:
+      return 'Main 10';
+    case 3:
+      return 'Main Still Picture';
+    case 66:
+      return 'Baseline';
+    case 77:
+      return 'Main';
+    case 100:
+      return 'High';
+    default:
+      return '';
+  }
+}
+
+module.exports = {
+  AUDIO_SOUND_RATE,
+  AUDIO_CODEC_NAME,
+  VIDEO_CODEC_NAME,
+  readAACSpecificConfig,
+  getAACProfileName,
+  readAVCSpecificConfig,
+  getAVCProfileName,
+};

libs/rtmpserver/node_core_bitop.js

@@ -0,0 +1,54 @@
+
+class Bitop {
+    constructor(buffer) {
+      this.buffer = buffer;
+      this.buflen = buffer.length;
+      this.bufpos = 0;
+      this.bufoff = 0;
+      this.iserro = false;
+    }
+  
+    read(n) {
+      let v = 0;
+      let d = 0;
+      while (n) {
+        if (n < 0 || this.bufpos >= this.buflen) {
+          this.iserro = true;
+          return 0;
+        }
+  
+        this.iserro = false;
+        d = this.bufoff + n > 8 ? 8 - this.bufoff : n;
+  
+        v <<= d;
+        v += (this.buffer[this.bufpos] >> (8 - this.bufoff - d)) & (0xff >> (8 - d))
+  
+        this.bufoff += d;
+        n -= d;
+  
+        if (this.bufoff == 8) {
+          this.bufpos++;
+          this.bufoff = 0;
+        }
+      }
+      return v;
+    }
+  
+    look(n) {
+      let p = this.bufpos;
+      let o = this.bufoff;
+      let v = this.read(n);
+      this.bufpos = p;
+      this.bufoff = o;
+      return v;
+    }
+
+    read_golomb() {
+      let n;
+      for (n = 0; this.read(1) == 0 && !this.iserro; n++);
+      return (1 << n) + this.read(n) - 1;
+    }
+  }
+  
+  module.exports = Bitop; 
+  

libs/rtmpserver/node_core_ctx.js

@@ -0,0 +1,17 @@
+//
+//  Created by Mingliang Chen on 18/3/2.
+//  illuspas[a]gmail.com
+//  Copyright (c) 2018 Nodemedia. All rights reserved.
+//
+const EventEmitter = require('events');
+
+let sessions = new Map();
+let publishers = new Map();
+let idlePlayers = new Set();
+let nodeEvent = new EventEmitter();
+let stat = {
+  inbytes: 0,
+  outbytes: 0,
+  accepted: 0
+};
+module.exports = { sessions, publishers, idlePlayers, nodeEvent, stat };

libs/rtmpserver/node_core_logger.js

@@ -0,0 +1,53 @@
+const chalk = require('chalk');
+
+LOG_TYPES = {
+  NONE: 0,
+  ERROR: 1,
+  NORMAL: 2,
+  DEBUG: 3,
+  FFDEBUG: 4
+};
+
+let logType = LOG_TYPES.NORMAL;
+
+const setLogType = (type) => {
+  if (typeof type !== 'number') return;
+
+  logType = type;
+};
+
+const logTime = () => {
+  let nowDate = new Date();
+  return nowDate.toLocaleDateString() + ' ' + nowDate.toLocaleTimeString([], { hour12: false });
+};
+
+const log = (...args) => {
+  if (logType < LOG_TYPES.NORMAL) return;
+
+  console.log(logTime(), process.pid, chalk.bold.green('[INFO]'), ...args);
+};
+
+const error = (...args) => {
+  if (logType < LOG_TYPES.ERROR) return;
+
+  console.log(logTime(), process.pid, chalk.bold.red('[ERROR]'), ...args);
+};
+
+const debug = (...args) => {
+  if (logType < LOG_TYPES.DEBUG) return;
+
+  console.log(logTime(), process.pid, chalk.bold.blue('[DEBUG]'), ...args);
+};
+
+const ffdebug = (...args) => {
+  if (logType < LOG_TYPES.FFDEBUG) return;
+
+  console.log(logTime(), process.pid, chalk.bold.blue('[FFDEBUG]'), ...args);
+};
+
+module.exports = {
+  LOG_TYPES,
+  setLogType,
+
+  log, error, debug, ffdebug
+}

libs/rtmpserver/node_core_utils.js

@@ -0,0 +1,95 @@
+//
+//  Created by Mingliang Chen on 17/8/23.
+//  illuspas[a]gmail.com
+//  Copyright (c) 2018 Nodemedia. All rights reserved.
+//
+const Crypto = require('crypto');
+const EventEmitter = require('events');
+const { spawn } = require('child_process');
+const readline = require('readline');
+const context = require('./node_core_ctx');
+
+function generateNewSessionID() {
+  let sessionID = '';
+  const possible = 'ABCDEFGHIJKLMNOPQRSTUVWKYZ0123456789';
+  const numPossible = possible.length;
+  do {
+    for (let i = 0; i < 8; i++) {
+      sessionID += possible.charAt((Math.random() * numPossible) | 0);
+    }
+  } while (context.sessions.has(sessionID))
+  return sessionID;
+}
+
+function genRandomName() {
+  let name = '';
+  const possible = 'abcdefghijklmnopqrstuvwxyz0123456789';
+  const numPossible = possible.length;
+  for (let i = 0; i < 4; i++) {
+    name += possible.charAt((Math.random() * numPossible) | 0);
+  }
+
+  return name;
+}
+
+function verifyAuth(signStr, streamId, secretKey) {
+  if (signStr === undefined) {
+    return false;
+  }
+  let now = Date.now() / 1000 | 0;
+  let exp = parseInt(signStr.split('-')[0]);
+  let shv = signStr.split('-')[1];
+  let str = streamId + '-' + exp + '-' + secretKey;
+  if (exp < now) {
+    return false;
+  }
+  let md5 = Crypto.createHash('md5');
+  let ohv = md5.update(str).digest('hex');
+  return shv === ohv;
+}
+
+function getFFmpegVersion(ffpath) {
+  return new Promise((resolve, reject) => {
+    let ffmpeg_exec = spawn(ffpath, ['-version']);
+    let version = '';
+    ffmpeg_exec.on('error', (e) => {
+      reject(e);
+    });
+    ffmpeg_exec.stdout.on('data', (data) => {
+      try {
+        version = data.toString().split(/(?:\r\n|\r|\n)/g)[0].split('\ ')[2];
+      } catch (e) {
+      }
+    });
+    ffmpeg_exec.on('close', (code) => {
+      resolve(version);
+    });
+  });
+}
+
+function getFFmpegUrl() {
+  let url = '';
+  switch (process.platform) {
+    case 'darwin':
+      url = 'https://ffmpeg.zeranoe.com/builds/macos64/static/ffmpeg-latest-macos64-static.zip';
+      break;
+    case 'win32':
+      url = 'https://ffmpeg.zeranoe.com/builds/win64/static/ffmpeg-latest-win64-static.zip | https://ffmpeg.zeranoe.com/builds/win32/static/ffmpeg-latest-win32-static.zip';
+      break;
+    case 'linux':
+      url = 'https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-64bit-static.tar.xz | https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-32bit-static.tar.xz';
+      break;
+    default:
+      url = 'http://ffmpeg.org/download.html';
+      break;
+  }
+  return url;
+}
+
+module.exports = {
+  generateNewSessionID,
+  verifyAuth,
+  genRandomName,
+  getFFmpegVersion,
+  getFFmpegUrl
+}

libs/rtmpserver/node_rtmp_handshake.js

@@ -0,0 +1,113 @@
+//
+//  Created by Mingliang Chen on 17/8/1.
+//  illuspas[a]gmail.com
+//  Copyright (c) 2018 Nodemedia. All rights reserved.
+//
+// const Logger = require('./node_core_logger');
+
+const Crypto = require('crypto');
+
+const MESSAGE_FORMAT_0 = 0;
+const MESSAGE_FORMAT_1 = 1;
+const MESSAGE_FORMAT_2 = 2;
+
+const RTMP_SIG_SIZE = 1536;
+const SHA256DL = 32;
+
+const RandomCrud = Buffer.from([
+  0xf0, 0xee, 0xc2, 0x4a, 0x80, 0x68, 0xbe, 0xe8,
+  0x2e, 0x00, 0xd0, 0xd1, 0x02, 0x9e, 0x7e, 0x57,
+  0x6e, 0xec, 0x5d, 0x2d, 0x29, 0x80, 0x6f, 0xab,
+  0x93, 0xb8, 0xe6, 0x36, 0xcf, 0xeb, 0x31, 0xae
+])
+
+const GenuineFMSConst = 'Genuine Adobe Flash Media Server 001';
+const GenuineFMSConstCrud = Buffer.concat([Buffer.from(GenuineFMSConst, 'utf8'), RandomCrud]);
+
+const GenuineFPConst = 'Genuine Adobe Flash Player 001';
+const GenuineFPConstCrud = Buffer.concat([Buffer.from(GenuineFPConst, 'utf8'), RandomCrud]);
+
+function calcHmac(data, key) {
+  let hmac = Crypto.createHmac('sha256', key);
+  hmac.update(data);
+  return hmac.digest();
+}
+
+function GetClientGenuineConstDigestOffset(buf) {
+  let offset = buf[0] + buf[1] + buf[2] + buf[3];
+  offset = (offset % 728) + 12;
+  return offset;
+}
+
+function GetServerGenuineConstDigestOffset(buf) {
+  let offset = buf[0] + buf[1] + buf[2] + buf[3];
+  offset = (offset % 728) + 776;
+  return offset;
+}
+
+function detectClientMessageFormat(clientsig) {
+  let computedSignature, msg, providedSignature, sdl;
+  sdl = GetServerGenuineConstDigestOffset(clientsig.slice(772, 776));
+  msg = Buffer.concat([clientsig.slice(0, sdl), clientsig.slice(sdl + SHA256DL)], 1504);
+  computedSignature = calcHmac(msg, GenuineFPConst);
+  providedSignature = clientsig.slice(sdl, sdl + SHA256DL);
+  if (computedSignature.equals(providedSignature)) {
+    return MESSAGE_FORMAT_2;
+  }
+  sdl = GetClientGenuineConstDigestOffset(clientsig.slice(8, 12));
+  msg = Buffer.concat([clientsig.slice(0, sdl), clientsig.slice(sdl + SHA256DL)], 1504);
+  computedSignature = calcHmac(msg, GenuineFPConst);
+  providedSignature = clientsig.slice(sdl, sdl + SHA256DL);
+  if (computedSignature.equals(providedSignature)) {
+    return MESSAGE_FORMAT_1;
+  }
+  return MESSAGE_FORMAT_0;
+}
+
+function generateS1(messageFormat) {
+  let randomBytes = Crypto.randomBytes(RTMP_SIG_SIZE - 8);
+  let handshakeBytes = Buffer.concat([Buffer.from([0, 0, 0, 0, 1, 2, 3, 4]), randomBytes], RTMP_SIG_SIZE);
+
+  let serverDigestOffset
+  if (messageFormat === 1) {
+    serverDigestOffset = GetClientGenuineConstDigestOffset(handshakeBytes.slice(8, 12));
+  } else {
+    serverDigestOffset = GetServerGenuineConstDigestOffset(handshakeBytes.slice(772, 776));
+  }
+
+  msg = Buffer.concat([handshakeBytes.slice(0, serverDigestOffset), handshakeBytes.slice(serverDigestOffset + SHA256DL)], RTMP_SIG_SIZE - SHA256DL);
+  hash = calcHmac(msg, GenuineFMSConst);
+  hash.copy(handshakeBytes, serverDigestOffset, 0, 32);
+  return handshakeBytes;
+}
+
+function generateS2(messageFormat, clientsig, callback) {
+  let randomBytes = Crypto.randomBytes(RTMP_SIG_SIZE - 32);
+  let challengeKeyOffset;
+  if (messageFormat === 1) {
+    challengeKeyOffset = GetClientGenuineConstDigestOffset(clientsig.slice(8, 12));
+  } else {
+    challengeKeyOffset = GetServerGenuineConstDigestOffset(clientsig.slice(772, 776));
+  }
+  let challengeKey = clientsig.slice(challengeKeyOffset, challengeKeyOffset + 32);
+  let hash = calcHmac(challengeKey, GenuineFMSConstCrud);
+  let signature = calcHmac(randomBytes, hash);
+  let s2Bytes = Buffer.concat([randomBytes, signature], RTMP_SIG_SIZE);
+  return s2Bytes
+}
+
+function generateS0S1S2(clientsig) {
+  let clientType = Buffer.alloc(1, 3);
+  let messageFormat = detectClientMessageFormat(clientsig);
+  let allBytes;
+  if (messageFormat === MESSAGE_FORMAT_0) {
+    //    Logger.debug('[rtmp handshake] using simple handshake.');
+    allBytes = Buffer.concat([clientType, clientsig, clientsig]);
+  } else {
+    //    Logger.debug('[rtmp handshake] using complex handshake.');
+    allBytes = Buffer.concat([clientType, generateS1(messageFormat), generateS2(messageFormat, clientsig)]);
+  }
+  return allBytes;
+}
+
+module.exports = { generateS0S1S2 };

libs/rtmpserver/node_rtmp_server.js

@@ -0,0 +1,50 @@
+//
+//  Created by Mingliang Chen on 17/8/1.
+//  illuspas[a]gmail.com
+//  Copyright (c) 2018 Nodemedia. All rights reserved.
+//
+// const Logger = require('./node_core_logger');
+
+const Net = require('net');
+const NodeRtmpSession = require('./node_rtmp_session');
+const NodeCoreUtils = require('./node_core_utils');
+
+const context = require('./node_core_ctx');
+
+const RTMP_PORT = 1935;
+
+class NodeRtmpServer {
+  constructor(config) {
+    config.rtmp.port = this.port = config.rtmp.port ? config.rtmp.port : RTMP_PORT;
+    this.tcpServer = Net.createServer((socket) => {
+      let session = new NodeRtmpSession(config, socket);
+      session.run();
+    })
+  }
+
+  run() {
+    this.tcpServer.listen(this.port, () => {
+      // Logger.log(`Node Media Rtmp Server started on port: ${this.port}`);
+    });
+
+    this.tcpServer.on('error', (e) => {
+      // Logger.error(`Node Media Rtmp Server ${e}`);
+    });
+
+    this.tcpServer.on('close', () => {
+      // Logger.log('Node Media Rtmp Server Close.');
+    });
+  }
+
+  stop() {
+    this.tcpServer.close();
+    context.sessions.forEach((session, id) => {
+      if (session instanceof NodeRtmpSession) {
+        session.socket.destroy();
+        context.sessions.delete(id);
+      }
+    });
+  }
+}
+
+module.exports = NodeRtmpServer

libs/scheduler.js

@@ -0,0 +1,263 @@
+module.exports = function(s,config,lang,app,io){
+    s.schedules = {}
+    //Get all Schedules
+    s.getAllSchedules = function(callback){
+        s.schedules = {}
+        s.sqlQuery('SELECT * FROM Schedules',function(err,rows){
+            rows.forEach(function(schedule){
+                s.updateSchedule(schedule)
+            })
+            if(callback)callback()
+        })
+    }
+    //update schedule
+    s.updateSchedule = function(row){
+        var schedule = Object.assign(row,{})
+        if(!s.schedules[schedule.ke])s.schedules[schedule.ke] = {}
+        s.checkDetails(schedule)
+        if(!s.schedules[schedule.ke][schedule.name]){
+            s.schedules[schedule.ke][schedule.name] = schedule
+        }else{
+            s.schedules[schedule.ke][schedule.name] = Object.assign(s.schedules[schedule.ke][schedule.name],schedule)
+        }
+    }
+    //check time in schedule
+    s.checkTimeAgainstSchedule = function(start,end,callback){
+        try{
+            if(
+                start
+            ){
+                var checkStartTime = new Date()
+                var startSplit = start.split(':')
+                var startHour = parseInt(startSplit[0])
+                var startMin = parseInt(startSplit[1])
+                checkStartTime.setHours(startHour)
+                checkStartTime.setMinutes(startMin)
+                if(end){
+                    var checkEndTime = new Date()
+                    var endSplit = end.split(':')
+                    var endHour = parseInt(endSplit[0])
+                    var endMin = parseInt(endSplit[1])
+                    checkEndTime.setHours(endHour)
+                    checkEndTime.setMinutes(endMin)
+                }
+                var currentDate = new Date()
+                if(
+                    (
+                        currentDate >= checkStartTime &&
+                        currentDate <= checkEndTime
+                    ) ||
+                    currentDate >= checkStartTime && !end
+                ){
+                    callback()
+                }else{
+                    callback({
+                        currentDate : currentDate,
+                        startTime : checkStartTime,
+                        endTime : checkEndTime
+                    })
+                }
+            }else{
+                callback()
+            }
+        }catch(err){
+            console.log(err)
+            callback()
+        }
+    }
+    //check all Schedules
+    s.checkSchedules = function(v,callback){
+        var groupKeys = Object.keys(s.schedules)
+        groupKeys.forEach(function(key){
+            var scheduleNames = Object.keys(s.schedules[key])
+            scheduleNames.forEach(function(name){
+                var schedule = s.schedules[key][name]
+                if(!schedule.active && schedule.enabled === 1 && schedule.start && schedule.details.monitorStates){
+                    s.checkTimeAgainstSchedule(schedule.start,schedule.end,function(err){
+                        if(!err){
+                            schedule.active = true
+                            var monitorStates = schedule.details.monitorStates
+                            monitorStates.forEach(function(stateName){
+                                s.activateMonitorStates(key,stateName,{
+                                    ke: key,
+                                    uid: 'System',
+                                    details: {},
+                                    permissions: {},
+                                    lang: lang
+                                },function(endData){
+                                    // console.log(endData)
+                                })
+                            })
+                        }else{
+                            schedule.active = false
+                        }
+                    })
+                }
+            })
+        })
+    }
+    //
+    s.findSchedule = function(groupKey,name,callback){
+        //presetQueryVals = [ke, type, name]
+        s.sqlQuery("SELECT * FROM Schedules WHERE ke=? AND name=? LIMIT 1",[groupKey,name],function(err,schedules){
+            var schedule
+            var notFound = false
+            if(schedules && schedules[0]){
+                schedule = schedules[0]
+                s.checkDetails(schedule)
+            }else{
+                notFound = true
+            }
+            callback(notFound,schedule)
+        })
+    }
+    //
+    var onProcessReady = function(){
+        s.getAllSchedules(function(){
+            s.checkSchedules()
+        })
+        setInterval(function(){
+            s.checkSchedules()
+        },1000 * 60 * 5)
+    }
+    /**
+    * WebServerPath : API : Get Schedule
+    */
+    app.all([
+        config.webPaths.apiPrefix+':auth/schedule/:ke',
+        config.webPaths.adminApiPrefix+':auth/schedule/:ke',
+        config.webPaths.apiPrefix+':auth/schedule/:ke/:name',
+        config.webPaths.adminApiPrefix+':auth/schedule/:ke/:name',
+        config.webPaths.apiPrefix+':auth/schedules/:ke',
+        config.webPaths.adminApiPrefix+':auth/schedules/:ke',
+        config.webPaths.apiPrefix+':auth/schedules/:ke/:name',
+        config.webPaths.adminApiPrefix+':auth/schedules/:ke/:name',
+    ],function (req,res){
+        s.auth(req.params,function(user){
+            var endData = {
+                ok : false
+            }
+            if(user.details.sub){
+                endData.msg = user.lang['Not Permitted']
+                s.closeJsonResponse(res,endData)
+                return
+            }
+            var theQuery = "SELECT * FROM Schedules WHERE ke=?"
+            var theQueryValues = [req.params.ke]
+            if(req.params.name){
+                theQuery += ' AND name=?'
+                theQueryValues.push(req.params.name)
+            }
+            s.sqlQuery(theQuery,theQueryValues,function(err,schedules){
+                if(schedules && schedules[0]){
+                    endData.ok = true
+                    schedules.forEach(function(schedule){
+                        s.checkDetails(schedule)
+                    })
+                    endData.schedules = schedules
+                }else{
+                    endData.msg = user.lang['Not Found']
+                }
+                s.closeJsonResponse(res,endData)
+            })
+        })
+    })
+    /**
+    * WebServerPath : API : Update Schedule
+    */
+    app.all([
+        config.webPaths.apiPrefix+':auth/schedule/:ke/:name/:action',
+        config.webPaths.adminApiPrefix+':auth/schedule/:ke/:name/:action',
+        config.webPaths.apiPrefix+':auth/schedules/:ke/:name/:action',
+        config.webPaths.adminApiPrefix+':auth/schedules/:ke/:name/:action'
+    ],function (req,res){
+        s.auth(req.params,function(user){
+            var endData = {
+                ok : false
+            }
+            if(user.details.sub){
+                endData.msg = user.lang['Not Permitted']
+                s.closeJsonResponse(res,endData)
+                return
+            }
+            switch(req.params.action){
+                case'insert':case'edit':
+                    var form = s.getPostData(req)
+                    s.checkDetails(form)
+                    if(!form || !form.details){
+                        endData.msg = user.lang['Form Data Not Found']
+                        s.closeJsonResponse(res,endData)
+                        return
+                    }
+                    form.enabled = parseInt(form.enabled) || 1;
+                    s.findSchedule(req.params.ke,req.params.name,function(notFound,preset){
+                        if(notFound === true){
+                            endData.msg = lang["Inserted Schedule Configuration"]
+                            var insertData = {
+                                ke: req.params.ke,
+                                name: req.params.name,
+                                details: s.stringJSON(form.details),
+                                start: form.start,
+                                end: form.end,
+                                enabled: form.enabled
+                            }
+                            s.sqlQuery('INSERT INTO Schedules ('+Object.keys(insertData).join(',')+') VALUES (?,?,?,?,?,?)',Object.values(insertData))
+                            s.tx({
+                                f: 'add_schedule',
+                                insertData: insertData,
+                                ke: req.params.ke,
+                                name: req.params.name
+                            },'GRP_'+req.params.ke)
+                        }else{
+                            endData.msg = lang["Edited Schedule Configuration"]
+                            var insertData = {
+                                details: s.stringJSON(form.details),
+                                start: form.start,
+                                end: form.end,
+                                enabled: form.enabled,
+                                ke: req.params.ke,
+                                name: req.params.name
+                            }
+                            s.sqlQuery('UPDATE Schedules SET details=?,start=?,end=?,enabled=? WHERE ke=? AND name=?',Object.values(insertData))
+                            s.tx({
+                                f: 'edit_schedule',
+                                insertData: insertData,
+                                ke: req.params.ke,
+                                name: req.params.name
+                            },'GRP_'+req.params.ke)
+                        }
+                        s.updateSchedule({
+                            ke: req.params.ke,
+                            name: req.params.name,
+                            details: s.stringJSON(form.details),
+                            start: form.start,
+                            end: form.end,
+                            enabled: form.enabled
+                        })
+                        endData.ok = true
+                        s.closeJsonResponse(res,endData)
+                    })
+                break;
+                case'delete':
+                    s.findSchedule(req.params.ke,req.params.name,function(notFound,schedule){
+                        if(notFound === true){
+                            endData.msg = user.lang['Schedule Configuration Not Found']
+                            s.closeJsonResponse(res,endData)
+                        }else{
+                            s.sqlQuery('DELETE FROM Schedules WHERE ke=? AND name=?',[req.params.ke,req.params.name],function(err){
+                                if(!err){
+                                    endData.msg = lang["Deleted Schedule Configuration"]
+                                    endData.ok = true
+                                    if(s.schedules[schedule.ke])delete(s.schedules[schedule.ke][schedule.name])
+                                }
+                                s.closeJsonResponse(res,endData)
+                            })
+                        }
+                    })
+                break;
+            }
+        })
+    })
+    //bind events
+    s.onProcessReady(onProcessReady)
+}

libs/socketio.js

@@ -9,12 +9,8 @@ module.exports = function(s,config,lang,io){
     s.clientSocketConnection = {}
     //send data to detector plugin
     s.ocvTx=function(data){
-        if(!s.ocv){return}
-        if(s.ocv.isClientPlugin===true){
-            s.tx(data,s.ocv.id)
-        }else{
-            s.connectedPlugins[s.ocv.plug].tx(data)
-        }
+        // chaining coming in future update
+        s.sendToAllDetectors(data)
     }
     //send data to socket client function
     s.tx = function(z,y,x){if(x){return x.broadcast.to(y).emit('f',z)};io.to(y).emit('f',z);}
@@ -446,8 +442,8 @@ module.exports = function(s,config,lang,io){
                         s.group[d.ke].mon={}
                         if(!s.group[d.ke].mon){s.group[d.ke].mon={}}
                     }
-                    if(s.ocv){
-                        tx({f:'detector_plugged',plug:s.ocv.plug,notice:s.ocv.notice})
+                    if(s.isAtleatOneDetectorPluginConnected){
+                        s.sendDetectorInfoToClient({f:'detector_plugged'},tx)
                         s.ocvTx({f:'readPlugins',ke:d.ke})
                     }
                     tx({f:'users_online',users:s.group[d.ke].users})
@@ -477,6 +473,9 @@ module.exports = function(s,config,lang,io){
                             console.log(err)
                         }
                     })
+                    s.onSocketAuthenticationExtensions.forEach(function(extender){
+                        extender(r,cn)
+                    })
                 }
                 s.sqlQuery('SELECT ke,uid,auth,mail,details FROM Users WHERE ke=? AND auth=? AND uid=?',[d.ke,d.auth,d.uid],function(err,r) {
                     if(r&&r[0]){
@@ -512,12 +511,23 @@ module.exports = function(s,config,lang,io){
                         s.ocvTx(d.data)
                     break;
                     case'monitorOrder':
-                        if(d.monitorOrder&&d.monitorOrder instanceof Object){
+                        if(d.monitorOrder && d.monitorOrder instanceof Object){
                             s.sqlQuery('SELECT details FROM Users WHERE uid=? AND ke=?',[cn.uid,cn.ke],function(err,r){
-                                if(r&&r[0]){
-                                    r=JSON.parse(r[0].details);
-                                    r.monitorOrder=d.monitorOrder;
-                                    s.sqlQuery('UPDATE Users SET details=? WHERE uid=? AND ke=?',[JSON.stringify(r),cn.uid,cn.ke])
+                                if(r && r[0]){
+                                    details = JSON.parse(r[0].details)
+                                    details.monitorOrder = d.monitorOrder
+                                    s.sqlQuery('UPDATE Users SET details=? WHERE uid=? AND ke=?',[s.s(details),cn.uid,cn.ke])
+                                }
+                            })
+                        }
+                    break;
+                    case'monitorListOrder':
+                        if(d.monitorListOrder && d.monitorListOrder instanceof Object){
+                            s.sqlQuery('SELECT details FROM Users WHERE uid=? AND ke=?',[cn.uid,cn.ke],function(err,r){
+                                if(r && r[0]){
+                                    details = JSON.parse(r[0].details)
+                                    details.monitorListOrder = d.monitorListOrder
+                                    s.sqlQuery('UPDATE Users SET details=? WHERE uid=? AND ke=?',[s.s(details),cn.uid,cn.ke])
                                 }
                             })
                         }
@@ -1355,22 +1365,29 @@ module.exports = function(s,config,lang,io){
                 }
             }
             if(cn.pluginEngine){
-                s.connectedPlugins[cn.pluginEngine].plugged=false
+                s.connectedPlugins[cn.pluginEngine].plugged = false
                 s.tx({f:'plugin_engine_unplugged',plug:cn.pluginEngine},'CPU')
-                delete(s.api[cn.pluginEngine])
             }
             if(cn.cron){
                 delete(s.cron);
             }
-            if(cn.ocv){
-                s.tx({f:'detector_unplugged',plug:s.ocv.plug},'CPU')
-                delete(s.ocv);
-                delete(s.api[cn.id])
+            if(cn.detectorPlugin){
+                s.tx({f:'detector_unplugged',plug:cn.detectorPlugin},'CPU')
+                s.removeDetectorPlugin(cn.detectorPlugin)
+                s.sendDetectorInfoToClient({f:'detector_plugged'},function(data){
+                    s.tx(data,'CPU')
+                })
             }
             if(cn.superSessionKey){
                 delete(s.superUsersApi[cn.superSessionKey])
             }
+            s.onWebSocketDisconnectionExtensions.forEach(function(extender){
+                extender(cn)
+            })
             delete(s.clientSocketConnection[cn.id])
         })
+        s.onWebSocketConnectionExtensions.forEach(function(extender){
+            extender(cn)
+        })
     });
 }

libs/sql.js

@@ -1,12 +1,24 @@
+moment = require('moment')
 module.exports = function(s,config){
+    s.onBeforeDatabaseLoadExtensions.forEach(function(extender){
+        extender(config)
+    })
     //sql/database connection with knex
     s.databaseOptions = {
       client: config.databaseType,
       connection: config.db,
     }
+    var isSqlite = false
     if(s.databaseOptions.client.indexOf('sqlite')>-1){
+        isSqlite = true
         s.databaseOptions.client = 'sqlite3';
         s.databaseOptions.useNullAsDefault = true;
+        try{
+            require('sqlite3')
+        }catch(err){
+            console.log('Installing SQlite3 Module...')
+            require('child_process').execSync('npm install sqlite3 --unsafe-perm')
+        }
     }
     if(s.databaseOptions.client === 'sqlite3' && s.databaseOptions.connection.filename === undefined){
         s.databaseOptions.connection.filename = s.mainDirectory+"/shinobi.sqlite"
@@ -38,7 +50,7 @@ module.exports = function(s,config){
         return newQuery
     }
     s.stringToSqlTime = function(value){
-        newValue = new Date(value.replace('T',' '))
+        newValue = new Date(s.nameToTime(value)).valueOf()
         return newValue
     }
     s.sqlQuery = function(query,values,onMoveOn,hideLog){
@@ -48,6 +60,11 @@ module.exports = function(s,config){
             var values = [];
         }
         if(!onMoveOn){onMoveOn=function(){}}
+        // if(s.databaseOptions.client === 'pg'){
+        //     query = query
+        //         .replace(/ NOT LIKE /g," NOT ILIKE ")
+        //         .replace(/ LIKE /g," ILIKE ")
+        // }
         var mergedQuery = s.mergeQueryValues(query,values)
         s.debugLog('s.sqlQuery QUERY',mergedQuery)
         if(!s.databaseEngine || !s.databaseEngine.raw){
@@ -73,25 +90,54 @@ module.exports = function(s,config){
             }
         })
     }
+    s.openDatabaseTable = function(tableName){
+        return s.databaseEngine(tableName)
+    }
     s.connectDatabase = function(){
         s.databaseEngine = require('knex')(s.databaseOptions)
     }
     s.preQueries = function(){
-        //add Cloud Videos table, will remove in future
-        s.sqlQuery('CREATE TABLE IF NOT EXISTS `Cloud Videos` (`mid` varchar(50) NOT NULL,`ke` varchar(50) DEFAULT NULL,`href` text NOT NULL,`size` float DEFAULT NULL,`time` timestamp NULL DEFAULT NULL,`end` timestamp NULL DEFAULT NULL,`status` int(1) DEFAULT \'0\' COMMENT \'0:Complete,1:Read,2:Archive\',`details` text) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;',[],function(err){
-            // if(err)console.log(err)
+        var knex = s.databaseEngine
+        var mySQLtail = ''
+        if(config.databaseType === 'mysql'){
+            mySQLtail = ' ENGINE=InnoDB DEFAULT CHARSET=utf8'
+        }
+        //add Presets table and modernize
+        var createPresetsTableQuery = 'CREATE TABLE IF NOT EXISTS `Presets` (  `ke` varchar(50) DEFAULT NULL,  `name` text,  `details` text,  `type` varchar(50) DEFAULT NULL)'
+        s.sqlQuery( createPresetsTableQuery + mySQLtail + ';',[],function(err){
+            if(err)console.error(err)
+            if(config.databaseType === 'sqlite3'){
+                var aQuery = "ALTER TABLE Presets RENAME TO _Presets_old;"
+                    aQuery += createPresetsTableQuery
+                    aQuery += "INSERT INTO Presets (`ke`, `name`, `details`, `type`) SELECT `ke`, `name`, `details`, `type` FROM _Presets_old;COMMIT;DROP TABLE _Presets_old;"
+            }else{
+                s.sqlQuery('ALTER TABLE `Presets` CHANGE COLUMN `type` `type` VARCHAR(50) NULL DEFAULT NULL AFTER `details`;',[],function(err){
+                    if(err)console.error(err)
+                },true)
+            }
         },true)
-        //add monitorStates to Preset ENUM
-        s.sqlQuery('ALTER TABLE `Presets` CHANGE COLUMN `type` `type` VARCHAR(50) NULL DEFAULT NULL AFTER `details`;',[],function(err){
-            // if(err)console.log(err)
+        //add Schedules table, will remove in future
+        s.sqlQuery("CREATE TABLE IF NOT EXISTS `Schedules` (`ke` varchar(50) DEFAULT NULL,`name` text,`details` text,`start` varchar(10) DEFAULT NULL,`end` varchar(10) DEFAULT NULL,`enabled` int(1) NOT NULL DEFAULT '1')" + mySQLtail + ';',[],function(err){
+            if(err)console.error(err)
+        },true)
+        //add Cloud Videos table, will remove in future
+        s.sqlQuery('CREATE TABLE IF NOT EXISTS `Cloud Videos` (`mid` varchar(50) NOT NULL,`ke` varchar(50) DEFAULT NULL,`href` text NOT NULL,`size` float DEFAULT NULL,`time` timestamp NULL DEFAULT NULL,`end` timestamp NULL DEFAULT NULL,`status` int(1) DEFAULT \'0\',`details` text)' + mySQLtail + ';',[],function(err){
+            if(err)console.error(err)
         },true)
         //create Files table
-        s.sqlQuery('CREATE TABLE IF NOT EXISTS `Files` (`ke` varchar(50) NOT NULL,`mid` varchar(50) NOT NULL,`name` tinytext NOT NULL,`size` float NOT NULL DEFAULT \'0\',`details` text NOT NULL,`status` int(1) NOT NULL DEFAULT \'0\') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;',[],function(err){
-            // if(err)console.log(err)
+        var createFilesTableQuery = "CREATE TABLE IF NOT EXISTS `Files` (`ke` varchar(50) NOT NULL,`mid` varchar(50) NOT NULL,`name` tinytext NOT NULL,`size` float NOT NULL DEFAULT '0',`details` text NOT NULL,`status` int(1) NOT NULL DEFAULT '0',`time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP)"
+        s.sqlQuery(createFilesTableQuery + mySQLtail + ';',[],function(err){
+            if(err)console.error(err)
             //add time column to Files table
-            s.sqlQuery('ALTER TABLE `Files`	ADD COLUMN `time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP AFTER `status`;',[],function(err){
-                // if(err)console.log(err)
-            },true)
+            if(config.databaseType === 'sqlite3'){
+                var aQuery = "ALTER TABLE Files RENAME TO _Files_old;"
+                    aQuery += createPresetsTableQuery
+                    aQuery += "INSERT INTO Files (`ke`, `mid`, `name`, `details`, `size`, `status`, `time`) SELECT `ke`, `mid`, `name`, `details`, `size`, `status`, `time` FROM _Files_old;COMMIT;DROP TABLE _Files_old;"
+            }else{
+                s.sqlQuery('ALTER TABLE `Files`	ADD COLUMN `time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP AFTER `status`;',[],function(err){
+                    if(err && err.sqlMessage.indexOf('Duplicate') === -1)console.error(err)
+                },true)
+            }
         },true)
         delete(s.preQueries)
     }

libs/startup.js

@@ -9,10 +9,34 @@ module.exports = function(s,config,lang,io){
     console.log('Node.js version : '+execSync("node -v"))
     s.processReady = function(){
         s.systemLog(lang.startUpText5)
+        s.onProcessReadyExtensions.forEach(function(extender){
+            extender(true)
+        })
         process.send('ready')
     }
+    var checkForTerminalCommands = function(callback){
+        var next = function(){
+            if(callback)callback()
+        }
+        if(!s.isWin){
+            var etcPath = '/etc/shinobisystems/cctv.txt'
+            fs.stat(etcPath,function(err,stat){
+                if(err || !stat){
+                    exec('node '+ s.mainDirectory + '/INSTALL/terminalCommands.js',function(err){
+                        if(err)console.log(err)
+                    })
+                }
+                next()
+            })
+        }else{
+            next()
+        }
+    }
     var loadedAccounts = []
     var loadMonitors = function(callback){
+        s.beforeMonitorsLoadedOnStartupExtensions.forEach(function(extender){
+            extender()
+        })
         s.systemLog(lang.startUpText4)
         //preliminary monitor start
         s.sqlQuery('SELECT * FROM Monitors', function(err,monitors) {
@@ -147,6 +171,9 @@ module.exports = function(s,config,lang,io){
             })
         })
     },10000)
+    //hourly check to see if sizePurge has failed to unlock
+    //checks to see if request count is the number of monitors + 10
+    s.checkForStalePurgeLocks()
     //run prerequsite queries, load users and monitors
     if(config.childNodes.mode !== 'child'){
         //sql/database connection with knex
@@ -154,11 +181,13 @@ module.exports = function(s,config,lang,io){
         //run prerequsite queries
         s.preQueries()
         setTimeout(function(){
-            //load administrators (groups)
-            loadAdminUsers(function(){
-                //load monitors (for groups)
-                loadMonitors(function(){
-                    s.processReady()
+            checkForTerminalCommands(function(){
+                //load administrators (groups)
+                loadAdminUsers(function(){
+                    //load monitors (for groups)
+                    loadMonitors(function(){
+                        s.processReady()
+                    })
                 })
             })
         },1500)

libs/uploaders.js

@@ -0,0 +1,11 @@
+module.exports = function(s,config,lang){
+    var loadLib = function(lib){
+        return require('./uploaders/' + lib + '.js')
+    }
+    loadLib('loader')(s,config,lang)
+    loadLib('backblazeB2')(s,config,lang)
+    loadLib('amazonS3')(s,config,lang)
+    loadLib('webdav')(s,config,lang)
+    loadLib('wasabi')(s,config,lang)
+    loadLib('sftp')(s,config,lang)
+}

libs/uploaders/amazonS3.js

@@ -0,0 +1,137 @@
+var fs = require('fs');
+module.exports = function(s,config,lang){
+    //Amazon S3
+    var beforeAccountSaveForAmazonS3 = function(d){
+        //d = save event
+        d.form.details.aws_use_global=d.d.aws_use_global
+        d.form.details.use_aws_s3=d.d.use_aws_s3
+    }
+    var cloudDiskUseStartupForAmazonS3 = function(group,userDetails){
+        group.cloudDiskUse['s3'].name = 'Amazon S3'
+        group.cloudDiskUse['s3'].sizeLimitCheck = (userDetails.use_aws_s3_size_limit === '1')
+        if(!userDetails.aws_s3_size_limit || userDetails.aws_s3_size_limit === ''){
+            group.cloudDiskUse['s3'].sizeLimit = 10000
+        }else{
+            group.cloudDiskUse['s3'].sizeLimit = parseFloat(userDetails.aws_s3_size_limit)
+        }
+    }
+    var loadAmazonS3ForUser = function(e){
+        // e = user
+        var userDetails = JSON.parse(e.details)
+        if(userDetails.aws_use_global === '1' && config.cloudUploaders && config.cloudUploaders.AmazonS3){
+            // {
+            //     aws_accessKeyId: "",
+            //     aws_secretAccessKey: "",
+            //     aws_region: "",
+            //     aws_s3_bucket: "",
+            //     aws_s3_dir: "",
+            // }
+            userDetails = Object.assign(userDetails,config.cloudUploaders.AmazonS3)
+        }
+        //Amazon S3
+        if(!s.group[e.ke].aws &&
+           !s.group[e.ke].aws_s3 &&
+           userDetails.aws_s3 !== '0' &&
+           userDetails.aws_accessKeyId !== ''&&
+           userDetails.aws_secretAccessKey &&
+           userDetails.aws_secretAccessKey !== ''&&
+           userDetails.aws_region &&
+           userDetails.aws_region !== ''&&
+           userDetails.aws_s3_bucket !== ''
+          ){
+            if(!userDetails.aws_s3_dir || userDetails.aws_s3_dir === '/'){
+                userDetails.aws_s3_dir = ''
+            }
+            if(userDetails.aws_s3_dir !== ''){
+                userDetails.aws_s3_dir = s.checkCorrectPathEnding(userDetails.aws_s3_dir)
+            }
+            s.group[e.ke].aws = new require("aws-sdk")
+            s.group[e.ke].aws.config = new s.group[e.ke].aws.Config({
+                accessKeyId: userDetails.aws_accessKeyId,
+                secretAccessKey: userDetails.aws_secretAccessKey,
+                region: userDetails.aws_region
+            })
+            s.group[e.ke].aws_s3 = new s.group[e.ke].aws.S3();
+        }
+    }
+    var unloadAmazonS3ForUser = function(user){
+        s.group[user.ke].aws = null
+        s.group[user.ke].aws_s3 = null
+    }
+    var deleteVideoFromAmazonS3 = function(e,video,callback){
+        // e = user
+        try{
+            var videoDetails = JSON.parse(video.details)
+        }catch(err){
+            var videoDetails = video.details
+        }
+        if(!videoDetails.location){
+            videoDetails.location = video.href.split('.amazonaws.com')[1]
+        }
+        s.group[e.ke].aws_s3.deleteObject({
+            Bucket: s.group[e.ke].init.aws_s3_bucket,
+            Key: videoDetails.location,
+        }, function(err, data) {
+            if (err) console.log(err);
+            callback()
+        });
+    }
+    var uploadVideoToAmazonS3 = function(e,k){
+        //e = video object
+        //k = temporary values
+        if(!k)k={};
+        //cloud saver - amazon s3
+        if(s.group[e.ke].aws_s3 && s.group[e.ke].init.use_aws_s3 !== '0' && s.group[e.ke].init.aws_s3_save === '1'){
+            var ext = k.filename.split('.')
+            ext = ext[ext.length - 1]
+            var fileStream = fs.createReadStream(k.dir+k.filename);
+            fileStream.on('error', function (err) {
+                console.error(err)
+            })
+            var saveLocation = s.group[e.ke].init.aws_s3_dir+e.ke+'/'+e.mid+'/'+k.filename
+            s.group[e.ke].aws_s3.upload({
+                Bucket: s.group[e.ke].init.aws_s3_bucket,
+                Key: saveLocation,
+                Body:fileStream,
+                ACL:'public-read',
+                ContentType:'video/'+ext
+            },function(err,data){
+                if(err){
+                    s.userLog(e,{type:lang['Amazon S3 Upload Error'],msg:err})
+                }
+                if(s.group[e.ke].init.aws_s3_log === '1' && data && data.Location){
+                    var save = [
+                        e.mid,
+                        e.ke,
+                        k.startTime,
+                        1,
+                        s.s({
+                            type : 's3',
+                            location : saveLocation
+                        }),
+                        k.filesize,
+                        k.endTime,
+                        data.Location
+                    ]
+                    s.sqlQuery('INSERT INTO `Cloud Videos` (mid,ke,time,status,details,size,end,href) VALUES (?,?,?,?,?,?,?,?)',save)
+                    s.setCloudDiskUsedForGroup(e,{
+                        amount : k.filesizeMB,
+                        storageType : 's3'
+                    })
+                    s.purgeCloudDiskForGroup(e,'s3')
+                }
+            })
+        }
+    }
+    //amazon s3
+    s.addCloudUploader({
+        name: 's3',
+        loadGroupAppExtender: loadAmazonS3ForUser,
+        unloadGroupAppExtender: unloadAmazonS3ForUser,
+        insertCompletedVideoExtender: uploadVideoToAmazonS3,
+        deleteVideoFromCloudExtensions: deleteVideoFromAmazonS3,
+        cloudDiskUseStartupExtensions: cloudDiskUseStartupForAmazonS3,
+        beforeAccountSave: beforeAccountSaveForAmazonS3,
+        onAccountSave: cloudDiskUseStartupForAmazonS3,
+    })
+}

libs/uploaders/backblazeB2.js

@@ -0,0 +1,170 @@
+var fs = require('fs');
+module.exports = function(s,config,lang){
+    //Backblaze B2
+    var beforeAccountSaveForBackblazeB2 = function(d){
+        //d = save event
+        d.form.details.b2_use_global=d.d.b2_use_global
+        d.form.details.use_bb_b2=d.d.use_bb_b2
+    }
+    var cloudDiskUseStartupForBackblazeB2 = function(group,userDetails){
+        group.cloudDiskUse['b2'].name = 'Backblaze B2'
+        group.cloudDiskUse['b2'].sizeLimitCheck = (userDetails.use_bb_b2_size_limit === '1')
+        if(!userDetails.bb_b2_size_limit || userDetails.bb_b2_size_limit === ''){
+            group.cloudDiskUse['b2'].sizeLimit = 10000
+        }else{
+            group.cloudDiskUse['b2'].sizeLimit = parseFloat(userDetails.bb_b2_size_limit)
+        }
+    }
+    var loadBackblazeB2ForUser = function(e){
+        var userDetails = JSON.parse(e.details);
+        try{
+            if(userDetails.b2_use_global === '1' && config.cloudUploaders && config.cloudUploaders.BackblazeB2){
+                // {
+                //     bb_b2_accountId: "",
+                //     bb_b2_applicationKey: "",
+                //     bb_b2_bucket: "",
+                //     bb_b2_dir: "",
+                // }
+                userDetails = Object.assign(userDetails,config.cloudUploaders.BackblazeB2)
+            }
+            if(!s.group[e.ke].bb_b2 &&
+               userDetails.bb_b2_accountId &&
+               userDetails.bb_b2_accountId !=='' &&
+               userDetails.bb_b2_applicationKey &&
+               userDetails.bb_b2_applicationKey !=='' &&
+               userDetails.bb_b2_bucket &&
+               userDetails.bb_b2_bucket !== ''
+              ){
+                var B2 = require('backblaze-b2')
+                if(!userDetails.bb_b2_dir || userDetails.bb_b2_dir === '/'){
+                  userDetails.bb_b2_dir = ''
+                }
+                if(userDetails.bb_b2_dir !== ''){
+                  userDetails.bb_b2_dir = s.checkCorrectPathEnding(userDetails.bb_b2_dir)
+                }
+                var backblazeErr = function(err){
+                    // console.log(err)
+                    s.userLog({mid:'$USER',ke:e.ke},{type:lang['Backblaze Error'],msg:err.data || err})
+                }
+                var createB2Connection = function(){
+                    var b2 = new B2({
+                        accountId: userDetails.bb_b2_accountId,
+                        applicationKey: userDetails.bb_b2_applicationKey
+                    })
+                    b2.authorize().then(function(resp){
+                        s.group[e.ke].bb_b2_downloadUrl = resp.data.downloadUrl
+                        b2.listBuckets().then(function(resp){
+                            var buckets = resp.data.buckets
+                            var bucketN = -2
+                            buckets.forEach(function(item,n){
+                                if(item.bucketName === userDetails.bb_b2_bucket){
+                                    bucketN = n
+                                }
+                            })
+                            if(bucketN > -1){
+                                s.group[e.ke].bb_b2_bucketId = buckets[bucketN].bucketId
+                            }else{
+                                b2.createBucket(
+                                    userDetails.bb_b2_bucket,
+                                    'allPublic'
+                                ).then(function(resp){
+                                    s.group[e.ke].bb_b2_bucketId = resp.data.bucketId
+                                }).catch(backblazeErr)
+                            }
+                        }).catch(backblazeErr)
+                    }).catch(backblazeErr)
+                    s.group[e.ke].bb_b2 = b2
+                }
+                createB2Connection()
+                s.group[e.ke].bb_b2_refreshTimer = setInterval(createB2Connection,1000 * 60 * 60)
+            }
+        }catch(err){
+            s.debugLog(err)
+        }
+    }
+    var unloadBackblazeB2ForUser = function(user){
+        s.group[user.ke].bb_b2 = null
+        clearInterval(s.group[user.ke].bb_b2_refreshTimer)
+    }
+    var deleteVideoFromBackblazeB2 = function(e,video,callback){
+        // e = user
+        try{
+            var videoDetails = JSON.parse(video.details)
+        }catch(err){
+            var videoDetails = video.details
+        }
+        s.group[e.ke].bb_b2.deleteFileVersion({
+            fileId: videoDetails.fileId,
+            fileName: videoDetails.fileName
+        }).then(function(resp){
+            // console.log('deleteFileVersion',resp.data)
+        }).catch(function(err){
+            console.log('deleteFileVersion',err)
+        })
+    }
+    var uploadVideoToBackblazeB2 = function(e,k){
+        //e = video object
+        //k = temporary values
+        if(!k)k={};
+        //cloud saver - Backblaze B2
+        if(s.group[e.ke].bb_b2 && s.group[e.ke].init.use_bb_b2 !== '0' && s.group[e.ke].init.bb_b2_save === '1'){
+            var backblazeErr = function(err){
+                // console.log(err)
+                s.userLog({mid:'$USER',ke:e.ke},{type:lang['Backblaze Error'],msg:err.data})
+            }
+            fs.readFile(k.dir+k.filename,function(err,data){
+                var backblazeSavePath = s.group[e.ke].init.bb_b2_dir+e.ke+'/'+e.mid+'/'+k.filename
+                var getUploadUrl = function(bucketId,callback){
+                    s.group[e.ke].bb_b2.getUploadUrl(bucketId).then(function(resp){
+                        callback(resp.data)
+                    }).catch(backblazeErr)
+                }
+                getUploadUrl(s.group[e.ke].bb_b2_bucketId,function(req){
+                    s.group[e.ke].bb_b2.uploadFile({
+                        uploadUrl: req.uploadUrl,
+                        uploadAuthToken: req.authorizationToken,
+                        filename: backblazeSavePath,
+                        data: data,
+                        onUploadProgress: null
+                    }).then(function(resp){
+                        if(s.group[e.ke].init.bb_b2_log === '1' && resp.data.fileId){
+                            var backblazeDownloadUrl = s.group[e.ke].bb_b2_downloadUrl + '/file/' + s.group[e.ke].init.bb_b2_bucket + '/' + backblazeSavePath
+                            var save = [
+                                e.mid,
+                                e.ke,
+                                k.startTime,
+                                1,
+                                s.s({
+                                    type : 'b2',
+                                    bucketId : resp.data.bucketId,
+                                    fileId : resp.data.fileId,
+                                    fileName : resp.data.fileName
+                                }),
+                                k.filesize,
+                                k.endTime,
+                                backblazeDownloadUrl
+                            ]
+                            s.sqlQuery('INSERT INTO `Cloud Videos` (mid,ke,time,status,details,size,end,href) VALUES (?,?,?,?,?,?,?,?)',save)
+                            s.setCloudDiskUsedForGroup(e,{
+                                amount : k.filesizeMB,
+                                storageType : 'b2'
+                            })
+                            s.purgeCloudDiskForGroup(e,'b2')
+                        }
+                    }).catch(backblazeErr)
+                })
+            })
+        }
+    }
+    //backblaze b2
+    s.addCloudUploader({
+        name: 'b2',
+        loadGroupAppExtender: loadBackblazeB2ForUser,
+        unloadGroupAppExtender: unloadBackblazeB2ForUser,
+        insertCompletedVideoExtender: uploadVideoToBackblazeB2,
+        deleteVideoFromCloudExtensions: deleteVideoFromBackblazeB2,
+        cloudDiskUseStartupExtensions: cloudDiskUseStartupForBackblazeB2,
+        beforeAccountSave: beforeAccountSaveForBackblazeB2,
+        onAccountSave: cloudDiskUseStartupForBackblazeB2,
+    })
+}

libs/uploaders/loader.js

@@ -0,0 +1,20 @@
+module.exports = function(s){
+    s.addCloudUploader = function(opt){
+        s.loadGroupAppExtender(opt.loadGroupAppExtender)
+        s.unloadGroupAppExtender(opt.unloadGroupAppExtender)
+        s.insertCompletedVideoExtender(opt.insertCompletedVideoExtender)
+        s.deleteVideoFromCloudExtensions[opt.name] = opt.deleteVideoFromCloudExtensions
+        s.cloudDiskUseStartupExtensions[opt.name] = opt.cloudDiskUseStartupExtensions
+        s.beforeAccountSave(opt.beforeAccountSave)
+        s.onAccountSave(opt.onAccountSave)
+        s.cloudDisksLoader(opt.name)
+    }
+    s.addSimpleUploader = function(opt){
+        s.loadGroupAppExtender(opt.loadGroupAppExtender)
+        s.unloadGroupAppExtender(opt.unloadGroupAppExtender)
+        s.insertCompletedVideoExtender(opt.insertCompletedVideoExtender)
+        s.beforeAccountSave(opt.beforeAccountSave)
+        s.onAccountSave(opt.onAccountSave)
+        s.onMonitorSave(opt.onMonitorSave)
+    }
+}

libs/uploaders/sftp.js

@@ -0,0 +1,90 @@
+var fs = require('fs');
+var ssh2SftpClient = require('node-ssh')
+module.exports = function(s,config,lang){
+    //SFTP
+    var sftpErr = function(err){
+        // console.log(err)
+        s.userLog({mid:'$USER',ke:e.ke},{type:lang['SFTP Error'],msg:err.data || err})
+    }
+    var beforeAccountSaveForSftp = function(d){
+        //d = save event
+        d.form.details.use_sftp = d.d.use_sftp
+    }
+    var loadSftpForUser = function(e){
+        // e = user
+        var userDetails = JSON.parse(e.details);
+        //SFTP
+        if(!s.group[e.ke].sftp &&
+            !s.group[e.ke].sftp &&
+            userDetails.sftp !== '0' &&
+            userDetails.sftp_host &&
+            userDetails.sftp_host !== ''&&
+            userDetails.sftp_port &&
+            userDetails.sftp_port !== ''
+          ){
+            if(!userDetails.sftp_dir || userDetails.sftp_dir === '/'){
+                userDetails.sftp_dir = ''
+            }
+            if(userDetails.sftp_dir !== ''){
+                userDetails.sftp_dir = s.checkCorrectPathEnding(userDetails.sftp_dir)
+            }
+            var sftp = new ssh2SftpClient()
+            var connectionDetails = {
+                host: userDetails.sftp_host,
+                port: userDetails.sftp_port
+            }
+            if(!userDetails.sftp_port)connectionDetails.port = 22
+            if(userDetails.sftp_username && userDetails.sftp_username !== '')connectionDetails.username = userDetails.sftp_username
+            if(userDetails.sftp_password && userDetails.sftp_password !== '')connectionDetails.password = userDetails.sftp_password
+            if(userDetails.sftp_privateKey && userDetails.sftp_privateKey !== '')connectionDetails.privateKey = userDetails.sftp_privateKey
+            sftp.connect(connectionDetails).catch(sftpErr)
+            s.group[e.ke].sftp = sftp
+        }
+    }
+    var unloadSftpForUser = function(user){
+        if(s.group[user.ke].sftp && s.group[user.ke].sftp.end)s.group[user.ke].sftp.end().then(function(){
+            s.group[user.ke].sftp = null
+        })
+    }
+    var uploadVideoToSftp = function(e,k){
+        //e = video object
+        //k = temporary values
+        if(!k)k={};
+        //cloud saver - SFTP
+        if(s.group[e.ke].sftp && s.group[e.ke].init.use_sftp !== '0' && s.group[e.ke].init.sftp_save === '1'){
+            var localPath = k.dir + k.filename
+            var saveLocation = s.group[e.ke].init.sftp_dir + e.ke + '/' + e.mid + '/' + k.filename
+            s.group[e.ke].sftp.putFile(localPath, saveLocation).catch(sftpErr)
+        }
+    }
+    var createSftpDirectory = function(monitorConfig){
+        var monitorSaveDirectory = s.group[monitorConfig.ke].init.sftp_dir + monitorConfig.ke + '/' + monitorConfig.mid
+        s.group[monitorConfig.ke].sftp.mkdir(monitorSaveDirectory, true).catch(function(err){
+            if(err.code !== 'ERR_ASSERTION'){
+                sftpErr(err)
+            }
+        })
+    }
+    var onMonitorSaveForSftp = function(monitorConfig){
+        if(s.group[monitorConfig.ke].sftp && s.group[monitorConfig.ke].init.use_sftp !== '0' && s.group[monitorConfig.ke].init.sftp_save === '1'){
+            createSftpDirectory(monitorConfig)
+        }
+    }
+    var onAccountSaveForSftp = function(group,userDetails,user){
+        if(s.group[user.ke] && s.group[user.ke].sftp && s.group[user.ke].init.use_sftp !== '0' && s.group[user.ke].init.sftp_save === '1'){
+            Object.keys(s.group[user.ke].mon_conf).forEach(function(monitorId){
+                createSftpDirectory(s.group[user.ke].mon_conf[monitorId])
+            })
+        }
+    }
+    //SFTP (Simple Uploader)
+    s.addSimpleUploader({
+        name: 'sftp',
+        loadGroupAppExtender: loadSftpForUser,
+        unloadGroupAppExtender: unloadSftpForUser,
+        insertCompletedVideoExtender: uploadVideoToSftp,
+        beforeAccountSave: beforeAccountSaveForSftp,
+        onAccountSave: onAccountSaveForSftp,
+        onMonitorSave: onMonitorSaveForSftp,
+    })
+}

libs/uploaders/wasabi.js

@@ -0,0 +1,138 @@
+var fs = require('fs');
+module.exports = function(s,config,lang){
+    //Wasabi Hot Cloud Storage
+    var beforeAccountSaveForWasabiHotCloudStorage = function(d){
+        //d = save event
+        d.form.details.whcs_use_global=d.d.whcs_use_global
+        d.form.details.use_whcs=d.d.use_whcs
+    }
+    var cloudDiskUseStartupForWasabiHotCloudStorage = function(group,userDetails){
+        group.cloudDiskUse['whcs'].name = 'Wasabi Hot Cloud Storage'
+        group.cloudDiskUse['whcs'].sizeLimitCheck = (userDetails.use_whcs_size_limit === '1')
+        if(!userDetails.whcs_size_limit || userDetails.whcs_size_limit === ''){
+            group.cloudDiskUse['whcs'].sizeLimit = 10000
+        }else{
+            group.cloudDiskUse['whcs'].sizeLimit = parseFloat(userDetails.whcs_size_limit)
+        }
+    }
+    var loadWasabiHotCloudStorageForUser = function(e){
+        // e = user
+        var userDetails = JSON.parse(e.details)
+        if(userDetails.whcs_use_global === '1' && config.cloudUploaders && config.cloudUploaders.WasabiHotCloudStorage){
+            // {
+            //     whcs_accessKeyId: "",
+            //     whcs_secretAccessKey: "",
+            //     whcs_region: "",
+            //     whcs_bucket: "",
+            //     whcs_dir: "",
+            // }
+            userDetails = Object.assign(userDetails,config.cloudUploaders.WasabiHotCloudStorage)
+        }
+        //Wasabi Hot Cloud Storage
+        if(!s.group[e.ke].whcs &&
+           userDetails.whcs !== '0' &&
+           userDetails.whcs_accessKeyId !== ''&&
+           userDetails.whcs_secretAccessKey &&
+           userDetails.whcs_secretAccessKey !== ''&&
+           userDetails.whcs_region &&
+           userDetails.whcs_region !== ''&&
+           userDetails.whcs_bucket !== ''
+          ){
+            if(!userDetails.whcs_dir || userDetails.whcs_dir === '/'){
+                userDetails.whcs_dir = ''
+            }
+            if(userDetails.whcs_dir !== ''){
+                userDetails.whcs_dir = s.checkCorrectPathEnding(userDetails.whcs_dir)
+            }
+            var AWS = new require("aws-sdk")
+            s.group[e.ke].whcs = AWS
+            var wasabiEndpoint = new AWS.Endpoint('s3.wasabisys.com')
+            s.group[e.ke].whcs.config = new s.group[e.ke].whcs.Config({
+                endpoint: wasabiEndpoint,
+                accessKeyId: userDetails.whcs_accessKeyId,
+                secretAccessKey: userDetails.whcs_secretAccessKey,
+                region: userDetails.whcs_region
+            })
+            s.group[e.ke].whcs = new s.group[e.ke].whcs.S3();
+        }
+    }
+    var unloadWasabiHotCloudStorageForUser = function(user){
+        s.group[user.ke].whcs = null
+    }
+    var deleteVideoFromWasabiHotCloudStorage = function(e,video,callback){
+        // e = user
+        try{
+            var videoDetails = JSON.parse(video.details)
+        }catch(err){
+            var videoDetails = video.details
+        }
+        if(!videoDetails.location){
+            videoDetails.location = video.href.split('wasabisys.com')[1]
+        }
+        s.group[e.ke].whcs.deleteObject({
+            Bucket: s.group[e.ke].init.whcs_bucket,
+            Key: videoDetails.location,
+        }, function(err, data) {
+            if (err) console.log(err);
+            callback()
+        });
+    }
+    var uploadVideoToWasabiHotCloudStorage = function(e,k){
+        //e = video object
+        //k = temporary values
+        if(!k)k={};
+        //cloud saver - Wasabi Hot Cloud Storage
+        if(s.group[e.ke].whcs && s.group[e.ke].init.use_whcs !== '0' && s.group[e.ke].init.whcs_save === '1'){
+            var ext = k.filename.split('.')
+            ext = ext[ext.length - 1]
+            var fileStream = fs.createReadStream(k.dir+k.filename);
+            fileStream.on('error', function (err) {
+                console.error(err)
+            })
+            var saveLocation = s.group[e.ke].init.whcs_dir+e.ke+'/'+e.mid+'/'+k.filename
+            s.group[e.ke].whcs.upload({
+                Bucket: s.group[e.ke].init.whcs_bucket,
+                Key: saveLocation,
+                Body:fileStream,
+                ACL:'public-read',
+                ContentType:'video/'+ext
+            },function(err,data){
+                if(err){
+                    s.userLog(e,{type:lang['Wasabi Hot Cloud Storage Upload Error'],msg:err})
+                }
+                if(s.group[e.ke].init.whcs_log === '1' && data && data.Location){
+                    var save = [
+                        e.mid,
+                        e.ke,
+                        k.startTime,
+                        1,
+                        s.s({
+                            type : 'whcs',
+                            location : saveLocation
+                        }),
+                        k.filesize,
+                        k.endTime,
+                        data.Location
+                    ]
+                    s.sqlQuery('INSERT INTO `Cloud Videos` (mid,ke,time,status,details,size,end,href) VALUES (?,?,?,?,?,?,?,?)',save)
+                    s.setCloudDiskUsedForGroup(e,{
+                        amount : k.filesizeMB,
+                        storageType : 'whcs'
+                    })
+                    s.purgeCloudDiskForGroup(e,'whcs')
+                }
+            })
+        }
+    }
+    //wasabi
+    s.addCloudUploader({
+        name: 'whcs',
+        loadGroupAppExtender: loadWasabiHotCloudStorageForUser,
+        unloadGroupAppExtender: unloadWasabiHotCloudStorageForUser,
+        insertCompletedVideoExtender: uploadVideoToWasabiHotCloudStorage,
+        deleteVideoFromCloudExtensions: deleteVideoFromWasabiHotCloudStorage,
+        cloudDiskUseStartupExtensions: cloudDiskUseStartupForWasabiHotCloudStorage,
+        beforeAccountSave: beforeAccountSaveForWasabiHotCloudStorage,
+        onAccountSave: cloudDiskUseStartupForWasabiHotCloudStorage,
+    })
+}

libs/uploaders/webdav.js

@@ -0,0 +1,169 @@
+var fs = require('fs');
+var webdav = require("webdav-fs");
+module.exports = function(s,config,lang){
+    // WebDAV
+    var beforeAccountSaveForWebDav = function(d){
+        //d = save event
+        d.form.details.webdav_use_global=d.d.webdav_use_global
+        d.form.details.use_webdav=d.d.use_webdav
+    }
+    var cloudDiskUseStartupForWebDav = function(group,userDetails){
+        group.cloudDiskUse['webdav'].name = 'WebDAV'
+        group.cloudDiskUse['webdav'].sizeLimitCheck = (userDetails.use_webdav_size_limit === '1')
+        if(!userDetails.webdav_size_limit || userDetails.webdav_size_limit === ''){
+            group.cloudDiskUse['webdav'].sizeLimit = 10000
+        }else{
+            group.cloudDiskUse['webdav'].sizeLimit = parseFloat(userDetails.webdav_size_limit)
+        }
+    }
+    var loadWebDavForUser = function(e){
+        // e = user
+        var userDetails = JSON.parse(e.details);
+        if(userDetails.webdav_use_global === '1' && config.cloudUploaders && config.cloudUploaders.WebDAV){
+            // {
+            //     webdav_user: "",
+            //     webdav_pass: "",
+            //     webdav_url: "",
+            //     webdav_dir: "",
+            // }
+            userDetails = Object.assign(userDetails,config.cloudUploaders.WebDAV)
+        }
+        //owncloud/webdav
+        if(!s.group[e.ke].webdav &&
+           userDetails.webdav_user&&
+           userDetails.webdav_user!==''&&
+           userDetails.webdav_pass&&
+           userDetails.webdav_pass!==''&&
+           userDetails.webdav_url&&
+           userDetails.webdav_url!==''
+          ){
+            if(!userDetails.webdav_dir||userDetails.webdav_dir===''){
+                userDetails.webdav_dir='/'
+            }
+            userDetails.webdav_dir = s.checkCorrectPathEnding(userDetails.webdav_dir)
+            s.group[e.ke].webdav = webdav(
+                userDetails.webdav_url,
+                userDetails.webdav_user,
+                userDetails.webdav_pass
+            )
+        }
+    }
+    var unloadWebDavForUser = function(user){
+        s.group[user.ke].webdav = null
+    }
+    var deleteVideoFromWebDav = function(e,video,callback){
+        // e = user
+        try{
+            var videoDetails = JSON.parse(video.details)
+        }catch(err){
+            var videoDetails = video.details
+        }
+        if(!videoDetails.location){
+            var prefix = s.addUserPassToUrl(s.checkCorrectPathEnding(s.group[e.ke].init.webdav_url),s.group[e.ke].init.webdav_user,s.group[e.ke].init.webdav_pass)
+            videoDetails.location = video.href.replace(prefix,'')
+        }
+        s.group[e.ke].webdav.unlink(videoDetails.location, function(err) {
+            if (err) console.log(videoDetails.location,err)
+            callback()
+        })
+    }
+    var uploadVideoToWebDav = function(e,k){
+        //e = video object
+        //k = temporary values
+        if(!k)k={};
+        //cloud saver - webdav
+       var wfs = s.group[e.ke].webdav
+       if(wfs && s.group[e.ke].init.use_webdav !== '0' && s.group[e.ke].init.webdav_save === "1"){
+           var webdavUploadDir = s.group[e.ke].init.webdav_dir+e.ke+'/'+e.mid+'/'
+           var startWebDavUpload = function(){
+               s.group[e.ke].mon[e.id].webdavDirExist = true
+               var wfsWriteStream =
+               fs.createReadStream(k.dir + k.filename).pipe(wfs.createWriteStream(webdavUploadDir + k.filename))
+               if(s.group[e.ke].init.webdav_log === '1'){
+                   var webdavRemoteUrl = s.addUserPassToUrl(s.checkCorrectPathEnding(s.group[e.ke].init.webdav_url),s.group[e.ke].init.webdav_user,s.group[e.ke].init.webdav_pass) + s.group[e.ke].init.webdav_dir + e.ke + '/'+e.mid+'/'+k.filename
+                   var save = [
+                       e.mid,
+                       e.ke,
+                       k.startTime,
+                       1,
+                       s.s({
+                           type : 'webdav',
+                           location : webdavUploadDir + k.filename
+                       }),
+                       k.filesize,
+                       k.endTime,
+                       webdavRemoteUrl
+                   ]
+                   s.sqlQuery('INSERT INTO `Cloud Videos` (mid,ke,time,status,details,size,end,href) VALUES (?,?,?,?,?,?,?,?)',save)
+                   s.setCloudDiskUsedForGroup(e,{
+                       amount : k.filesizeMB,
+                       storageType : 'webdav'
+                   })
+                   s.purgeCloudDiskForGroup(e,'webdav')
+               }
+           }
+           if(s.group[e.ke].mon[e.id].webdavDirExist !== true){
+               //check if webdav dir exist
+               var parentPoint = 0
+               var webDavParentz = webdavUploadDir.split('/')
+               var webDavParents = []
+               webDavParentz.forEach(function(v){
+                   if(v && v !== '')webDavParents.push(v)
+               })
+               var stitchPieces = './'
+               var lastParentCheck = function(){
+                   ++parentPoint
+                   if(parentPoint === webDavParents.length){
+                       startWebDavUpload()
+                   }
+                   checkPathPiece(webDavParents[parentPoint])
+               }
+               var checkPathPiece = function(pathPiece){
+                   if(pathPiece && pathPiece !== ''){
+                       stitchPieces += pathPiece + '/'
+                       wfs.stat(stitchPieces, function(error, stats) {
+                           if(error){
+                               reply = {
+                                   status : error.status,
+                                   msg : lang.WebdavErrorTextTryCreatingDir,
+                                   dir : stitchPieces,
+                               }
+                               s.userLog(e,{type:lang['Webdav Error'],msg:reply})
+                               wfs.mkdir(stitchPieces, function(error) {
+                                   if(error){
+                                       reply = {
+                                           status : error.status,
+                                           msg : lang.WebdavErrorTextCreatingDir,
+                                           dir : stitchPieces,
+                                       }
+                                       s.userLog(e,{type:lang['Webdav Error'],msg:reply})
+                                   }else{
+                                       lastParentCheck()
+                                   }
+                               })
+                           }else{
+                               lastParentCheck()
+                           }
+                       })
+                   }else{
+                       ++parentPoint
+                   }
+               }
+               checkPathPiece(webDavParents[0])
+           }else{
+               startWebDavUpload()
+           }
+       }
+    }
+    //webdav
+    s.addCloudUploader({
+        name: 'webdav',
+        loadGroupAppExtender: loadWebDavForUser,
+        unloadGroupAppExtender: unloadWebDavForUser,
+        insertCompletedVideoExtender: uploadVideoToWebDav,
+        deleteVideoFromCloudExtensions: deleteVideoFromWebDav,
+        cloudDiskUseStartupExtensions: cloudDiskUseStartupForWebDav,
+        beforeAccountSave: beforeAccountSaveForWebDav,
+        onAccountSave: cloudDiskUseStartupForWebDav,
+    })
+}

libs/user.js

@@ -15,7 +15,7 @@ module.exports = function(s,config){
                         if(s.group[e.ke].sizePurgeQueue.length > 0){
                             checkQueue()
                         }else{
-                            s.group[e.ke].sizePurging=false
+                            s.group[e.ke].sizePurging = false
                             s.sendDiskUsedAmountToClients(e)
                         }
                     }
@@ -116,6 +116,9 @@ module.exports = function(s,config){
         s.tx({f:'log',ke:e.ke,mid:e.mid,log:x,time:s.timeObject()},'GRPLOG_'+e.ke);
     }
     s.loadGroup = function(e){
+        s.loadGroupExtensions.forEach(function(extender){
+            extender(e)
+        })
         if(!s.group[e.ke]){
             s.group[e.ke]={}
         }
@@ -145,7 +148,7 @@ module.exports = function(s,config){
                 ar=JSON.parse(r.details);
                 //load extenders
                 s.loadGroupAppExtensions.forEach(function(extender){
-                    extender(r)
+                    extender(r,ar)
                 })
                 //disk Used Emitter
                 if(!s.group[e.ke].diskUsedEmitter){
@@ -257,11 +260,11 @@ module.exports = function(s,config){
                     d.form.details.use_admin=d.d.use_admin
                     d.form.details.use_ldap=d.d.use_ldap
                     //check
-                    if(d.d.edit_days=="0"){
-                        d.form.details.days=d.d.days;
+                    if(d.d.edit_days == "0"){
+                        d.form.details.days = d.d.days;
                     }
-                    if(d.d.edit_size=="0"){
-                        d.form.details.size=d.d.size;
+                    if(d.d.edit_size == "0"){
+                        d.form.details.size = d.d.size;
                     }
                     if(d.d.sub){
                         d.form.details.sub=d.d.sub;
@@ -292,7 +295,7 @@ module.exports = function(s,config){
                             var userDetails = JSON.parse(d.form.details)
                             s.group[d.ke].sizeLimit = parseFloat(newSize)
                             s.onAccountSaveExtensions.forEach(function(extender){
-                                extender(s.group[d.ke],userDetails)
+                                extender(s.group[d.ke],userDetails,user)
                             })
                             s.unloadGroupAppExtensions.forEach(function(extender){
                                 extender(user)
@@ -305,4 +308,45 @@ module.exports = function(s,config){
             }
         })
     }
+    s.findPreset = function(presetQueryVals,callback){
+        //presetQueryVals = [ke, type, name]
+        s.sqlQuery("SELECT * FROM Presets WHERE ke=? AND type=? AND name=? LIMIT 1",presetQueryVals,function(err,presets){
+            var preset
+            var notFound = false
+            if(presets && presets[0]){
+                preset = presets[0]
+                s.checkDetails(preset)
+            }else{
+                notFound = true
+            }
+            callback(notFound,preset)
+        })
+    }
+    s.checkUserPurgeLock = function(groupKey){
+        var userGroup = s.group[groupKey]
+        if(s.group[groupKey].usedSpace > s.group[groupKey].sizeLimit){
+            s.group[groupKey].sizePurgeQueue = []
+            s.group[groupKey].sizePurging = false
+            s.systemLog(lang.sizePurgeLockedText + ' : ' + groupKey)
+            s.onStalePurgeLockExtensions.forEach(function(extender){
+                extender(groupKey,s.group[groupKey].usedSpace,s.group[groupKey].sizeLimit)
+            })
+        }
+    }
+    if(config.cron.deleteOverMax === true){
+        s.checkForStalePurgeLocks = function(){
+            var doCheck = function(){
+                Object.keys(s.group).forEach(function(groupKey){
+                    s.checkUserPurgeLock(groupKey)
+                })
+            }
+            clearTimeout(s.checkForStalePurgeLocksInterval)
+            s.checkForStalePurgeLocksInterval = setInterval(function(){
+                doCheck()
+            },1000 * 60 * 60)
+            doCheck()
+        }
+    }else{
+        s.checkForStalePurgeLocks = function(){}
+    }
 }

libs/videos.js

@@ -230,11 +230,6 @@ module.exports = function(s,config,lang){
                         })
                     })
                 })
-                fs.chmod(videoSnap,0o777,function(err){
-                    if(!err){
-                        fs.unlink(videoSnap,function(err){})
-                    }
-                })
             }else{
                 console.log(new Error())
                 console.log(lang['Database row does not exist'],queryValues)
@@ -242,55 +237,60 @@ module.exports = function(s,config,lang){
         })
     }
     s.deleteListOfVideos = function(videos){
-        var query = 'DELETE FROM Videos WHERE '
-        var videoQuery = []
-        var queryValues = []
-        videos.forEach(function(video){
-            s.checkDetails(video)
-            //e = video object
-            video.dir = s.getVideoDirectory(video)
-            if(!video.filename && video.time){
-                video.filename = s.formattedTime(video.time)
-            }
-            var filename,
-                time
-            if(video.filename.indexOf('.')>-1){
-                filename = video.filename
-            }else{
-                filename = video.filename+'.'+video.ext
-            }
-            if(video.filename && !video.time){
-                time = s.nameToTime(filename)
-            }else{
-                time = video.time
-            }
-            time = new Date(time)
-            fs.chmod(video.dir+filename,0o777,function(err){
-                s.tx({
-                    f: 'video_delete',
-                    filename: filename,
-                    mid: video.id,
-                    ke: video.ke,
-                    time: s.nameToTime(filename),
-                    end: s.formattedTime(new Date,'YYYY-MM-DD HH:mm:ss')
-                },'GRP_'+video.ke);
-                s.setDiskUsedForGroup(video,-(video.size / 1000000))
-                fs.unlink(video.dir+filename,function(err){
-                    fs.stat(video.dir+filename,function(err){
-                        if(!err){
-                            s.file('delete',video.dir+filename)
-                        }
+        var deleteSetOfVideos = function(videos){
+            var query = 'DELETE FROM Videos WHERE '
+            var videoQuery = []
+            var queryValues = []
+            videos.forEach(function(video){
+                s.checkDetails(video)
+                //e = video object
+                video.dir = s.getVideoDirectory(video)
+                if(!video.filename && video.time){
+                    video.filename = s.formattedTime(video.time)
+                }
+                var filename,
+                    time
+                if(video.filename.indexOf('.')>-1){
+                    filename = video.filename
+                }else{
+                    filename = video.filename+'.'+video.ext
+                }
+                if(video.filename && !video.time){
+                    time = s.nameToTime(filename)
+                }else{
+                    time = video.time
+                }
+                time = new Date(time)
+                fs.chmod(video.dir+filename,0o777,function(err){
+                    s.tx({
+                        f: 'video_delete',
+                        filename: filename,
+                        mid: video.id,
+                        ke: video.ke,
+                        time: s.nameToTime(filename),
+                        end: s.formattedTime(new Date,'YYYY-MM-DD HH:mm:ss')
+                    },'GRP_'+video.ke);
+                    s.setDiskUsedForGroup(video,-(video.size / 1000000))
+                    fs.unlink(video.dir+filename,function(err){
+                        fs.stat(video.dir+filename,function(err){
+                            if(!err){
+                                s.file('delete',video.dir+filename)
+                            }
+                        })
                     })
                 })
+                videoQuery.push('(`mid`=? AND `ke`=? AND `time`=?)')
+                queryValues = queryValues.concat([video.id,video.ke,time])
             })
-            videoQuery.push('(`mid`=? AND `ke`=? AND `time`=?)')
-            queryValues = queryValues.concat([video.id,video.ke,time])
-        })
-        query += videoQuery.join(' OR ')
-        s.sqlQuery(query,queryValues,function(err){
-            if(err){
-                s.systemLog(lang['List of Videos Delete Error'],err)
-            }
+            query += videoQuery.join(' OR ')
+            s.sqlQuery(query,queryValues,function(err){
+                if(err){
+                    s.systemLog(lang['List of Videos Delete Error'],err)
+                }
+            })
+        }
+        videos.chunk(100).forEach(function(videosChunk){
+            deleteSetOfVideos(videosChunk)
         })
     }
     s.deleteVideoFromCloudExtensions = {}
@@ -378,4 +378,40 @@ module.exports = function(s,config,lang){
             finish()
         }
     }
+    s.streamMp4FileOverHttp = function(filePath,req,res){
+        var ext = filePath.split('.')
+        ext = filePath[filePath.length - 1]
+        var total = fs.statSync(filePath).size;
+        if (req.headers['range']) {
+            try{
+                var range = req.headers.range;
+                var parts = range.replace(/bytes=/, "").split("-");
+                var partialstart = parts[0];
+                var partialend = parts[1];
+                var start = parseInt(partialstart, 10);
+                var end = partialend ? parseInt(partialend, 10) : total-1;
+                var chunksize = (end-start)+1;
+                var file = fs.createReadStream(filePath, {start: start, end: end});
+                req.headerWrite={ 'Content-Range': 'bytes ' + start + '-' + end + '/' + total, 'Accept-Ranges': 'bytes', 'Content-Length': chunksize, 'Content-Type': 'video/'+req.ext }
+                req.writeCode=206
+            }catch(err){
+                req.headerWrite={ 'Content-Length': total, 'Content-Type': 'video/'+req.ext};
+                var file = fs.createReadStream(filePath)
+                req.writeCode=200
+            }
+        } else {
+            req.headerWrite={ 'Content-Length': total, 'Content-Type': 'video/'+req.ext};
+            var file = fs.createReadStream(filePath)
+            req.writeCode=200
+        }
+        if(req.query.downloadName){
+            req.headerWrite['content-disposition']='attachment; filename="'+req.query.downloadName+'"';
+        }
+        res.writeHead(req.writeCode,req.headerWrite);
+        file.on('close',function(){
+            res.end()
+        })
+        file.pipe(res)
+        return file
+    }
 }

libs/webServer.js

@@ -50,6 +50,8 @@ module.exports = function(s,config,lang,io){
         if(config.renderPaths.grid === undefined){config.renderPaths.grid='pages/grid'}
         //slick.js (cycle) page
         if(config.renderPaths.cycle === undefined){config.renderPaths.cycle='pages/cycle'}
+    // Use uws/cws
+    if(config.useUWebsocketJs === undefined){config.useUWebsocketJs=true}
     //SSL options
     if(config.ssl&&config.ssl.key&&config.ssl.cert){
         config.ssl.key=fs.readFileSync(s.checkRelativePath(config.ssl.key),'utf8')
@@ -111,5 +113,11 @@ module.exports = function(s,config,lang,io){
         path:s.checkCorrectPathEnding(config.webPaths.super)+'socket.io',
         transports: ['websocket']
     })
+    if(config.useUWebsocketJs === true){
+        io.engine.ws = new (require('cws').Server)({
+            noServer: true,
+            perMessageDeflate: false
+        })
+    }
     return app
 }

libs/webServerAdminPaths.js

@@ -21,10 +21,10 @@ module.exports = function(s,config,lang,app){
                 return
             }
             var form = s.getPostData(req)
-            var uid = s.getPostData(req,'uid',false)
-            var mail = s.getPostData(req,'mail',false)
+            var uid = form.uid || s.getPostData(req,'uid',false)
+            var mail = form.mail || s.getPostData(req,'mail',false)
             if(form){
-                var keys = Object.keys(form)
+                var keys = ['details']
                 var condition = []
                 var value = []
                 keys.forEach(function(v){
@@ -68,8 +68,9 @@ module.exports = function(s,config,lang,app){
                 s.closeJsonResponse(res,endData)
                 return
             }
-            var uid = s.getPostData(req,'uid',false)
-            var mail = s.getPostData(req,'mail',false)
+            var form = s.getPostData(req)
+            var uid = form.uid || s.getPostData(req,'uid',false)
+            var mail = form.mail || s.getPostData(req,'mail',false)
             s.sqlQuery('DELETE FROM Users WHERE uid=? AND ke=? AND mail=?',[uid,req.params.ke,mail])
             s.sqlQuery("SELECT * FROM API WHERE ke=? AND uid=?",[req.params.ke,uid],function(err,rows){
                 if(rows && rows[0]){
@@ -132,6 +133,12 @@ module.exports = function(s,config,lang,app){
                                 uid: newId,
                                 mail: form.mail
                             },'ADM_'+req.params.ke)
+                            endData.user = {
+                                details: s.parseJSON(details),
+                                ke: req.params.ke,
+                                uid: newId,
+                                mail: form.mail
+                            }
                         }
                         res.end(s.prettyPrint(endData))
                     })
@@ -159,7 +166,6 @@ module.exports = function(s,config,lang,app){
             ok: false
         }
         res.setHeader('Content-Type', 'application/json');
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         s.auth(req.params,function(user){
             var hasRestrictions = user.details.sub && user.details.allmonitors !== '1'
             if(req.params.f !== 'delete'){
@@ -238,7 +244,6 @@ module.exports = function(s,config,lang,app){
     ],function (req,res){
         var endData = {ok:false}
         res.setHeader('Content-Type', 'application/json');
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         s.auth(req.params,function(user){
             var endData = {
                 ok : false
@@ -266,6 +271,7 @@ module.exports = function(s,config,lang,app){
                         },'GRP_' + req.params.ke)
                         endData.ok = true
                     }
+                    endData.api = insert
                     s.closeJsonResponse(res,endData)
                 })
             }else{
@@ -283,7 +289,6 @@ module.exports = function(s,config,lang,app){
     ],function (req,res){
         var endData = {ok:false}
         res.setHeader('Content-Type', 'application/json');
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         s.auth(req.params,function(user){
             var endData = {
                 ok : false
@@ -336,7 +341,6 @@ module.exports = function(s,config,lang,app){
     ],function (req,res){
         var endData = {ok:false}
         res.setHeader('Content-Type', 'application/json');
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         s.auth(req.params,function(user){
             var endData = {
                 ok : false
@@ -366,7 +370,7 @@ module.exports = function(s,config,lang,app){
     /**
     * API : Administrator : Get Monitor State Presets List
     */
-    app.get([
+    app.all([
         config.webPaths.apiPrefix+':auth/monitorStates/:ke',
         config.webPaths.adminApiPrefix+':auth/monitorStates/:ke'
     ],function (req,res){
@@ -411,19 +415,7 @@ module.exports = function(s,config,lang,app){
                 s.closeJsonResponse(res,endData)
                 return
             }
-            var findPreset = function(callback){
-                s.sqlQuery("SELECT * FROM Presets WHERE ke=? AND type=? AND name=? LIMIT 1",[req.params.ke,'monitorStates',req.params.stateName],function(err,presets){
-                    var preset
-                    var notFound = false
-                    if(presets && presets[0]){
-                        preset = presets[0]
-                        s.checkDetails(preset)
-                    }else{
-                        notFound = true
-                    }
-                    callback(notFound,preset)
-                })
-            }
+            var presetQueryVals = [req.params.ke,'monitorStates',req.params.stateName]
             switch(req.params.action){
                 case'insert':case'edit':
                     var form = s.getPostData(req)
@@ -433,7 +425,7 @@ module.exports = function(s,config,lang,app){
                         s.closeJsonResponse(res,endData)
                         return
                     }
-                    findPreset(function(notFound,preset){
+                    s.findPreset(presetQueryVals,function(notFound,preset){
                         if(notFound === true){
                             endData.msg = lang["Inserted State Configuration"]
                             var details = {
@@ -470,7 +462,7 @@ module.exports = function(s,config,lang,app){
                     })
                 break;
                 case'delete':
-                    findPreset(function(notFound,preset){
+                    s.findPreset(presetQueryVals,function(notFound,preset){
                         if(notFound === true){
                             endData.msg = user.lang['State Configuration Not Found']
                             s.closeJsonResponse(res,endData)
@@ -486,43 +478,8 @@ module.exports = function(s,config,lang,app){
                     })
                 break;
                 default://change monitors according to state
-                    findPreset(function(notFound,preset){
-                        if(notFound === false){
-                            var sqlQuery = 'SELECT * FROM Monitors WHERE ke=? AND '
-                            var monitorQuery = []
-                            var sqlQueryValues = [req.params.ke]
-                            var monitorPresets = {}
-                            preset.details.monitors.forEach(function(monitor){
-                                monitorQuery.push('mid=?')
-                                sqlQueryValues.push(monitor.mid)
-                                monitorPresets[monitor.mid] = monitor
-                            })
-                            sqlQuery += '('+monitorQuery.join(' OR ')+')'
-                            s.sqlQuery(sqlQuery,sqlQueryValues,function(err,monitors){
-                                if(monitors && monitors[0]){
-                                    monitors.forEach(function(monitor){
-                                        s.checkDetails(monitor)
-                                        s.checkDetails(monitorPresets[monitor.mid])
-                                        var monitorPreset = monitorPresets[monitor.mid]
-                                        monitorPreset.details = Object.assign(monitor.details,monitorPreset.details)
-                                        monitor = s.cleanMonitorObjectForDatabase(Object.assign(monitor,monitorPreset))
-                                        monitor.details = JSON.stringify(monitor.details)
-                                        s.addOrEditMonitor(Object.assign(monitor,{}),function(err,endData){
-
-                                        },user)
-                                    })
-                                    endData.ok = true
-                                    s.tx({f:'change_group_state',ke:req.params.ke,name:req.params.stateName},'GRP_'+req.params.ke)
-                                    s.closeJsonResponse(res,endData)
-                                }else{
-                                    endData.msg = user.lang['State Configuration has no monitors associated']
-                                    s.closeJsonResponse(res,endData)
-                                }
-                            })
-                        }else{
-                            endData.msg = user.lang['State Configuration Not Found']
-                            s.closeJsonResponse(res,endData)
-                        }
+                    s.activateMonitorStates(req.params.ke,req.params.stateName,user,function(endData){
+                        s.closeJsonResponse(res,endData)
                     })
                 break;
             }

libs/webServerPaths.js

@@ -8,6 +8,7 @@ var execSync = require('child_process').execSync;
 var exec = require('child_process').exec;
 var spawn = require('child_process').spawn;
 var httpProxy = require('http-proxy');
+var onvif = require('node-onvif');
 var proxy = httpProxy.createProxyServer({})
 var ejs = require('ejs');
 var CircularJSON = require('circular-json');
@@ -67,6 +68,10 @@ module.exports = function(s,config,lang,app,io){
     app.use(s.checkCorrectPathEnding(config.webPaths.super)+'libs',express.static(s.mainDirectory + '/web/libs'))
     app.use(bodyParser.json());
     app.use(bodyParser.urlencoded({extended: true}));
+    app.use(function (req,res,next){
+        res.header("Access-Control-Allow-Origin",req.headers.origin);
+        next()
+    })
     app.set('views', s.mainDirectory + '/web');
     app.set('view engine','ejs');
     //add template handler
@@ -125,7 +130,6 @@ module.exports = function(s,config,lang,app,io){
     app.get(config.webPaths.apiPrefix+':auth/userInfo/:ke',function (req,res){
         req.ret={ok:false};
         res.setHeader('Content-Type', 'application/json');
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         s.auth(req.params,function(user){
             req.ret.ok=true
             req.ret.user=user
@@ -151,9 +155,6 @@ module.exports = function(s,config,lang,app,io){
         s.checkCorrectPathEnding(config.webPaths.super)+':screen',
     ],function (req,res){
         req.ip = s.getClientIp(req)
-        if(req.query.json === 'true'){
-            res.header("Access-Control-Allow-Origin",req.headers.origin);
-        }
         var screenChooser = function(screen){
             var search = function(screen){
                 if(req.url.indexOf(screen) > -1){
@@ -181,7 +182,7 @@ module.exports = function(s,config,lang,app,io){
                 s.renderPage(req,res,config.renderPaths.index,{
                     failedLogin: true,
                     message: lang.failedLoginText1,
-                    lang: lang,
+                    lang: s.copySystemDefaultLanguage(),
                     config: config,
                     screen: screenChooser(req.params.screen)
                 },function(err,html){
@@ -239,7 +240,7 @@ module.exports = function(s,config,lang,app,io){
                 s.renderPage(req,res,config.renderPaths.index,{
                     failedLogin: true,
                     message: lang.failedLoginText2,
-                    lang: lang,
+                    lang: s.copySystemDefaultLanguage(),
                     config: config,
                     screen: screenChooser(req.params.screen)
                 },function(err,html){
@@ -286,7 +287,8 @@ module.exports = function(s,config,lang,app,io){
                             // config: config,
                             $user: req.resp,
                             lang: r.lang,
-                            define: s.getDefinitonFile(r.details.lang)
+                            define: s.getDefinitonFile(r.details.lang),
+                            customAutoLoad: s.customAutoLoadTree
                         })
                     })
                 break;
@@ -297,7 +299,8 @@ module.exports = function(s,config,lang,app,io){
                             // config: config,
                             $user: req.resp,
                             lang: r.lang,
-                            define: s.getDefinitonFile(r.details.lang)
+                            define: s.getDefinitonFile(r.details.lang),
+                            customAutoLoad: s.customAutoLoadTree
                         })
                     })
                 break;
@@ -311,17 +314,36 @@ module.exports = function(s,config,lang,app,io){
                                     $subs: rr,
                                     $mons: rrr,
                                     lang: r.lang,
-                                    define: s.getDefinitonFile(r.details.lang)
+                                    define: s.getDefinitonFile(r.details.lang),
+                                    customAutoLoad: s.customAutoLoadTree
                                 })
                             })
                         })
                     }else{
                         //not admin user
-                        renderPage(config.renderPaths.home,{$user:req.resp,config:config,lang:r.lang,define:s.getDefinitonFile(r.details.lang),addStorage:s.dir.addStorage,fs:fs,__dirname:s.mainDirectory});
+                        renderPage(config.renderPaths.home,{
+                            $user:req.resp,
+                            config:config,
+                            lang:r.lang,
+                            define:s.getDefinitonFile(r.details.lang),
+                            addStorage:s.dir.addStorage,
+                            fs:fs,
+                            __dirname:s.mainDirectory,
+                            customAutoLoad: s.customAutoLoadTree
+                        });
                     }
                 break;
                 default:
-                    renderPage(config.renderPaths.home,{$user:req.resp,config:config,lang:r.lang,define:s.getDefinitonFile(r.details.lang),addStorage:s.dir.addStorage,fs:fs,__dirname:s.mainDirectory});
+                    renderPage(config.renderPaths.home,{
+                        $user:req.resp,
+                        config:config,
+                        lang:r.lang,
+                        define:s.getDefinitonFile(r.details.lang),
+                        addStorage:s.dir.addStorage,
+                        fs:fs,
+                        __dirname:s.mainDirectory,
+                        customAutoLoad: s.customAutoLoadTree
+                    });
                 break;
             }
             s.userLog({ke:r.ke,mid:'$USER'},{type:r.lang['New Authentication Token'],msg:{for:req.body.function,mail:r.mail,id:r.uid,ip:req.ip}})
@@ -511,6 +533,7 @@ module.exports = function(s,config,lang,app,io){
                                 r=[]
                             }
                             data.Logs = r
+                            data.customAutoLoad = s.customAutoLoadTree
                             fs.readFile(s.location.config,'utf8',function(err,file){
                                 data.plainConfig = JSON.parse(file)
                                 renderPage(config.renderPaths.super,data)
@@ -558,7 +581,6 @@ module.exports = function(s,config,lang,app,io){
     * API : Brute Protection Lock Reset by API
     */
     app.get([config.webPaths.apiPrefix+':auth/resetBruteProtection/:ke'], function (req,res){
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         s.auth(req.params,function(user){
             if(s.failedLoginAttempts[user.mail]){
                 clearTimeout(s.failedLoginAttempts[user.mail].timeout)
@@ -576,7 +598,6 @@ module.exports = function(s,config,lang,app,io){
         config.webPaths.apiPrefix+':auth/cycle/:ke',
         config.webPaths.apiPrefix+':auth/cycle/:ke/:group'
     ], function(req,res) {
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         s.auth(req.params,function(user){
             if(user.permissions.get_monitors==="0"){
                 res.end(user.lang['Not Permitted'])
@@ -705,7 +726,6 @@ module.exports = function(s,config,lang,app,io){
         }else{
             res.setHeader('Content-Type', 'application/json');
         }
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         req.fn=function(user){
             if(user.permissions.get_monitors==="0"){
                 res.end(s.prettyPrint([]))
@@ -821,7 +841,6 @@ module.exports = function(s,config,lang,app,io){
     app.get([config.webPaths.apiPrefix+':auth/monitor/:ke',config.webPaths.apiPrefix+':auth/monitor/:ke/:id'], function (req,res){
         req.ret={ok:false};
         res.setHeader('Content-Type', 'application/json');
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         req.fn=function(user){
         if(user.permissions.get_monitors==="0"){
             res.end(s.prettyPrint([]))
@@ -903,6 +922,54 @@ module.exports = function(s,config,lang,app,io){
         s.auth(req.params,req.fn,res,req);
     });
     /**
+    * API : Merge Recorded Videos into one file
+     */
+     app.get(config.webPaths.apiPrefix+':auth/videosMerge/:ke', function (req,res){
+         var failed = function(resp){
+             res.setHeader('Content-Type', 'application/json');
+             res.end(s.prettyPrint(resp))
+         }
+         if(req.query.videos && req.query.videos !== ''){
+             s.auth(req.params,function(user){
+                 var videosSelected = JSON.parse(req.query.videos)
+                 var where = []
+                 var values = []
+                 videosSelected.forEach(function(video){
+                     where.push("(ke=? AND mid=? AND `time`=?)")
+                     if(!video.ke)video.ke = req.params.ke
+                     values.push(video.ke)
+                     values.push(video.mid)
+                     var time = s.nameToTime(video.filename)
+                     if(req.query.isUTC === 'true'){
+                         time = s.utcToLocal(time)
+                     }
+                     time = new Date(time)
+                     values.push(time)
+                 })
+                 s.sqlQuery('SELECT * FROM Videos WHERE '+where.join(' OR '),values,function(err,r){
+                     var resp = {ok: false}
+                     if(r && r[0]){
+                         s.mergeRecordedVideos(r,req.params.ke,function(fullPath,filename){
+                             res.setHeader('Content-Disposition', 'attachment; filename="'+filename+'"')
+                             var file = fs.createReadStream(fullPath)
+                             file.on('close',function(){
+                                 setTimeout(function(){
+                                     s.file('delete',fullPath)
+                                 },1000 * 60 * 3)
+                                 res.end()
+                             })
+                             file.pipe(res)
+                         })
+                     }else{
+                         failed({ok:false,msg:'No Videos Found'})
+                     }
+                 })
+             },res,req);
+         }else{
+             failed({ok:false,msg:'"videos" query variable is missing from request.'})
+         }
+    })
+    /**
     * API : Get Videos
      */
     app.get([
@@ -912,7 +979,6 @@ module.exports = function(s,config,lang,app,io){
         config.webPaths.apiPrefix+':auth/cloudVideos/:ke/:id'
     ], function (req,res){
         res.setHeader('Content-Type', 'application/json');
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         s.auth(req.params,function(user){
             var hasRestrictions = user.details.sub && user.details.allmonitors !== '1'
             if(
@@ -1033,7 +1099,6 @@ module.exports = function(s,config,lang,app,io){
     app.get([config.webPaths.apiPrefix+':auth/events/:ke',config.webPaths.apiPrefix+':auth/events/:ke/:id',config.webPaths.apiPrefix+':auth/events/:ke/:id/:limit',config.webPaths.apiPrefix+':auth/events/:ke/:id/:limit/:start',config.webPaths.apiPrefix+':auth/events/:ke/:id/:limit/:start/:end'], function (req,res){
         req.ret={ok:false};
         res.setHeader('Content-Type', 'application/json');
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         s.auth(req.params,function(user){
             if(user.permissions.watch_videos==="0"||user.details.sub&&user.details.allmonitors!=='1'&&user.details.video_view.indexOf(req.params.id)===-1){
                 res.end(s.prettyPrint([]))
@@ -1091,7 +1156,6 @@ module.exports = function(s,config,lang,app,io){
     app.get([config.webPaths.apiPrefix+':auth/logs/:ke',config.webPaths.apiPrefix+':auth/logs/:ke/:id'], function (req,res){
         req.ret={ok:false};
         res.setHeader('Content-Type', 'application/json');
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         s.auth(req.params,function(user){
             if(user.permissions.get_logs==="0" || user.details.sub && user.details.view_logs !== '1'){
                 res.end(s.prettyPrint([]))
@@ -1156,7 +1220,6 @@ module.exports = function(s,config,lang,app,io){
     app.get(config.webPaths.apiPrefix+':auth/smonitor/:ke', function (req,res){
         req.ret={ok:false};
         res.setHeader('Content-Type', 'application/json');
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         req.fn=function(user){
             if(user.permissions.get_monitors==="0"){
                 res.end(s.prettyPrint([]))
@@ -1193,7 +1256,6 @@ module.exports = function(s,config,lang,app,io){
     app.get([config.webPaths.apiPrefix+':auth/monitor/:ke/:id/:f',config.webPaths.apiPrefix+':auth/monitor/:ke/:id/:f/:ff',config.webPaths.apiPrefix+':auth/monitor/:ke/:id/:f/:ff/:fff'], function (req,res){
         req.ret={ok:false};
         res.setHeader('Content-Type', 'application/json');
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         s.auth(req.params,function(user){
             if(user.permissions.control_monitors==="0"||user.details.sub&&user.details.allmonitors!=='1'&&user.details.monitor_edit.indexOf(req.params.id)===-1){
                 res.end(user.lang['Not Permitted'])
@@ -1288,7 +1350,6 @@ module.exports = function(s,config,lang,app,io){
      */
     app.get([config.webPaths.apiPrefix+':auth/fileBin/:ke',config.webPaths.apiPrefix+':auth/fileBin/:ke/:id'],function (req,res){
         res.setHeader('Content-Type', 'application/json');
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         req.fn=function(user){
             req.sql='SELECT * FROM Files WHERE ke=?';req.ar=[req.params.ke];
             if(user.details.sub&&user.details.monitors&&user.details.allmonitors!=='1'){
@@ -1321,7 +1382,6 @@ module.exports = function(s,config,lang,app,io){
     * API : Get fileBin file
      */
     app.get(config.webPaths.apiPrefix+':auth/fileBin/:ke/:id/:year/:month/:day/:file', function (req,res){
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         req.fn=function(user){
             req.failed=function(){
                 res.end(user.lang['File Not Found'])
@@ -1352,7 +1412,6 @@ module.exports = function(s,config,lang,app,io){
     * API : Zip Videos and Get Link from fileBin
      */
     app.get(config.webPaths.apiPrefix+':auth/zipVideos/:ke', function (req,res){
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         var failed = function(resp){
             res.setHeader('Content-Type', 'application/json');
             res.end(s.prettyPrint(resp))
@@ -1375,7 +1434,7 @@ module.exports = function(s,config,lang,app,io){
                     values.push(time)
                 })
                 s.sqlQuery('SELECT * FROM Videos WHERE '+where.join(' OR '),values,function(err,r){
-                    var resp = {ok:false}
+                    var resp = {ok: false}
                     if(r && r[0]){
                         resp.ok = true
                         var zipDownload = null
@@ -1396,7 +1455,7 @@ module.exports = function(s,config,lang,app,io){
                             fs.mkdirSync(fileBinDir);
                         }
                         r.forEach(function(video){
-                            timeFormatted = s.formattedTime(video.time)
+                            var timeFormatted = s.formattedTime(video.time)
                             video.filename = timeFormatted+'.'+video.ext
                             var dir = s.getVideoDirectory(video)+video.filename
                             var tempVideoFile = timeFormatted+' - '+video.mid+'.'+video.ext
@@ -1418,16 +1477,27 @@ module.exports = function(s,config,lang,app,io){
                             var zipDownload = fs.createReadStream(zippedFile)
                             zipDownload.pipe(res)
                             zipDownload.on('error', function (error) {
-                                s.userLog({ke:req.params.ke,mid:'$USER'},{title:'Zip Download Error',msg:error.toString()})
+                                var errorString = error.toString()
+                                s.userLog({
+                                    ke: req.params.ke,
+                                    mid: '$USER'
+                                },{
+                                    title: 'Zip Download Error',
+                                    msg: errorString
+                                })
                                 if(zipDownload && zipDownload.destroy){
                                     zipDownload.destroy()
                                 }
-                            });
+                                res.end(s.prettyPrint({
+                                    ok: false,
+                                    msg: errorString
+                                }))
+                            })
                             zipDownload.on('close', function () {
                                 res.end()
-                                zipDownload.destroy();
-                                fs.unlinkSync(zippedFile);
-                            });
+                                zipDownload.destroy()
+                                fs.unlinkSync(zippedFile)
+                            })
                         })
                     }else{
                         failed({ok:false,msg:'No Videos Found'})
@@ -1437,7 +1507,120 @@ module.exports = function(s,config,lang,app,io){
         }else{
             failed({ok:false,msg:'"videos" query variable is missing from request.'})
         }
-    });
+    })
+    /**
+    * API : Zip Cloud Videos and Get Link from fileBin
+     */
+    app.get(config.webPaths.apiPrefix+':auth/zipCloudVideos/:ke', function (req,res){
+        var failed = function(resp){
+            res.setHeader('Content-Type', 'application/json');
+            res.end(s.prettyPrint(resp))
+        }
+        if(req.query.videos && req.query.videos !== ''){
+            s.auth(req.params,function(user){
+                var videosSelected = JSON.parse(req.query.videos)
+                var where = []
+                var values = []
+                videosSelected.forEach(function(video){
+                    where.push("(ke=? AND mid=? AND `time`=?)")
+                    if(!video.ke)video.ke = req.params.ke
+                    values.push(video.ke)
+                    values.push(video.mid)
+                    var time = s.nameToTime(video.filename)
+                    if(req.query.isUTC === 'true'){
+                        time = s.utcToLocal(time)
+                    }
+                    time = new Date(time)
+                    values.push(time)
+                })
+                s.sqlQuery('SELECT * FROM `Cloud Videos` WHERE '+where.join(' OR '),values,function(err,r){
+                    var resp = {ok: false}
+                    if(r && r[0]){
+                        resp.ok = true
+                        var zipDownload = null
+                        var tempFiles = []
+                        var fileId = s.gid()
+                        var fileBinDir = s.dir.fileBin+req.params.ke+'/'
+                        var tempScript = s.dir.streams+req.params.ke+'/'+fileId+'.sh'
+                        var zippedFilename = s.formattedTime()+'-'+fileId+'-Shinobi_Cloud_Backed_Recordings.zip'
+                        var zippedFile = fileBinDir+zippedFilename
+                        var script = 'cd '+fileBinDir+' && zip -9 -r '+zippedFile
+                        res.on('close', () => {
+                            if(zipDownload && zipDownload.destroy){
+                                zipDownload.destroy()
+                            }
+                            fs.unlink(zippedFile);
+                        })
+                        if(!fs.existsSync(fileBinDir)){
+                            fs.mkdirSync(fileBinDir);
+                        }
+                        var cloudDownloadCount = 0
+                        var getFile = function(video,completed){
+                            if(!video)completed();
+                            s.checkDetails(video)
+                            var filename = video.href.split('/')
+                            filename = filename[filename.length - 1]
+                            var timeFormatted = s.formattedTime(video.time)
+                            var tempVideoFile = video.details.type + '-' + video.mid + '-' + filename
+                            var tempFileWriteStream = fs.createWriteStream(fileBinDir+tempVideoFile)
+                            tempFileWriteStream.on('finish', function() {
+                                ++cloudDownloadCount
+                                getFile(r[cloudDownloadCount],completed)
+                            })
+                            var cloudVideoDownload = request(video.href)
+                            cloudVideoDownload.on('response',  function (res) {
+                                res.pipe(tempFileWriteStream)
+                            })
+                            tempFiles.push(fileBinDir+tempVideoFile)
+                            script += ' "'+tempVideoFile+'"'
+                        }
+                        getFile(r[cloudDownloadCount],function(){
+                            fs.writeFileSync(tempScript,script,'utf8')
+                            var zipCreate = spawn('sh',(tempScript).split(' '),{detached: true})
+                            zipCreate.stderr.on('data',function(data){
+                                s.userLog({ke:req.params.ke,mid:'$USER'},{title:'Zip Create Error',msg:data.toString()})
+                            })
+                            zipCreate.on('exit',function(data){
+                                fs.unlinkSync(tempScript)
+                                tempFiles.forEach(function(file){
+                                    fs.unlink(file,function(){})
+                                })
+                                res.setHeader('Content-Disposition', 'attachment; filename="' + zippedFilename + '"')
+                                var zipDownload = fs.createReadStream(zippedFile)
+                                zipDownload.pipe(res)
+                                zipDownload.on('error', function (error) {
+                                    var errorString = error.toString()
+                                    s.userLog({
+                                        ke: req.params.ke,
+                                        mid: '$USER'
+                                    },{
+                                        title: 'Zip Download Error',
+                                        msg: errorString
+                                    })
+                                    if(zipDownload && zipDownload.destroy){
+                                        zipDownload.destroy()
+                                    }
+                                    res.end(s.prettyPrint({
+                                        ok: false,
+                                        msg: errorString
+                                    }))
+                                })
+                                zipDownload.on('close', function () {
+                                    res.end()
+                                    zipDownload.destroy()
+                                    fs.unlinkSync(zippedFile)
+                                })
+                            })
+                        })
+                    }else{
+                        failed({ok:false,msg:'No Videos Found'})
+                    }
+                })
+            },res,req);
+        }else{
+            failed({ok:false,msg:'"videos" query variable is missing from request.'})
+        }
+    })
     /**
     * API : Get Cloud Video File (proxy)
      */
@@ -1480,38 +1663,7 @@ module.exports = function(s,config,lang,app,io){
                 if(r&&r[0]){
                     req.dir=s.getVideoDirectory(r[0])+req.params.file
                     if (fs.existsSync(req.dir)){
-                        req.ext=req.params.file.split('.')[1];
-                        var total = fs.statSync(req.dir).size;
-                        if (req.headers['range']) {
-                            try{
-                                var range = req.headers.range;
-                                var parts = range.replace(/bytes=/, "").split("-");
-                                var partialstart = parts[0];
-                                var partialend = parts[1];
-                                var start = parseInt(partialstart, 10);
-                                var end = partialend ? parseInt(partialend, 10) : total-1;
-                                var chunksize = (end-start)+1;
-                                var file = fs.createReadStream(req.dir, {start: start, end: end});
-                                req.headerWrite={ 'Content-Range': 'bytes ' + start + '-' + end + '/' + total, 'Accept-Ranges': 'bytes', 'Content-Length': chunksize, 'Content-Type': 'video/'+req.ext }
-                                req.writeCode=206
-                            }catch(err){
-                                req.headerWrite={ 'Content-Length': total, 'Content-Type': 'video/'+req.ext};
-                                var file = fs.createReadStream(req.dir)
-                                req.writeCode=200
-                            }
-                        } else {
-                            req.headerWrite={ 'Content-Length': total, 'Content-Type': 'video/'+req.ext};
-                            var file=fs.createReadStream(req.dir)
-                            req.writeCode=200
-                        }
-                        if(req.query.downloadName){
-                            req.headerWrite['content-disposition']='attachment; filename="'+req.query.downloadName+'"';
-                        }
-                        res.writeHead(req.writeCode,req.headerWrite);
-                        file.on('close',function(){
-                            res.end();
-                        })
-                        file.pipe(res);
+                        s.streamMp4FileOverHttp(req.dir,req,res)
                     }else{
                         res.end(user.lang['File Not Found in Filesystem'])
                     }
@@ -1524,27 +1676,34 @@ module.exports = function(s,config,lang,app,io){
     /**
     * API : Motion Trigger via GET request
      */
-    app.get(config.webPaths.apiPrefix+':auth/motion/:ke/:id', function (req,res){
-        s.auth(req.params,function(user){
-            if(req.query.data){
-                try{
-                    var d={id:req.params.id,ke:req.params.ke,details:JSON.parse(req.query.data)};
-                }catch(err){
-                    res.end('Data Broken',err);
-                    return;
-                }
-            }else{
-                res.end('No Data');
-                return;
-            }
-            if(!d.ke||!d.id||!s.group[d.ke]){
-                res.end(user.lang['No Group with this key exists']);
-                return;
-            }
-            s.triggerEvent(d)
-            res.end(user.lang['Trigger Successful'])
-        },res,req);
-    })
+     app.get(config.webPaths.apiPrefix+':auth/motion/:ke/:id', function (req,res){
+         s.auth(req.params,function(user){
+             var endData = {
+
+             }
+             if(req.query.data){
+                 try{
+                     var d = {
+                         id: req.params.id,
+                         ke: req.params.ke,
+                         details: JSON.parse(req.query.data)
+                     }
+                 }catch(err){
+                     res.end('Data Broken',err)
+                     return
+                 }
+             }else{
+                 res.end('No Data')
+                 return
+             }
+             if(!d.ke||!d.id||!s.group[d.ke]){
+                 res.end(user.lang['No Group with this key exists'])
+                 return
+             }
+             s.triggerEvent(d)
+             res.end(user.lang['Trigger Successful'])
+         },res,req)
+     })
     /**
     * API : WebHook Tester
      */
@@ -1560,7 +1719,6 @@ module.exports = function(s,config,lang,app,io){
      */
     app.get(config.webPaths.apiPrefix+':auth/control/:ke/:id/:direction', function (req,res){
         res.setHeader('Content-Type', 'application/json');
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         s.auth(req.params,function(user){
             s.cameraControl(req.params,function(resp){
                 res.end(s.prettyPrint(resp))
@@ -1578,7 +1736,6 @@ module.exports = function(s,config,lang,app,io){
     ], function (req,res){
         req.ret={ok:false};
         res.setHeader('Content-Type', 'application/json');
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         s.auth(req.params,function(user){
             if(user.permissions.watch_videos==="0"||user.details.sub&&user.details.allmonitors!=='1'&&user.details.video_delete.indexOf(req.params.id)===-1){
                 res.end(user.lang['Not Permitted'])
@@ -1673,7 +1830,6 @@ module.exports = function(s,config,lang,app,io){
     app.get(config.webPaths.apiPrefix+':auth/probe/:ke',function (req,res){
         req.ret={ok:false};
         res.setHeader('Content-Type', 'application/json');
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         s.auth(req.params,function(user){
             switch(req.query.action){
     //            case'stop':
@@ -1720,7 +1876,6 @@ module.exports = function(s,config,lang,app,io){
     app.all([config.webPaths.apiPrefix+':auth/onvif/:ke/:id/:action',config.webPaths.apiPrefix+':auth/onvif/:ke/:id/:service/:action'],function (req,res){
         var response = {ok:false};
         res.setHeader('Content-Type', 'application/json');
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         s.auth(req.params,function(user){
             var errorMessage = function(msg,error){
                 response.ok = false
@@ -1865,4 +2020,13 @@ module.exports = function(s,config,lang,app,io){
             s.closeJsonResponse(res,endData)
         },res,req)
     })
+    /**
+    * Robots.txt
+    */
+    app.get('/robots.txt', function (req,res){
+        res.on('finish',function(){
+            res.end()
+        })
+        fs.createReadStream(s.mainDirectory + '/web/pages/robots.txt').pipe(res)
+    })
 }

libs/webServerStreamPaths.js

@@ -16,7 +16,6 @@ module.exports = function(s,config,lang,app){
     * Page : Get Embed Stream
      */
     app.get([config.webPaths.apiPrefix+':auth/embed/:ke/:id',config.webPaths.apiPrefix+':auth/embed/:ke/:id/:addon'], function (req,res){
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         req.params.protocol=req.protocol;
         s.auth(req.params,function(user){
             if(user.permissions.watch_stream==="0"||user.details.sub&&user.details.allmonitors!=='1'&&user.details.monitors.indexOf(req.params.id)===-1){
@@ -100,7 +99,6 @@ module.exports = function(s,config,lang,app){
      * @param {string} full - if `true` page will load the MJPEG iframe page
      */
     app.get([config.webPaths.apiPrefix+':auth/mjpeg/:ke/:id',config.webPaths.apiPrefix+':auth/mjpeg/:ke/:id/:channel'], function(req,res) {
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         if(req.query.full=='true'){
             s.renderPage(req,res,config.renderPaths.mjpeg,{url:config.webPaths.apiPrefix + req.params.auth+'/mjpeg/'+req.params.ke+'/'+req.params.id,originalURL:s.getOriginalUrl(req)});
             res.end()
@@ -163,7 +161,6 @@ module.exports = function(s,config,lang,app){
     * API : Get HLS Stream
     */
     app.get([config.webPaths.apiPrefix+':auth/hls/:ke/:id/:file',config.webPaths.apiPrefix+':auth/hls/:ke/:id/:channel/:file'], function (req,res){
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         req.fn=function(user){
             s.checkChildProxy(req.params,function(){
                 req.dir=s.dir.streams+req.params.ke+'/'+req.params.id+'/'
@@ -186,7 +183,6 @@ module.exports = function(s,config,lang,app){
     * API : Get JPEG Snapshot
     */
     app.get(config.webPaths.apiPrefix+':auth/jpeg/:ke/:id/s.jpg', function(req,res){
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         s.auth(req.params,function(user){
             s.checkChildProxy(req.params,function(){
                 if(user.details.sub&&user.details.allmonitors!=='1'&&user.details.monitors&&user.details.monitors.indexOf(req.params.id)===-1){
@@ -212,7 +208,6 @@ module.exports = function(s,config,lang,app){
     * API : Get FLV Stream
     */
     app.get([config.webPaths.apiPrefix+':auth/flv/:ke/:id/s.flv',config.webPaths.apiPrefix+':auth/flv/:ke/:id/:channel/s.flv'], function(req,res) {
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         s.auth(req.params,function(user){
             s.checkChildProxy(req.params,function(){
                 var Emitter,chunkChannel
@@ -263,7 +258,6 @@ module.exports = function(s,config,lang,app){
     * API : Get H.265/h265 HEVC stream
     */
     app.get([config.webPaths.apiPrefix+':auth/h265/:ke/:id/s.hevc',config.webPaths.apiPrefix+':auth/h265/:ke/:id/:channel/s.hevc'], function(req,res) {
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         s.auth(req.params,function(user){
             s.checkChildProxy(req.params,function(){
                 var Emitter,chunkChannel
@@ -313,7 +307,6 @@ module.exports = function(s,config,lang,app){
         config.webPaths.apiPrefix+':auth/h264/:ke/:id/:feed',
         config.webPaths.apiPrefix+':auth/h264/:ke/:id'
     ], function (req, res) {
-        res.header("Access-Control-Allow-Origin",req.headers.origin);
         s.auth(req.params,function(user){
             s.checkChildProxy(req.params,function(){
                 if(!req.query.feed){req.query.feed='1'}

libs/webServerSuperPaths.js

@@ -288,6 +288,7 @@ module.exports = function(s,config,lang,app){
                                     ]
                                 )
                                 s.tx({f:'add_account',details:form.details,ke:form.ke,uid:form.uid,mail:form.mail},'$')
+                                endData.user = Object.assign(form,{})
                                 //init user
                                 s.loadGroup(form)
                             }
@@ -324,7 +325,7 @@ module.exports = function(s,config,lang,app){
                         r = r[0]
                         var details = JSON.parse(r.details)
                         if(form.pass && form.pass !== ''){
-                           if(form.pass === form.password_again){
+                           if(form.pass === form.password_again || form.pass_again){
                                form.pass = s.createHash(form.pass);
                            }else{
                                endData.msg = lang["Passwords Don't Match"]
@@ -335,11 +336,18 @@ module.exports = function(s,config,lang,app){
                             delete(form.pass);
                         }
                         delete(form.password_again);
+                        delete(form.pass_again);
                         var keys = Object.keys(form)
                         var set = []
                         var values = []
                         keys.forEach(function(v,n){
-                            if(set==='ke'||set==='password_again'||!form[v]){return}
+                            if(
+                                set === 'ke' ||
+                                !form[v]
+                            ){
+                                //skip
+                                return
+                            }
                             set.push(v+'=?')
                             if(v === 'details'){
                                 form[v] = s.stringJSON(Object.assign(details,s.parseJSON(form[v])))
@@ -360,6 +368,9 @@ module.exports = function(s,config,lang,app){
                             }
                             close()
                         })
+                    }else{
+                        endData.msg = lang['User Not Found']
+                        close()
                     }
                 })
             }else{
@@ -409,53 +420,218 @@ module.exports = function(s,config,lang,app){
         },res,req)
     })
     /**
-    * API : Superuser : Export Entire System
+    * API : Superuser : Get Entire System
     */
     app.all(config.webPaths.superApiPrefix+':auth/export/system', function (req,res){
+        s.superAuth(req.params,function(resp){
+            s.systemLog('Copy of the Database Exported',{
+                by: resp.$user.mail,
+                ip: resp.ip
+            })
+            var endData = {
+                ok : true
+            }
+            // var database = s.getPostData(req,'database')
+            endData.database = {}
+            var tableNames = [
+                'Users',
+                'Monitors',
+                'API',
+                'Videos',
+                'Cloud Videos',
+                'Logs',
+                'Files',
+                'Presets',
+            ]
+            var completedTables = 0
+            var tableExportLoop = function(callback){
+                var tableName = tableNames[completedTables]
+                if(tableName){
+                    var tableIsSelected = s.getPostData(req,tableName) == 1
+                    if(tableIsSelected){
+                        s.sqlQuery('SELECT * FROM `' + tableName +'`',[],function(err,dataRows){
+                            endData.database[tableName] = dataRows
+                            ++completedTables
+                            tableExportLoop(callback)
+                        })
+                    }else{
+                        ++completedTables
+                        tableExportLoop(callback)
+                    }
+                }else{
+                    callback()
+                }
+            }
+            tableExportLoop(function(){
+                s.closeJsonResponse(res,endData)
+            })
+        },res,req)
+    })
+    /**
+    * API : Superuser : Import Entire System
+    */
+    app.all(config.webPaths.superApiPrefix+':auth/import/system', function (req,res){
+        s.superAuth(req.params,function(resp){
+            var endData = {
+                ok : false
+            }
+            console.log(req.files)
+            // insert data
+            var data = s.getPostData(req)
+            var database = s.getPostData(req,'database')
+            if(data && data.database)database = data.database
+            if(database){
+                var rowsExistingAlready = {}
+                var countOfRowsInserted = {}
+                var countOfRowsExistingAlready = {}
+                var insertRow = function(tableName,row,callback){
+                    if(!rowsExistingAlready[tableName])rowsExistingAlready[tableName] = []
+                    if(!countOfRowsExistingAlready[tableName])countOfRowsExistingAlready[tableName] = 0
+                    if(!countOfRowsInserted[tableName])countOfRowsInserted[tableName] = 0
+                    var fieldsToCheck = ['ke']
+                    switch(tableName){
+                        case'API':
+                            fieldsToCheck = fieldsToCheck.concat([
+                                'code',
+                                'uid'
+                            ])
+                        break;
+                        case'Cloud Videos':
+                            fieldsToCheck = fieldsToCheck.concat([
+                                'href',
+                                'mid'
+                            ])
+                        break;
+                        case'Videos':
+                            fieldsToCheck = fieldsToCheck.concat([
+                                'time',
+                                'mid'
+                            ])
+                        break;
+                        case'Users':
+                            fieldsToCheck = fieldsToCheck.concat([
+                                'uid',
+                                'mail'
+                            ])
+                        break;
+                        case'Presets':
+                            fieldsToCheck = fieldsToCheck.concat([
+                                'name',
+                                'type'
+                            ])
+                        break;
+                        case'Logs':
+                            fieldsToCheck = fieldsToCheck.concat([
+                                'time',
+                                'info',
+                                'mid'
+                            ])
+                        break;
+                        case'Events':
+                            fieldsToCheck = fieldsToCheck.concat([
+                                'time',
+                                'details',
+                                'mid'
+                            ])
+                        break;
+                        case'Files':
+                            fieldsToCheck = fieldsToCheck.concat([
+                                'details',
+                                'name',
+                                'mid'
+                            ])
+                        break;
+                        case'Monitors':
+                            fieldsToCheck = fieldsToCheck.concat([
+                                'host',
+                                'protocol',
+                                'port',
+                                'path',
+                                'mid'
+                            ])
+                        break;
+                    }
+                    var keysToCheck = []
+                    var valuesToCheck = []
+                    fieldsToCheck.forEach(function(key){
+                        keysToCheck.push(key + '= ?')
+                        valuesToCheck.push(row[key])
+                    })
+                    s.sqlQuery('SELECT * FROM ' + tableName + ' WHERE ' + keysToCheck.join(' AND '),valuesToCheck,function(err,selected){
+                        if(selected && selected[0]){
+                            selected = selected[0]
+                            rowsExistingAlready[tableName].push(selected)
+                            callback()
+                        }else{
+                            var rowKeys = Object.keys(row)
+                            var insertEscapes = []
+                            var insertValues = []
+                            rowKeys.forEach(function(key){
+                                insertEscapes.push('?')
+                                insertValues.push(row[key])
+                            })
+                            s.sqlQuery('INSERT INTO ' + tableName + ' (' + rowKeys.join(',') +') VALUES (' + insertEscapes.join(',') + ')',insertValues,function(){
+                                if(!err){
+                                    ++countOfRowsInserted[tableName]
+                                }
+                                callback()
+                            })
+                        }
+                    })
+                }
+                var actionCount = {}
+                var insertTableRows = function(tableName,rows,callback){
+                    if(!actionCount[tableName])actionCount[tableName] = 0
+                    var insertLoop = function(){
+                        var row = rows[actionCount[tableName]]
+                        if(row){
+                            insertRow(tableName,row,function(){
+                                ++actionCount[tableName]
+                                insertLoop()
+                            })
+                        }else{
+                            callback()
+                        }
+                    }
+                    insertLoop()
+                }
+                var databaseTableKeys = Object.keys(database)
+                var completedTables = 0
+                var tableInsertLoop = function(callback){
+                    var tableName = databaseTableKeys[completedTables]
+                    var rows = database[databaseTableKeys[completedTables]]
+                    if(tableName){
+                        insertTableRows(tableName,rows,function(){
+                            ++completedTables
+                            tableInsertLoop(callback)
+                        })
+                    }else{
+                        callback()
+                    }
+                }
+                tableInsertLoop(function(){
+                    endData.ok = true
+                    endData.tablesInsertedTo = databaseTableKeys
+                    endData.countOfRowsInserted = countOfRowsInserted
+                    endData.rowsExistingAlready = rowsExistingAlready
+                    s.closeJsonResponse(res,endData)
+                })
+            }else{
+                endData.msg = lang['Database Not Found']
+                s.closeJsonResponse(res,endData)
+            }
+        },res,req)
+    })
+    /**
+    * API : Superuser : Force Check for Stale Purge Locks
+    */
+    app.all(config.webPaths.superApiPrefix+':auth/system/checkForStalePurgeLocks', function (req,res){
         s.superAuth(req.params,function(resp){
             var endData = {
                 ok : true
             }
-            var close = function(){
-                res.end(s.prettyPrint(endData))
-            }
-            var account = s.getPostData(req,'account')
-            s.sqlQuery('SELECT FROM Users',[],function(err,users){
-                s.sqlQuery('SELECT FROM Monitors',[],function(err,monitors){
-                    s.sqlQuery('SELECT FROM API',[],function(err,monitors){
-                        s.sqlQuery('SELECT FROM Videos',[],function(err,monitors){
-                            s.sqlQuery('SELECT FROM Logs',[],function(err,monitors){
-                                
-                            })
-                        })
-                    })
-                })
-            })
-            s.sqlQuery('DELETE FROM API WHERE uid=? AND ke=?',[account.uid,account.ke])
-            if(s.getPostData(req,'deleteSubAccounts',false) === '1'){
-                s.sqlQuery('DELETE FROM Users WHERE ke=?',[account.ke])
-            }
-            if(s.getPostData(req,'deleteMonitors',false) == '1'){
-                s.sqlQuery('SELECT * FROM Monitors WHERE ke=?',[account.ke],function(err,monitors){
-                    if(monitors && monitors[0]){
-                        monitors.forEach(function(monitor){
-                            s.camera('stop',monitor)
-                        })
-                        s.sqlQuery('DELETE FROM Monitors WHERE ke=?',[account.ke])
-                    }
-                })
-            }
-            if(s.getPostData(req,'deleteVideos',false) == '1'){
-                s.sqlQuery('DELETE FROM Videos WHERE ke=?',[account.ke])
-                fs.chmod(s.dir.videos+account.ke,0o777,function(err){
-                    fs.unlink(s.dir.videos+account.ke,function(err){})
-                })
-            }
-            if(s.getPostData(req,'deleteEvents',false) == '1'){
-                s.sqlQuery('DELETE FROM Events WHERE ke=?',[account.ke])
-            }
-            s.tx({f:'delete_account',ke:account.ke,uid:account.uid,mail:account.mail},'$')
-            close()
+            s.checkForStalePurgeLocks()
+            res.end(s.prettyPrint(endData))
         },res,req)
     })
 }