[push] Proxy server socket error handling, generally for FCM

plugins/push/api/api.js

@@ -98,7 +98,7 @@ plugins.register('/master/runners', runners => {
                 sender = undefined;
             }
             catch (e) {
-                log.e('Sender crached', e);
+                log.e('Sending stopped with an error', e);
                 sender = undefined;
             }
         }

plugins/push/api/proxy.js

@@ -37,6 +37,7 @@ function proxyAgent(url, proxy, agentConfig = {}) {
         //     socket.ref();
         // };
 
+        let rejects = [];
         this.createConnection = (options, callback) => {
             let connectOptions = {
                     method: 'CONNECT',
@@ -83,6 +84,49 @@ function proxyAgent(url, proxy, agentConfig = {}) {
                         // sock.removeAllListeners();
                         callback(err);
                     });
+                    socket.on('error', err => {
+                        err = err || new Error('ProxySocketError');
+                        log.e('socket errored, rejecting %d promises', rejects.length, err);
+                        rejects.forEach(rej => rej(err));
+
+                        try {
+                            sock.emit('timeout', err);
+                        }
+                        catch (e) {
+                            log.e('Failed to emit err on client socket', e);
+                        }
+                        try {
+                            sock.destroy(err);
+                        }
+                        catch (e) {
+                            log.e('Failed to destroy client socket on proxy err', e);
+                        }
+
+                        callback(err);
+
+                    });
+                    socket.on('timeout', err => {
+                        err = err || new Error('ProxyTimeout');
+                        log.e('socket timeouted, rejecting %d promises', rejects.length, err);
+                        rejects.forEach(rej => rej(err));
+
+                        try {
+                            sock.emit('timeout', err);
+                        }
+                        catch (e) {
+                            log.e('Failed to emit timeout on client socket', e);
+                        }
+                        try {
+                            sock.destroy(err);
+                        }
+                        catch (e) {
+                            log.e('Failed to destroy client socket on ProxyTimeout', e);
+                        }
+                        callback(err);
+                    });
+                    socket.on('end', err => {
+                        log.e('socket ended', err);
+                    });
                     callback(null, sock);
                     // callback(null, sock);
                     // Super.createConnection.apply(self, [{...options, socket}, callback]);
@@ -110,16 +154,31 @@ function proxyAgent(url, proxy, agentConfig = {}) {
             });
 
             connect.once('error', err => {
-                connect.removeAllListeners();
+                // connect.removeAllListeners();
                 callback(err);
             });
 
             connect.once('timeout', () => {
-                connect.destroy(new Error('Proxy timeout'));
+                connect.destroy(new Error('ProxyTimeout'));
             });
 
+            setTimeout(() => {
+                connect.destroy(new Error('ProxyTimeout'));
+            }, 50);
+
             connect.end();
         };
+
+        this.pushReject = rej => {
+            rejects.push(rej);
+        };
+
+        this.popReject = rej => {
+            let i = rejects.indexOf(rej);
+            if (i !== -1) {
+                rejects.splice(i, 1);
+            }
+        };
     };
 
     if (url.protocol === 'http:') {

plugins/push/api/send/platforms/utils/splitter.js

@@ -83,7 +83,12 @@ class Splitter extends Base {
                 res.on('data', d => {
                     res.reply += d;
                 });
-                res.on('end', () => handler(res));
+                res.on('end', () => {
+                    if (this.opts.agent && this.opts.agent.popReject) {
+                        this.opts.agent.popReject(reject);
+                    }
+                    handler(res);
+                });
                 // res.on('close', () => handler(res));
             });
             // req.on('socket', socket => {
@@ -93,6 +98,10 @@ class Splitter extends Base {
                 this.log.d('send request error', error);
                 reject([0, error]);
             });
+
+            if (this.opts.agent && this.opts.agent.pushReject) {
+                this.opts.agent.pushReject(reject);
+            }
             req.end(content);
         });
     }

plugins/push/frontend/public/localization/push.properties

@@ -475,6 +475,8 @@ push-notification.error-code.NoMessage.desc = The message was not found when sen
 push-notification.error-code.NoCredentials.desc = Push notification credentials were not found when sending the notification.
 push-notification.error-code.NoConnection.desc = Failed to connect to push notifications provider, please check APN / FCM / HPK are available from Countly host. In case you use proxy server, please check it's up and running.
 push-notification.error-code.NoProxyConnection.desc = Failed to connect to push notifications provider while using proxy server. Please check proxy server settings.
+push-notification.error-code.ProxyTimeout.desc = Connection to proxy server timed out while while sending push notifications. Please check proxy server availability.
+push-notification.error-code.ProxyError.desc = Connection to proxy server resulted in an error while while sending push notifications. Please check proxy server availability.
 push-notification.error-code.TooLateToSend.desc = Countly was unable to send these notifications in time (1 hour from scheduled date).
 push-notification.error-code.ExpiredCreds.desc = Push Notification credentials have probably expired. Please upload new credentials.
 push-notification.error-code.BadDeviceToken.desc = The push token received from your app by Countly Server was rejected by APNS as invalid. Please make sure you set `pushTestMode` property on the SDK's initial configuration correctly. Also please make sure provisioning profile (Development or Distribution) is valid, bundle ID is correct and entitlements are properly set.