Merge remote-tracking branch 'origin/candidate-8.10.x' into candidate…

…-8.12.x

Signed-off-by: Gavin Halliday <[email protected]>

.gitattributes

@@ -10,3 +10,4 @@
 *.psd           -text
 *.doc           -text
 *.mk            -text
+ecl/regress/crmultiline*.ecl eol=crlf

common/thorhelper/thorsoapcall.cpp

@@ -821,7 +821,8 @@ class CWSCHelper : implements IWSCHelper, public CInterface
     Linked<ClientCertificate> clientCert;
 
     static CriticalSection secureContextCrit;
-    static Owned<ISecureSocketContext> secureContext;
+    static Owned<ISecureSocketContext> tlsSecureContext;
+    static Owned<ISecureSocketContext> localMtlsSecureContext;
 
     Owned<ISecureSocketContext> customSecureContext;
 
@@ -1138,7 +1139,9 @@ class CWSCHelper : implements IWSCHelper, public CInterface
     ISecureSocketContext *ensureStaticSecureContext()
     {
         CriticalBlock b(secureContextCrit);
-        return ensureSecureContext(secureContext);
+        if (localClientCert)
+            return ensureSecureContext(localMtlsSecureContext);
+        return ensureSecureContext(tlsSecureContext);
     }
     ISecureSocket *createSecureSocket(ISocket *sock, const char *fqdn = nullptr)
     {
@@ -1266,7 +1269,9 @@ class CWSCHelper : implements IWSCHelper, public CInterface
 };
 
 CriticalSection CWSCHelper::secureContextCrit;
-Owned<ISecureSocketContext> CWSCHelper::secureContext; // created on first use
+Owned<ISecureSocketContext> CWSCHelper::tlsSecureContext; // created on first use
+Owned<ISecureSocketContext> CWSCHelper::localMtlsSecureContext; // created on first use
+
 
 //=================================================================================================
 

ecl/hql/hqlgram.hpp

@@ -1223,6 +1223,8 @@ class HqlLex
 
     private:
         static void doEnterEmbeddedMode(yyscan_t yyscanner);
+        void stripSlashNewline(attribute & returnToken, StringBuffer & target, size_t len, const char * data);
+
         void declareXmlSymbol(const attribute & errpos, const char *name);
         bool lookupXmlSymbol(const attribute & errpos, const char *name, StringBuffer &value);
         void setXmlSymbol(const attribute & errpos, const char *name, const char *value, bool append);

ecl/hql/hqllex.l

@@ -1659,8 +1659,8 @@ FUNCTIONMACRO|MACRO {
                             int len = endpos-startpos;
                             Owned<IValue> unicodeValue;
                             // Special handling required for trailing \ char which suppresses the following linefeed, as unicode unescape does not handle it
-                            StringBuffer source(len, lexer->yyBuffer+startpos);
-                            source.replaceString("\\\n","");
+                            StringBuffer source;
+                            lexer->stripSlashNewline(returnToken, source, len, lexer->yyBuffer+startpos);
                             if (isUtf8)
                             {
                                 size32_t chars = rtlUtf8Length(source.length(), source.str());

ecl/hql/hqlparse.cpp

@@ -1997,6 +1997,14 @@ int HqlLex::processStringLiteral(attribute & returnToken, char *CUR_TOKEN_TEXT,
                 finger++;
                 continue;  // A \ at end of line in a multiline constant means remove the end-of-line
             }
+            else if (next == '\r')
+            {
+                if (finger[2] == '\n')
+                    finger += 2;
+                else
+                    finger++;
+                continue;  // A \ at end of line in a multiline constant means remove the end-of-line
+            }
             else if (next == 'a')
             {
                 next = '\a';
@@ -2076,6 +2084,14 @@ int HqlLex::processStringLiteral(attribute & returnToken, char *CUR_TOKEN_TEXT,
             }
             *bf++ = next;
         }
+        else if (next == '\r')
+        {
+            //Convert \r\n to \n and \r to \n
+            next = finger[1];
+            if (next == '\n')
+                finger++;
+            *bf++ = '\n';
+        }
         else if (next == '\'' && !isMultiline)
         {
             returnToken.setPosition(yyLineNo, oldColumn+delta, oldPosition+delta, querySourcePath());
@@ -2146,6 +2162,50 @@ int HqlLex::processStringLiteral(attribute & returnToken, char *CUR_TOKEN_TEXT,
     throwUnexpected();
 }
 
+
+void HqlLex::stripSlashNewline(attribute & returnToken, StringBuffer & target, size_t len, const char * text)
+{
+    target.ensureCapacity(len);
+    for (size_t i = 0; i < len; i++)
+    {
+        char cur = text[i];
+        if (cur == '\\')
+        {
+            if (i+1 < len)
+            {
+                byte next = text[i+1];
+                if (next == '\n')
+                {
+                    i++;
+                    continue;
+                }
+                if (next == '\r')
+                {
+                    i++;
+                    if ((i+1) < len && text[i+1] == '\n')
+                        i++;
+                    continue;
+                }
+            }
+            else
+            {
+                StringBuffer msg("Cannot terminate a string with escape char '\\': ");
+                msg.append(len, text);
+                reportError(returnToken, RRR_ESCAPE_ENDWITHSLASH, "%s", msg.str());
+            }
+        }
+        else if (cur == '\r')
+        {
+            //Normalize unusual end of line sequences
+            if ((i+1 < len) && (text[i+1] == '\n'))
+                i++;
+            cur = '\n';
+        }
+
+        target.append(cur);
+    }
+}
+
 //====================================== Error Reporting  ======================================
 
 bool HqlLex::checkAborting()

ecl/regress/crmultiline.ecl

@@ -0,0 +1,23 @@
+
+x := '''\
+   one
+   two
+   three''';
+
+y := u'''\
+   one
+   two
+   three''';
+
+output(length('''
+'''));
+output(length(u'''
+'''));
+output(length('''\
+'''));
+output(length(u'''\
+'''));
+output('!'+x+'!');
+output(u'!'+y+u'!');
+output((DATA)('!'+x+'!'));
+output((DATA)(u'!'+y+u'!'));

ecl/regress/crmultiline_err.ecl

@@ -0,0 +1,2 @@
+z1 := '''\''';
+z2 := u'''\''';

esp/bindings/http/platform/httpservice.cpp

@@ -1521,7 +1521,14 @@ EspAuthState CEspHttpServer::checkUserAuthPerSession(EspAuthRequest& authReq, St
     const char* userName = (authReq.requestParams) ? authReq.requestParams->queryProp("username") : NULL;
     const char* password = (authReq.requestParams) ? authReq.requestParams->queryProp("password") : NULL;
     if (!isEmptyString(userName) && !isEmptyString(password))
-        return authNewSession(authReq, userName, password, urlCookie.isEmpty() ? "/" : urlCookie.str(), unlock);
+    {
+        const char *url = "/";
+        //if the cookie came from us it would be encoded, but check for newline just in case the cookie was injected somehow
+        //  unescaped newlines can be made to look like nefarious http headers
+        if (!urlCookie.isEmpty() && !strchr(urlCookie, '\n'))
+            url = urlCookie.str();
+        return authNewSession(authReq, userName, password, url, unlock);
+    }
 
     authReq.ctx->setAuthStatus(AUTH_STATUS_FAIL);
     if (unlock)
@@ -2195,6 +2202,7 @@ EspAuthState CEspHttpServer::handleAuthFailed(bool sessionAuth, EspAuthRequest&
 
 void CEspHttpServer::askUserLogin(EspAuthRequest& authReq, const char* msg)
 {
+    StringBuffer encoded;
     StringBuffer urlCookie;
     readCookie(SESSION_START_URL_COOKIE, urlCookie);
     if (urlCookie.isEmpty())
@@ -2205,18 +2213,18 @@ void CEspHttpServer::askUserLogin(EspAuthRequest& authReq, const char* msg)
         if (sessionStartURL.isEmpty() || streq(sessionStartURL.str(), "/WsSMC/") || strieq(sessionStartURL, authReq.authBinding->queryLoginURL()))
             sessionStartURL.set("/");
 
-        addCookie(SESSION_START_URL_COOKIE, sessionStartURL.str(), 0, true); //time out when browser is closed
+        addCookie(SESSION_START_URL_COOKIE, encodeURL(encoded.clear(), sessionStartURL.str()), 0, true); //time out when browser is closed
     }
     else if (changeRedirectURL(authReq))
     {
         StringBuffer sessionStartURL(authReq.httpPath);
         if (authReq.requestParams && authReq.requestParams->hasProp("__querystring"))
             sessionStartURL.append("?").append(authReq.requestParams->queryProp("__querystring"));
-        addCookie(SESSION_START_URL_COOKIE, sessionStartURL.str(), 0, true); //time out when browser is closed
+        addCookie(SESSION_START_URL_COOKIE, encodeURL(encoded.clear(), sessionStartURL.str()), 0, true); //time out when browser is closed
     }
     if (!isEmptyString(msg))
         addCookie(SESSION_AUTH_MSG_COOKIE, msg, 0, false); //time out when browser is closed
-    m_response->redirect(*m_request, authReq.authBinding->queryLoginURL());
+    m_response->redirect(*m_request, encodeURL(encoded.clear(), authReq.authBinding->queryLoginURL()));
 }
 
 bool CEspHttpServer::changeRedirectURL(EspAuthRequest& authReq)

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -239,15 +239,22 @@ void WsWuInfo::readWorkunitComponentLogs(const char* outFile, unsigned maxLogRec
     else if (logFormat == LOGACCESS_LOGFORMAT_xml)
         writeStringToStream(*outIOS, "<lines>");
 
-    while (logReader->readLogEntries(logcontent.clear(), recsRead))
+    bool moreToRead = false;
+    do
     {
-        if (logFormat == LOGACCESS_LOGFORMAT_json && totalRecsRead > 0 && recsRead > 0)
+        moreToRead = logReader->readLogEntries(logcontent.clear(), recsRead);
+
+        if (recsRead > 0)
         {
-            writeCharToStream(*outIOS, ',');
+            if (logFormat == LOGACCESS_LOGFORMAT_json && totalRecsRead > 0)
+            {
+                writeCharToStream(*outIOS, ',');
+            }
+            totalRecsRead += recsRead;
+            writeStringToStream(*outIOS, logcontent.str());
         }
-        totalRecsRead += recsRead;
-        writeStringToStream(*outIOS, logcontent.str());
     }
+    while (moreToRead == true);
 
     if (totalRecsRead == maxLogRecords) //Warn of possible truncation
     {
@@ -400,23 +407,31 @@ unsigned WsWuInfo::sendComponentLogContent(IEspContext* context, IRemoteLogAcces
     StringBuffer logContent;
     unsigned totalRecsRead = 0;
     unsigned recsRead = 0;
-    while (logReader->readLogEntries(logContent.clear(), recsRead))
+    bool moreToRead = false;
+    do
     {
-        if (options.logFormat == LOGACCESS_LOGFORMAT_json && totalRecsRead > 0 && recsRead > 0)
+        moreToRead = logReader->readLogEntries(logContent.clear(), recsRead);
+
+        if (recsRead > 0)
         {
-            StringBuffer s(',');
-            flusher->flushXML(s, false);
-        }
+            if (options.logFormat == LOGACCESS_LOGFORMAT_json && totalRecsRead > 0)
+            {
+                StringBuffer s(',');
+                flusher->flushXML(s, false);
+            }
 
-        totalRecsRead += recsRead;
-        flusher->flushXML(logContent, false);
+            totalRecsRead += recsRead;
+            flusher->flushXML(logContent, false);
+        }
 
         if (abortCallback.abortRequested())
         {
             LOG(MCdebugInfo, unknownJob, "WsWuInfo::sendComponentLogContent(): abort requested via callback");
             break;
         }
     }
+    while (moreToRead == true);
+
     return totalRecsRead;
 }