merged in main; resolved merge conflicts

.github/workflows/prod-801-ci-cd.yaml

@@ -66,7 +66,7 @@ jobs:
         run: |
           export tag="${GITHUB_REF#refs/heads/}"
           export tag="${tag//\//-}.${GITHUB_SHA}"
-          sed -i -e "s/BASE_TAG/${tag}" ${{ github.workspace }}/deploy/prod-801/prod-801-install.sh
+          sed -i "s/BASE_TAG/${tag}/" ${{ github.workspace }}/deploy/prod-801/prod-801-install.sh
           ssh submit-t3desk 'bash -s' < ${{ github.workspace }}/deploy/prod-801/prod-801-install.sh
 
       # clean up secret files

.github/workflows/prod-ci-cd.yaml

@@ -99,7 +99,7 @@ jobs:
         run: |
           export tag="${GITHUB_REF#refs/heads/}"
           export tag="${tag//\//-}.${GITHUB_SHA}"
-          sed -i -e "s/BASE_TAG/${tag}" ${{ github.workspace }}/deploy/prod/prod-install.sh
+          sed -i "s/BASE_TAG/${tag}/" ${{ github.workspace }}/deploy/prod/prod-install.sh
           ssh submit-t3desk 'bash -s' < ${{ github.workspace }}/deploy/prod/prod-install.sh
 
       # clean up secret files

.github/workflows/prod-meta-ci-cd.yaml

@@ -32,7 +32,7 @@ jobs:
         run: |
           mkdir -p ${{ github.workspace }}/deploy/prod-meta/secrets/
           touch ${{ github.workspace }}/deploy/prod-meta/secrets/imap_user.txt
-          echo "${{ secrets.PROD_IMAP_USER }}" >> ${{ github.workspace }}/deploy/prod-meta/secrets/imap_user.txt
+          echo "${{ secrets.PROD_META_IMAP_USER }}" >> ${{ github.workspace }}/deploy/prod-meta/secrets/imap_user.txt
           chmod 400 ${{ github.workspace }}/deploy/prod-meta/secrets/imap_user.txt
           touch ${{ github.workspace }}/deploy/prod-meta/secrets/imap_pw.txt
           echo "${{ secrets.PROD_IMAP_PW }}" >> ${{ github.workspace }}/deploy/prod-meta/secrets/imap_pw.txt
@@ -78,7 +78,7 @@ jobs:
           export tag="${GITHUB_REF#refs/heads/}"
           export tag="${tag//\//-}.${GITHUB_SHA}"
           echo "TAG=${tag}" >> ${{ github.workspace }}/deploy/prod-meta/.env
-
+          
       # stop any existing docker compose that's running
       - name: Stop Docker Compose
         run: |
@@ -94,7 +94,7 @@ jobs:
         run: |
           export tag="${GITHUB_REF#refs/heads/}"
           export tag="${tag//\//-}.${GITHUB_SHA}"
-          sed -i -e "s/BASE_TAG/${tag}" ${{ github.workspace }}/deploy/prod-meta/prod-meta-install.sh
+          sed -i "s/BASE_TAG/${tag}/" ${{ github.workspace }}/deploy/prod-meta/prod-meta-install.sh
           ssh submit06 'bash -s' < ${{ github.workspace }}/deploy/prod-meta/prod-meta-install.sh
 
       # clean up secret files

.github/workflows/prod-root-ci-cd.yaml

@@ -66,7 +66,7 @@ jobs:
         run: |
           export tag="${GITHUB_REF#refs/heads/}"
           export tag="${tag//\//-}.${GITHUB_SHA}"
-          sed -i -e "s/BASE_TAG/${tag}" ${{ github.workspace }}/deploy/prod-root/prod-root-install.sh
+          sed -i "s/BASE_TAG/${tag}/" ${{ github.workspace }}/deploy/prod-root/prod-root-install.sh
           ssh submit06 'bash -s' < ${{ github.workspace }}/deploy/prod-root/prod-root-install.sh
 
       # clean up secret files

A2rchi/interfaces/chat_app/app.py

@@ -5,11 +5,29 @@
 from A2rchi.utils.sql import SQL_INSERT_CONVO, SQL_INSERT_FEEDBACK, SQL_QUERY_CONVO
 
 from datetime import datetime
+from pygments import highlight
+from pygments.lexers import (
+    BashLexer,
+    PythonLexer,
+    JavaLexer,
+    JavascriptLexer,
+    CppLexer,
+    CLexer,
+    TypeScriptLexer,
+    HtmlLexer,
+    FortranLexer,
+    JuliaLexer,
+    MathematicaLexer,
+    MatlabLexer
+)
+from pygments.formatters import HtmlFormatter
+
 from flask import request, jsonify, render_template
 from flask_cors import CORS
 from threading import Lock
-from typing import Optional, List, Tuple
+from typing import List
 
+import mistune as mt
 import numpy as np
 
 import os
@@ -21,6 +39,51 @@
 QUERY_LIMIT = 1000 # max number of queries per conversation
 
 
+class AnswerRenderer(mt.HTMLRenderer):
+    """
+    Class for custom rendering of A2rchi output. Child of mistune's HTMLRenderer, with custom overrides.
+    Code blocks are structured and colored according to pygment lexers
+    """
+    RENDERING_LEXER_MAPPING = {
+            "python": PythonLexer,
+            "java": JavaLexer,
+            "javascript": JavascriptLexer,
+            "bash": BashLexer,
+            "c++": CppLexer,
+            "cpp": CppLexer,
+            "c": CLexer,
+            "typescript": TypeScriptLexer,
+            "html": HtmlLexer,
+            "Fortran" : FortranLexer,
+            "Julia" : JuliaLexer,
+            "Mathematica" : MathematicaLexer,
+            "MATLAB": MatlabLexer
+        }
+
+    def __init__(self):
+        super().__init__()
+
+    def block_text(self,text):
+         #Handle blocks of text (the negatives of blocks of code) and sets them in paragraphs
+         return f"""<p>{text}</p>"""
+
+    def block_code(self, code, info=None):
+        # Handle code blocks (triple backticks)
+        if info not in self.RENDERING_LEXER_MAPPING.keys(): info = 'bash' #defaults in bash
+        code_block_highlighted = highlight(code.strip(), self.RENDERING_LEXER_MAPPING[info](stripall=True), HtmlFormatter())
+        return f"""<div class="code-box">
+                <div class="code-box-header"> 
+                <span>{info}</span> <button class="copy-code-btn" onclick="copyCode(this)"> Copy Code </button>
+                </div>
+                <div class="code-box-body">{code_block_highlighted}
+                </div>
+                </div>"""
+
+    def codespan(self, text):
+        # Handle inline code snippets (single backticks)
+        return f"""<code class="code-snippet">{text}</code>"""
+
+
 class ChatWrapper:
     """
     Wrapper which holds functionality for the chatbot
@@ -60,6 +123,21 @@ def convert_to_app_history(history):
         the author of the text and the second entry is the text itself 
         """
         return [list(entry) for entry in history]
+
+
+    @staticmethod
+    def format_code_in_text(text):
+        """
+        Takes in input plain text (the output from A2rchi); 
+        Recognizes structures in canonical Markdown format, and processes according to the custom renderer; 
+        Returns it formatted in HTML 
+        """
+        markdown = mt.create_markdown(renderer=AnswerRenderer())
+        try:
+            return markdown(text)
+        except: 
+             print("Rendering error: markdown formatting failed")
+             return text
 
 
     def insert_feedback(self, feedback):
@@ -214,9 +292,9 @@ def __call__(self, message: List[str], conversation_id: int, is_refresh: bool, m
             embedding_name = self.config["utils"]["embeddings"]["EMBEDDING_NAME"]
             similarity_score_reference = self.config["utils"]["embeddings"]["EMBEDDING_CLASS_MAP"][embedding_name]["similarity_score_reference"]
             if score < similarity_score_reference and source in sources.keys(): 
-                output = "<p>" + result["answer"] + "</p>" + "\n\n<br /><br /><p><a href= " + sources[source] + ">Click here to read more</a></p>"
+                output = "<p>" + self.format_code_in_text(result["answer"]) + "</p>" + "\n\n<br /><br /><p><a href= " + sources[source] + ">Click here to read more</a></p>"
             else:
-                output = "<p>" + result["answer"] + "</p>"
+                output = "<p>" + self.format_code_in_text(result["answer"]) + "</p>"
 
             # write user message and A2rchi response to database
             user_message = (sender, content, msg_ts)

A2rchi/interfaces/chat_app/static/script.js-template

@@ -107,6 +107,12 @@ const getChatResponse = async (incomingChatDiv, isRefresh=false) => {
     setTimeout(showFeedbackRequest, 500);
 }
 
+const copyCode = (copyCodeBtn) => {
+    // Copy the text content of the response to the clipboard
+    const reponseTextElement = copyCodeBtn.parentElement.parentElement.querySelector(".code-box-body");
+    navigator.clipboard.writeText(reponseTextElement.innerText);
+}
+
 const copyResponse = (copyBtn) => {
     // Copy the text content of the response to the clipboard
     const reponseTextElement = copyBtn.parentElement.previousElementSibling.querySelector("p");

A2rchi/interfaces/chat_app/static/style.css

@@ -4,7 +4,6 @@
   margin: 0;
   padding: 0;
   box-sizing: border-box;
-  font-family: "Poppins", sans-serif;
 }
 :root {
   --text-color: #FFFFFF;
@@ -15,6 +14,7 @@
   --incoming-chat-bg: #444654;
   --outgoing-chat-border: #343541;
   --incoming-chat-border: #444654;
+  --incoming-chat-code-snippet: #545764;
 }
 .light-mode {
   --text-color: #343541;
@@ -25,9 +25,19 @@
   --incoming-chat-bg: #F7F7F8;
   --outgoing-chat-border: #FFFFFF;
   --incoming-chat-border: #D9D9E3;
+  --incoming-chat-code-snippet: #FFFFFF;
 }
 body {
   background: var(--outgoing-chat-bg);
+  font-family: "Poppins", sans-serif;
+}
+
+ul {
+  margin: 0px 3.5rem;
+}
+
+ol {
+  margin: 1rem 3.5rem;
 }
 
 /* Pop up container styling */
@@ -134,12 +144,6 @@ span.material-symbols-rounded {
   user-select: none;
   cursor: pointer;
 }
-.chat .chat-content span {
-  cursor: pointer;
-  font-size: 1.3rem;
-  color: var(--icon-color);
-  visibility: hidden;
-}
 .chat:hover .chat-content:not(:has(.typing-animation), :has(.error)) span {
   visibility: visible;
 }
@@ -228,6 +232,7 @@ span.material-symbols-rounded {
   overflow-y: auto;
   background: var(--incoming-chat-bg);
   outline: 1px solid var(--incoming-chat-border);
+  font-family:  "Poppins", sans-serif;
 }
 .typing-textarea textarea::placeholder {
   color: var(--placeholder-color);
@@ -324,4 +329,149 @@ span.material-symbols-rounded {
   span.material-symbols-rounded {
     font-size: 1.25rem!important;
   }
+}
+
+
+/* Code rendering styling */
+/* pre { line-height: 125%; }
+td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
+span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; }
+td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; }
+span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } */
+.highlight .hll { background-color: #ffffcc }
+.highlight { 
+  font-family: Consolas,Monaco,Lucida Console,Liberation Mono,DejaVu Sans Mono,Bitstream Vera Sans Mono,Courier New;
+  font-size: 1.2em;
+}
+.highlight .c { color: #3D7B7B; font-style: italic } /* Comment */
+.highlight .err { border: 1px solid #FF0000 } /* Error */
+.highlight .k { color: #008000; font-weight: bold } /* Keyword */
+.highlight .o { color: #666666 } /* Operator */
+.highlight .ch { color: #3D7B7B; font-style: italic } /* Comment.Hashbang */
+.highlight .cm { color: #3D7B7B; font-style: italic } /* Comment.Multiline */
+.highlight .cp { color: #9C6500 } /* Comment.Preproc */
+.highlight .cpf { color: #3D7B7B; font-style: italic } /* Comment.PreprocFile */
+.highlight .c1 { color: #3D7B7B; font-style: italic } /* Comment.Single */
+.highlight .cs { color: #3D7B7B; font-style: italic } /* Comment.Special */
+.highlight .gd { color: #A00000 } /* Generic.Deleted */
+.highlight .ge { font-style: italic } /* Generic.Emph */
+.highlight .ges { font-weight: bold; font-style: italic } /* Generic.EmphStrong */
+.highlight .gr { color: #E40000 } /* Generic.Error */
+.highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */
+.highlight .gi { color: #008400 } /* Generic.Inserted */
+.highlight .go { color: #717171 } /* Generic.Output */
+.highlight .gp { color: #000080; font-weight: bold } /* Generic.Prompt */
+.highlight .gs { font-weight: bold } /* Generic.Strong */
+.highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */
+.highlight .gt { color: #0044DD } /* Generic.Traceback */
+.highlight .kc { color: #008000; font-weight: bold } /* Keyword.Constant */
+.highlight .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */
+.highlight .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */
+.highlight .kp { color: #008000 } /* Keyword.Pseudo */
+.highlight .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */
+.highlight .kt { color: #B00040 } /* Keyword.Type */
+.highlight .m { color: #666666 } /* Literal.Number */
+.highlight .s { color: #BA2121 } /* Literal.String */
+.highlight .na { color: #687822 } /* Name.Attribute */
+.highlight .nb { color: #008000 } /* Name.Builtin */
+.highlight .nc { color: #0000FF; font-weight: bold } /* Name.Class */
+.highlight .no { color: #880000 } /* Name.Constant */
+.highlight .nd { color: #AA22FF } /* Name.Decorator */
+.highlight .ni { color: #717171; font-weight: bold } /* Name.Entity */
+.highlight .ne { color: #CB3F38; font-weight: bold } /* Name.Exception */
+.highlight .nf { color: #0000FF } /* Name.Function */
+.highlight .nl { color: #767600 } /* Name.Label */
+.highlight .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */
+.highlight .nt { color: #008000; font-weight: bold } /* Name.Tag */
+.highlight .nv { color: #19177C } /* Name.Variable */
+.highlight .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */
+.highlight .w { color: #bbbbbb } /* Text.Whitespace */
+.highlight .mb { color: #666666 } /* Literal.Number.Bin */
+.highlight .mf { color: #666666 } /* Literal.Number.Float */
+.highlight .mh { color: #666666 } /* Literal.Number.Hex */
+.highlight .mi { color: #666666 } /* Literal.Number.Integer */
+.highlight .mo { color: #666666 } /* Literal.Number.Oct */
+.highlight .sa { color: #BA2121 } /* Literal.String.Affix */
+.highlight .sb { color: #BA2121 } /* Literal.String.Backtick */
+.highlight .sc { color: #BA2121 } /* Literal.String.Char */
+.highlight .dl { color: #BA2121 } /* Literal.String.Delimiter */
+.highlight .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */
+.highlight .s2 { color: #BA2121 } /* Literal.String.Double */
+.highlight .se { color: #AA5D1F; font-weight: bold } /* Literal.String.Escape */
+.highlight .sh { color: #BA2121 } /* Literal.String.Heredoc */
+.highlight .si { color: #A45A77; font-weight: bold } /* Literal.String.Interpol */
+.highlight .sx { color: #008000 } /* Literal.String.Other */
+.highlight .sr { color: #A45A77 } /* Literal.String.Regex */
+.highlight .s1 { color: #BA2121 } /* Literal.String.Single */
+.highlight .ss { color: #19177C } /* Literal.String.Symbol */
+.highlight .bp { color: #008000 } /* Name.Builtin.Pseudo */
+.highlight .fm { color: #0000FF } /* Name.Function.Magic */
+.highlight .vc { color: #19177C } /* Name.Variable.Class */
+.highlight .vg { color: #19177C } /* Name.Variable.Global */
+.highlight .vi { color: #19177C } /* Name.Variable.Instance */
+.highlight .vm { color: #19177C } /* Name.Variable.Magic */
+.highlight .il { color: #666666 } /* Literal.Number.Integer.Long */
+
+.code-box {
+	background: #e8e8e8;
+    border-radius: .375rem;
+    color: black;
+    margin-top: 1.5rem;
+    margin-bottom: 1.5rem;
+    margin-left: 2rem;
+    margin-right: 2rem;
+
+}
+.code-box-header {
+	background: #c8c8c8;
+    margin-bottom: 1rem;
+    border-top-left-radius: .375rem;
+	border-top-right-radius: .375rem;
+    padding-bottom: .5rem;
+	padding-top: .5rem;
+    padding-left: 1rem;
+	padding-right: 1rem;
+    display: flex;
+    align-items: center;
+}
+
+.code-box-body {
+    margin-bottom: 1rem;
+    padding-bottom: .5rem;
+    margin-top: -1rem;
+	padding-top: .5rem;
+    padding-left: 1rem;
+	padding-right: 1rem;
+    overflow-y: auto;
+}
+
+.copy-code-btn {
+	cursor: pointer;
+    background-color: transparent;
+    border: transparent;
+    display: flex;
+    align-items: center;
+    margin-left: auto;
+    color: rgb(51, 51, 51);
+}
+
+.code-snippet {
+  font-weight: bold;
+  font-family: monospace;
+  background-color: var(--incoming-chat-code-snippet);
+  padding: 2px 4px; /* Add padding for spacing around the code */
+  border: 1px solid #d1d1d1; /* Add a border for a visual separation */
+  border-radius: 4px; /* Rounded corners for aesthetics */
+}
+
+.highlight .pre {
+  background-color: transparent;
+  border-radius: .375rem;
+  color: currentColor;
+  font-size: .875em;
+  font-weight: 400;
+  line-height: 1.7142857;
+  margin: 0;
+  overflow-x: auto;
+  padding: 0;
 }