adding services for scraper and mailbox; minor tweaks

.github/workflows/ci-cd.yaml

@@ -26,6 +26,12 @@ jobs:
       # create secrets files for docker-compose
       - name: Create Secrets Files
         run: |
+          touch ${{ github.workspace }}/deploy/imap_user.txt
+          echo "${{ secrets.DEV_IMAP_USER }}" >> ${{ github.workspace }}/deploy/imap_user.txt
+          chmod 400 ${{ github.workspace }}/deploy/imap_user.txt
+          touch ${{ github.workspace }}/deploy/imap_pw.txt
+          echo "${{ secrets.DEV_IMAP_PW }}" >> ${{ github.workspace }}/deploy/imap_pw.txt
+          chmod 400 ${{ github.workspace }}/deploy/imap_pw.txt
           touch ${{ github.workspace }}/deploy/cleo_url.txt
           echo "${{ secrets.DEV_CLEO_URL }}" >> ${{ github.workspace }}/deploy/cleo_url.txt
           chmod 400 ${{ github.workspace }}/deploy/cleo_url.txt
@@ -38,6 +44,21 @@ jobs:
           touch ${{ github.workspace }}/deploy/cleo_project.txt
           echo "${{ secrets.DEV_CLEO_PROJECT }}" >> ${{ github.workspace }}/deploy/cleo_project.txt
           chmod 400 ${{ github.workspace }}/deploy/cleo_project.txt
+          touch ${{ github.workspace }}/deploy/sender_server.txt
+          echo "${{ secrets.DEV_SENDER_SERVER }}" >> ${{ github.workspace }}/deploy/sender_server.txt
+          chmod 400 ${{ github.workspace }}/deploy/sender_server.txt
+          touch ${{ github.workspace }}/deploy/sender_port.txt
+          echo "${{ secrets.DEV_SENDER_PORT }}" >> ${{ github.workspace }}/deploy/sender_port.txt
+          chmod 400 ${{ github.workspace }}/deploy/sender_port.txt
+          touch ${{ github.workspace }}/deploy/sender_replyto.txt
+          echo "${{ secrets.DEV_SENDER_REPLYTO }}" >> ${{ github.workspace }}/deploy/sender_replyto.txt
+          chmod 400 ${{ github.workspace }}/deploy/sender_replyto.txt
+          touch ${{ github.workspace }}/deploy/sender_user.txt
+          echo "${{ secrets.DEV_SENDER_USER }}" >> ${{ github.workspace }}/deploy/sender_user.txt
+          chmod 400 ${{ github.workspace }}/deploy/sender_user.txt
+          touch ${{ github.workspace }}/deploy/sender_pw.txt
+          echo "${{ secrets.DEV_SENDER_PW }}" >> ${{ github.workspace }}/deploy/sender_pw.txt
+          chmod 400 ${{ github.workspace }}/deploy/sender_pw.txt
           touch ${{ github.workspace }}/deploy/openai_api_key.txt
           echo "${{ secrets.OPENAI_API_KEY }}" >> ${{ github.workspace }}/deploy/openai_api_key.txt
           chmod 400 ${{ github.workspace }}/deploy/openai_api_key.txt
@@ -60,10 +81,9 @@ jobs:
       # clean up secret files
       - name: Remove Secrets from Runner
         run: |
-          rm ${{ github.workspace }}/deploy/cleo_url.txt
-          rm ${{ github.workspace }}/deploy/cleo_user.txt
-          rm ${{ github.workspace }}/deploy/cleo_pw.txt
-          rm ${{ github.workspace }}/deploy/cleo_project.txt
+          rm ${{ github.workspace }}/deploy/cleo_*.txt
+          rm ${{ github.workspace }}/deploy/imap_*.txt
+          rm ${{ github.workspace }}/deploy/sender_*.txt
           rm ${{ github.workspace }}/deploy/openai_api_key.txt
 
       # print job status

A2rchi/bin/service_mailbox.py

@@ -1,8 +1,10 @@
 #!/bin/python
-import os
 from A2rchi.interfaces import cleo
 from A2rchi.utils import mailbox
 
+import os
+
+print("Starting Mailbox Service")
 cleo = cleo.Cleo('Cleo_Helpdesk')
 
 while True:

A2rchi/bin/service_scraper.py

@@ -1,6 +1,8 @@
-import os
 from A2rchi.utils.scraper import Scraper
 
+import os
+
+print("Starting Scraper Service")
 scraper=Scraper()
 
 while True:

A2rchi/utils/mailbox.py

@@ -1,9 +1,10 @@
 #!/bin/python
-import os,sys
-import getpass, imaplib, email
-
 from A2rchi.utils.config_loader import Config_Loader
-config = Config_Loader().config["utils"]["mailbox"]
+from A2rchi.utils.env import read_secret
+
+import email
+import imaplib
+
 
 class Mailbox:
     'A class to describe the mailbox usage.'
@@ -13,8 +14,12 @@ def __init__(self):
         The mailbox (should be a singleton).
         """
         self.mailbox = None
+        self.user = read_secret('IMAP_USER')
+        self.password = read_secret('IMAP_PW')
+        self.config = Config_Loader().config["utils"]["mailbox"]
+
         # make sure to open the mailbox
-        if self._verify:
+        if self._verify():
             self.mailbox = self._connect()
 
     def find_issue_id(self,description):
@@ -27,7 +32,7 @@ def find_issue_id(self,description):
             issue_id = int(description[index+9:].split()[0])
         return issue_id
 
-    def process_messages(self,cleo):
+    def process_messages(self, cleo):
         """
         Select all messages in the mailbx and process them.
         """
@@ -105,9 +110,9 @@ def _get_email_body(self,msg):
             try:
                 body = body.decode(charset)
             except UnicodeDecodeError:
-                handle_error("UnicodeDecodeError: encountered.",msg,charset)
+                self._handle_error("UnicodeDecodeError: encountered.",msg,charset)
             except AttributeError:
-                handle_error("AttributeError: encountered" ,msg,charset)
+                self._handle_error("AttributeError: encountered" ,msg,charset)
 
         return body, body_html
 
@@ -131,25 +136,27 @@ def _connect(self):
         """
         Open the mailbox
         """
-        print(f" Open mailbox (U:{os.getenv('IMAP_USER')} P:*********)")
-        mailbox = imaplib.IMAP4(host='ppc.mit.edu', port=config["IMAP4_PORT"], timeout=None)
-        mailbox.login(os.getenv('IMAP_USER'),os.getenv('IMAP_PW'))
+        print(f" Open mailbox (U:{self.user} P:*********)")
+        mailbox = imaplib.IMAP4(host='ppc.mit.edu', port=self.config["IMAP4_PORT"], timeout=None)
+        mailbox.login(self.user, self.password)
         return mailbox
 
-    def _handle_error(self,errmsg, emailmsg, cs):
+    def _handle_error(self, errmsg, emailmsg, cs):
         print()
         print(errmsg)
         print("This error occurred while decoding with ",cs," charset.")
         print("These charsets were found in this email.",self._get_charsets(emailmsg))
         print("This is the subject:",emailmsg['subject'])
         print("This is the sender:",emailmsg['From'])
+
         return
 
     def _verify(self):
         """
         Make sure the environment is setup
         """
-        if os.getenv('IMAP_USER') == None or os.getenv('IMAP_PW') == None:
+        if self.user == None or self.password == None:
             print(" Did not find all cleo configs: IMAP_USER, IMAP_PW (source ~/.imap).")
             return False
+
         return True

A2rchi/utils/scraper.py

@@ -22,7 +22,6 @@ class Scraper():
 
     def __init__(self):
         from A2rchi.utils.config_loader import Config_Loader
-        config = Config_Loader().config["chains"]["chain"]
         global_config = Config_Loader().config["global"]
 
         #Check if target folders exist 
@@ -47,7 +46,7 @@ def hard_scrape(self,verbose=False):
         
         """
         self.scrape_submit_files()
-        self.scrape_rst_files(self.github_url,self.raw_url)
+        self.scrape_rst_files(self.github_url, self.raw_url)
         if verbose: print("Scraping was completed successfully")
 
     def scrape_submit_files(self):
@@ -86,11 +85,3 @@ def scrape_rst_files(self,url,raw_url):
                     print(f"Error downloading {file_url}: {file_response.status_code}")
         else:
             print(f"Error: {response.status_code}")
-
-
-
-# Example usage
-s = Scraper()
-url,raw_url = s.github_url,s.raw_url
-# s.scrape_rst_files(url,raw_url)
-# s.scrape_all()

A2rchi/utils/sender.py

@@ -1,45 +1,51 @@
-import os,sys
-import smtplib
+from A2rchi.utils.env import read_secret
+
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 
+import os
+import smtplib
+
+
 class Sender:
-    'A class to send emails in an uncomplicated fashion.'
+    """A class to send emails in an uncomplicated fashion."""
 
     def __init__(self):
         """
         Give it a name and generate a conncetion to the database (should be a singleton).
         """
+        self.server_name = read_secret('SENDER_SERVER')
+        self.port = read_secret('SENDER_PORT')
+        self.user = read_secret('SENDER_USER')
+        self.password = read_secret('SENDER_PW')
+        self.reply_to = read_secret('SENDER_REPLYTO')
 
-        print(f" Open smtp (SERVER:{os.getenv('SENDER_SERVER')} PORT:{os.getenv('SENDER_PORT')} U:{os.getenv('SENDER_USER')} P:*********)")
+        print(f" Open smtp (SERVER:{self.server_name} PORT:{self.port} U:{self.user} P:*********)")
 
-        self.server_name = os.getenv('SENDER_SERVER')
-        self.user = os.getenv('SENDER_USER')
-        self.password = os.getenv('SENDER_PW')
 
-    def send_message(self,to,cc,subject,body):
+    def send_message(self, to, cc, subject, body):
 
-        #start and login to SMTP server 
-        self.server = smtplib.SMTP(self.server_name,os.getenv('SENDER_PORT'))
+        #start and login to SMTP server
+        self.server = smtplib.SMTP(self.server_name, self.port)
         self.server.starttls()
-        self.server.login(self.user,self.password)
+        self.server.login(self.user, self.password)
 
         # generate the message
         msg = MIMEMultipart()
         msg['To'] = to
         msg['CC'] = cc
         msg['Subject'] = subject
-        if os.getenv('SENDER_REPLYTO'):
-            msg.add_header('reply-to',os.getenv('SENDER_REPLYTO'))
+        if self.reply_to:
+            msg.add_header('reply-to', self.reply_to)
 
         # show what we are going to do
-        print(f" Sending message - TO: {to}, CC: {cc}, REPLYTO: {os.getenv('SENDER_REPLYTO')}")
+        print(f" Sending message - TO: {to}, CC: {cc}, REPLYTO: {self.reply_to}")
         print(f" ===============\n SUBJECT: {subject}")
         print(f" BODY:\n{body}")
 
         # add the message body
-        msg.attach(MIMEText(body,'plain'))
-        self.server.sendmail(self.user,"%s,%s"%(to,cc),msg.as_string())
+        msg.attach(MIMEText(body, 'plain'))
+        self.server.sendmail(self.user, f"{to},{cc}", msg.as_string())
 
         #finally, quit the server
         self.server.quit()

deploy/compose.yaml

@@ -8,12 +8,22 @@ services:
       CLEO_USER_FILE: /run/secrets/cleo_user
       CLEO_PW_FILE: /run/secrets/cleo_pw
       CLEO_PROJECT_FILE: /run/secrets/cleo_project
+      SENDER_SERVER_FILE: /run/secrets/sender_server
+      SENDER_PORT_FILE: /run/secrets/sender_port
+      SENDER_REPLYTO_FILE: /run/secrets/sender_replyto
+      SENDER_USER_FILE: /run/secrets/sender_user
+      SENDER_PW_FILE: /run/secrets/sender_pw
       OPENAI_API_KEY_FILE: /run/secrets/openai_api_key
     secrets:
       - cleo_url
       - cleo_user
       - cleo_pw
       - cleo_project
+      - sender_server
+      - sender_port
+      - sender_replyto
+      - sender_user
+      - sender_pw
       - openai_api_key
 
   chat:
@@ -25,10 +35,39 @@ services:
     secrets:
       - openai_api_key
 
-    # ports:
-    #   - "8000:8000"
+  mailbox:
+    build:
+      context: ..
+      dockerfile: deploy/mailbox/Dockerfile-mailbox
+    environment:
+      IMAP_USER_FILE: /run/secrets/imap_user
+      IMAP_PW_FILE: /run/secrets/imap_pw
+      CLEO_URL_FILE: /run/secrets/cleo_url
+      CLEO_USER_FILE: /run/secrets/cleo_user
+      CLEO_PW_FILE: /run/secrets/cleo_pw
+      CLEO_PROJECT_FILE: /run/secrets/cleo_project
+    secrets:
+      - imap_user
+      - imap_pw
+      - cleo_url
+      - cleo_user
+      - cleo_pw
+      - cleo_project
+
+  # TODO: do mailbox, chat service, or cleo need ports open?
+  # ports:
+  #   - "8000:8000"
+
+  scraper:
+    build:
+      context: ..
+      dockerfile: deploy/scraper/Dockerfile-scraper
 
 secrets:
+  imap_user:
+    file: imap_user.txt
+  imap_pw:
+    file: imap_pw.txt
   cleo_url:
     file: cleo_url.txt
   cleo_user:
@@ -37,5 +76,15 @@ secrets:
     file: cleo_pw.txt
   cleo_project:
     file: cleo_project.txt
+  sender_server:
+    file: sender_server.txt
+  sender_port:
+    file: sender_port.txt
+  sender_replyto:
+    file: sender_replyto.txt
+  sender_user:
+    file: sender_user.txt
+  sender_pw:
+    file: sender_pw.txt
   openai_api_key:
     file: openai_api_key.txt

deploy/dev-install.sh

@@ -6,4 +6,6 @@ docker compose up -d --build --force-recreate --always-recreate-deps
 
 # secrets files are created by CI pipeline and destroyed here
 rm cleo_*.txt
+rm imap_*.txt
+rm sender_*.txt
 rm openai_api_key.txt

deploy/mailbox/Dockerfile-mailbox

@@ -0,0 +1,12 @@
+# syntax=docker/dockerfile:1
+FROM python:3.10
+RUN mkdir -p /root/A2rchi
+WORKDIR /root/A2rchi
+COPY pyproject.toml pyproject.toml
+COPY README.md README.md
+COPY LICENSE LICENSE
+COPY config config
+COPY A2rchi A2rchi
+RUN pip install --upgrade pip && pip install .
+
+CMD ["python", "-u", "A2rchi/bin/service_mailbox.py"]

deploy/scraper/Dockerfile-scraper

@@ -0,0 +1,12 @@
+# syntax=docker/dockerfile:1
+FROM python:3.10
+RUN mkdir -p /root/A2rchi
+WORKDIR /root/A2rchi
+COPY pyproject.toml pyproject.toml
+COPY README.md README.md
+COPY LICENSE LICENSE
+COPY config config
+COPY A2rchi A2rchi
+RUN pip install --upgrade pip && pip install .
+
+CMD ["python", "-u", "A2rchi/bin/service_scraper.py"]