diff --git a/.github/workflows/ci-cd.yaml b/.github/workflows/ci-cd.yaml index 7c7bbb24..29441c2e 100644 --- a/.github/workflows/ci-cd.yaml +++ b/.github/workflows/ci-cd.yaml @@ -26,6 +26,12 @@ jobs: # create secrets files for docker-compose - name: Create Secrets Files run: | + touch ${{ github.workspace }}/deploy/imap_user.txt + echo "${{ secrets.DEV_IMAP_USER }}" >> ${{ github.workspace }}/deploy/imap_user.txt + chmod 400 ${{ github.workspace }}/deploy/imap_user.txt + touch ${{ github.workspace }}/deploy/imap_pw.txt + echo "${{ secrets.DEV_IMAP_PW }}" >> ${{ github.workspace }}/deploy/imap_pw.txt + chmod 400 ${{ github.workspace }}/deploy/imap_pw.txt touch ${{ github.workspace }}/deploy/cleo_url.txt echo "${{ secrets.DEV_CLEO_URL }}" >> ${{ github.workspace }}/deploy/cleo_url.txt chmod 400 ${{ github.workspace }}/deploy/cleo_url.txt @@ -38,6 +44,21 @@ jobs: touch ${{ github.workspace }}/deploy/cleo_project.txt echo "${{ secrets.DEV_CLEO_PROJECT }}" >> ${{ github.workspace }}/deploy/cleo_project.txt chmod 400 ${{ github.workspace }}/deploy/cleo_project.txt + touch ${{ github.workspace }}/deploy/sender_server.txt + echo "${{ secrets.DEV_SENDER_SERVER }}" >> ${{ github.workspace }}/deploy/sender_server.txt + chmod 400 ${{ github.workspace }}/deploy/sender_server.txt + touch ${{ github.workspace }}/deploy/sender_port.txt + echo "${{ secrets.DEV_SENDER_PORT }}" >> ${{ github.workspace }}/deploy/sender_port.txt + chmod 400 ${{ github.workspace }}/deploy/sender_port.txt + touch ${{ github.workspace }}/deploy/sender_replyto.txt + echo "${{ secrets.DEV_SENDER_REPLYTO }}" >> ${{ github.workspace }}/deploy/sender_replyto.txt + chmod 400 ${{ github.workspace }}/deploy/sender_replyto.txt + touch ${{ github.workspace }}/deploy/sender_user.txt + echo "${{ secrets.DEV_SENDER_USER }}" >> ${{ github.workspace }}/deploy/sender_user.txt + chmod 400 ${{ github.workspace }}/deploy/sender_user.txt + touch ${{ github.workspace }}/deploy/sender_pw.txt + echo "${{ secrets.DEV_SENDER_PW }}" >> ${{ github.workspace }}/deploy/sender_pw.txt + chmod 400 ${{ github.workspace }}/deploy/sender_pw.txt touch ${{ github.workspace }}/deploy/openai_api_key.txt echo "${{ secrets.OPENAI_API_KEY }}" >> ${{ github.workspace }}/deploy/openai_api_key.txt chmod 400 ${{ github.workspace }}/deploy/openai_api_key.txt @@ -60,10 +81,9 @@ jobs: # clean up secret files - name: Remove Secrets from Runner run: | - rm ${{ github.workspace }}/deploy/cleo_url.txt - rm ${{ github.workspace }}/deploy/cleo_user.txt - rm ${{ github.workspace }}/deploy/cleo_pw.txt - rm ${{ github.workspace }}/deploy/cleo_project.txt + rm ${{ github.workspace }}/deploy/cleo_*.txt + rm ${{ github.workspace }}/deploy/imap_*.txt + rm ${{ github.workspace }}/deploy/sender_*.txt rm ${{ github.workspace }}/deploy/openai_api_key.txt # print job status diff --git a/A2rchi/bin/service_mailbox.py b/A2rchi/bin/service_mailbox.py index 268712ef..544f45ce 100755 --- a/A2rchi/bin/service_mailbox.py +++ b/A2rchi/bin/service_mailbox.py @@ -1,8 +1,10 @@ #!/bin/python -import os from A2rchi.interfaces import cleo from A2rchi.utils import mailbox +import os + +print("Starting Mailbox Service") cleo = cleo.Cleo('Cleo_Helpdesk') while True: diff --git a/A2rchi/bin/service_scraper.py b/A2rchi/bin/service_scraper.py index 5e974611..02a27e08 100644 --- a/A2rchi/bin/service_scraper.py +++ b/A2rchi/bin/service_scraper.py @@ -1,6 +1,8 @@ -import os from A2rchi.utils.scraper import Scraper +import os + +print("Starting Scraper Service") scraper=Scraper() while True: diff --git a/A2rchi/utils/mailbox.py b/A2rchi/utils/mailbox.py index 846e41b0..98c3f528 100644 --- a/A2rchi/utils/mailbox.py +++ b/A2rchi/utils/mailbox.py @@ -1,9 +1,10 @@ #!/bin/python -import os,sys -import getpass, imaplib, email - from A2rchi.utils.config_loader import Config_Loader -config = Config_Loader().config["utils"]["mailbox"] +from A2rchi.utils.env import read_secret + +import email +import imaplib + class Mailbox: 'A class to describe the mailbox usage.' @@ -13,8 +14,12 @@ def __init__(self): The mailbox (should be a singleton). """ self.mailbox = None + self.user = read_secret('IMAP_USER') + self.password = read_secret('IMAP_PW') + self.config = Config_Loader().config["utils"]["mailbox"] + # make sure to open the mailbox - if self._verify: + if self._verify(): self.mailbox = self._connect() def find_issue_id(self,description): @@ -27,7 +32,7 @@ def find_issue_id(self,description): issue_id = int(description[index+9:].split()[0]) return issue_id - def process_messages(self,cleo): + def process_messages(self, cleo): """ Select all messages in the mailbx and process them. """ @@ -105,9 +110,9 @@ def _get_email_body(self,msg): try: body = body.decode(charset) except UnicodeDecodeError: - handle_error("UnicodeDecodeError: encountered.",msg,charset) + self._handle_error("UnicodeDecodeError: encountered.",msg,charset) except AttributeError: - handle_error("AttributeError: encountered" ,msg,charset) + self._handle_error("AttributeError: encountered" ,msg,charset) return body, body_html @@ -131,25 +136,27 @@ def _connect(self): """ Open the mailbox """ - print(f" Open mailbox (U:{os.getenv('IMAP_USER')} P:*********)") - mailbox = imaplib.IMAP4(host='ppc.mit.edu', port=config["IMAP4_PORT"], timeout=None) - mailbox.login(os.getenv('IMAP_USER'),os.getenv('IMAP_PW')) + print(f" Open mailbox (U:{self.user} P:*********)") + mailbox = imaplib.IMAP4(host='ppc.mit.edu', port=self.config["IMAP4_PORT"], timeout=None) + mailbox.login(self.user, self.password) return mailbox - def _handle_error(self,errmsg, emailmsg, cs): + def _handle_error(self, errmsg, emailmsg, cs): print() print(errmsg) print("This error occurred while decoding with ",cs," charset.") print("These charsets were found in this email.",self._get_charsets(emailmsg)) print("This is the subject:",emailmsg['subject']) print("This is the sender:",emailmsg['From']) + return def _verify(self): """ Make sure the environment is setup """ - if os.getenv('IMAP_USER') == None or os.getenv('IMAP_PW') == None: + if self.user == None or self.password == None: print(" Did not find all cleo configs: IMAP_USER, IMAP_PW (source ~/.imap).") return False + return True diff --git a/A2rchi/utils/scraper.py b/A2rchi/utils/scraper.py index 56f4b318..e589a2b0 100644 --- a/A2rchi/utils/scraper.py +++ b/A2rchi/utils/scraper.py @@ -22,7 +22,6 @@ class Scraper(): def __init__(self): from A2rchi.utils.config_loader import Config_Loader - config = Config_Loader().config["chains"]["chain"] global_config = Config_Loader().config["global"] #Check if target folders exist @@ -47,7 +46,7 @@ def hard_scrape(self,verbose=False): """ self.scrape_submit_files() - self.scrape_rst_files(self.github_url,self.raw_url) + self.scrape_rst_files(self.github_url, self.raw_url) if verbose: print("Scraping was completed successfully") def scrape_submit_files(self): @@ -86,11 +85,3 @@ def scrape_rst_files(self,url,raw_url): print(f"Error downloading {file_url}: {file_response.status_code}") else: print(f"Error: {response.status_code}") - - - -# Example usage -s = Scraper() -url,raw_url = s.github_url,s.raw_url -# s.scrape_rst_files(url,raw_url) -# s.scrape_all() diff --git a/A2rchi/utils/sender.py b/A2rchi/utils/sender.py index d53b3118..a01410b9 100644 --- a/A2rchi/utils/sender.py +++ b/A2rchi/utils/sender.py @@ -1,45 +1,51 @@ -import os,sys -import smtplib +from A2rchi.utils.env import read_secret + from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText +import os +import smtplib + + class Sender: - 'A class to send emails in an uncomplicated fashion.' + """A class to send emails in an uncomplicated fashion.""" def __init__(self): """ Give it a name and generate a conncetion to the database (should be a singleton). """ + self.server_name = read_secret('SENDER_SERVER') + self.port = read_secret('SENDER_PORT') + self.user = read_secret('SENDER_USER') + self.password = read_secret('SENDER_PW') + self.reply_to = read_secret('SENDER_REPLYTO') - print(f" Open smtp (SERVER:{os.getenv('SENDER_SERVER')} PORT:{os.getenv('SENDER_PORT')} U:{os.getenv('SENDER_USER')} P:*********)") + print(f" Open smtp (SERVER:{self.server_name} PORT:{self.port} U:{self.user} P:*********)") - self.server_name = os.getenv('SENDER_SERVER') - self.user = os.getenv('SENDER_USER') - self.password = os.getenv('SENDER_PW') - def send_message(self,to,cc,subject,body): + def send_message(self, to, cc, subject, body): - #start and login to SMTP server - self.server = smtplib.SMTP(self.server_name,os.getenv('SENDER_PORT')) + #start and login to SMTP server + self.server = smtplib.SMTP(self.server_name, self.port) self.server.starttls() - self.server.login(self.user,self.password) + self.server.login(self.user, self.password) # generate the message msg = MIMEMultipart() msg['To'] = to msg['CC'] = cc msg['Subject'] = subject - if os.getenv('SENDER_REPLYTO'): - msg.add_header('reply-to',os.getenv('SENDER_REPLYTO')) + if self.reply_to: + msg.add_header('reply-to', self.reply_to) # show what we are going to do - print(f" Sending message - TO: {to}, CC: {cc}, REPLYTO: {os.getenv('SENDER_REPLYTO')}") + print(f" Sending message - TO: {to}, CC: {cc}, REPLYTO: {self.reply_to}") print(f" ===============\n SUBJECT: {subject}") print(f" BODY:\n{body}") # add the message body - msg.attach(MIMEText(body,'plain')) - self.server.sendmail(self.user,"%s,%s"%(to,cc),msg.as_string()) + msg.attach(MIMEText(body, 'plain')) + self.server.sendmail(self.user, f"{to},{cc}", msg.as_string()) #finally, quit the server self.server.quit() diff --git a/deploy/compose.yaml b/deploy/compose.yaml index e28c0c9e..cd6b6839 100644 --- a/deploy/compose.yaml +++ b/deploy/compose.yaml @@ -8,12 +8,22 @@ services: CLEO_USER_FILE: /run/secrets/cleo_user CLEO_PW_FILE: /run/secrets/cleo_pw CLEO_PROJECT_FILE: /run/secrets/cleo_project + SENDER_SERVER_FILE: /run/secrets/sender_server + SENDER_PORT_FILE: /run/secrets/sender_port + SENDER_REPLYTO_FILE: /run/secrets/sender_replyto + SENDER_USER_FILE: /run/secrets/sender_user + SENDER_PW_FILE: /run/secrets/sender_pw OPENAI_API_KEY_FILE: /run/secrets/openai_api_key secrets: - cleo_url - cleo_user - cleo_pw - cleo_project + - sender_server + - sender_port + - sender_replyto + - sender_user + - sender_pw - openai_api_key chat: @@ -25,10 +35,39 @@ services: secrets: - openai_api_key - # ports: - # - "8000:8000" + mailbox: + build: + context: .. + dockerfile: deploy/mailbox/Dockerfile-mailbox + environment: + IMAP_USER_FILE: /run/secrets/imap_user + IMAP_PW_FILE: /run/secrets/imap_pw + CLEO_URL_FILE: /run/secrets/cleo_url + CLEO_USER_FILE: /run/secrets/cleo_user + CLEO_PW_FILE: /run/secrets/cleo_pw + CLEO_PROJECT_FILE: /run/secrets/cleo_project + secrets: + - imap_user + - imap_pw + - cleo_url + - cleo_user + - cleo_pw + - cleo_project + + # TODO: do mailbox, chat service, or cleo need ports open? + # ports: + # - "8000:8000" + + scraper: + build: + context: .. + dockerfile: deploy/scraper/Dockerfile-scraper secrets: + imap_user: + file: imap_user.txt + imap_pw: + file: imap_pw.txt cleo_url: file: cleo_url.txt cleo_user: @@ -37,5 +76,15 @@ secrets: file: cleo_pw.txt cleo_project: file: cleo_project.txt + sender_server: + file: sender_server.txt + sender_port: + file: sender_port.txt + sender_replyto: + file: sender_replyto.txt + sender_user: + file: sender_user.txt + sender_pw: + file: sender_pw.txt openai_api_key: file: openai_api_key.txt diff --git a/deploy/dev-install.sh b/deploy/dev-install.sh index e8c5336d..e5308c25 100755 --- a/deploy/dev-install.sh +++ b/deploy/dev-install.sh @@ -6,4 +6,6 @@ docker compose up -d --build --force-recreate --always-recreate-deps # secrets files are created by CI pipeline and destroyed here rm cleo_*.txt +rm imap_*.txt +rm sender_*.txt rm openai_api_key.txt diff --git a/deploy/mailbox/Dockerfile-mailbox b/deploy/mailbox/Dockerfile-mailbox new file mode 100644 index 00000000..e0d918a1 --- /dev/null +++ b/deploy/mailbox/Dockerfile-mailbox @@ -0,0 +1,12 @@ +# syntax=docker/dockerfile:1 +FROM python:3.10 +RUN mkdir -p /root/A2rchi +WORKDIR /root/A2rchi +COPY pyproject.toml pyproject.toml +COPY README.md README.md +COPY LICENSE LICENSE +COPY config config +COPY A2rchi A2rchi +RUN pip install --upgrade pip && pip install . + +CMD ["python", "-u", "A2rchi/bin/service_mailbox.py"] diff --git a/deploy/mailbox/Dockerfile-mailer b/deploy/mailbox/Dockerfile-mailer deleted file mode 100644 index e69de29b..00000000 diff --git a/deploy/scraper/Dockerfile-scraper b/deploy/scraper/Dockerfile-scraper index e69de29b..e72c55e7 100644 --- a/deploy/scraper/Dockerfile-scraper +++ b/deploy/scraper/Dockerfile-scraper @@ -0,0 +1,12 @@ +# syntax=docker/dockerfile:1 +FROM python:3.10 +RUN mkdir -p /root/A2rchi +WORKDIR /root/A2rchi +COPY pyproject.toml pyproject.toml +COPY README.md README.md +COPY LICENSE LICENSE +COPY config config +COPY A2rchi A2rchi +RUN pip install --upgrade pip && pip install . + +CMD ["python", "-u", "A2rchi/bin/service_scraper.py"]