Merge pull request #69 from DaITssu/fix/#68

[#68] postgres로 작성되어있는 코드를 mysql로 변경합니다.

notice/common/control_db.py

@@ -5,7 +5,7 @@
 from sqlalchemy.orm import Session
 
 import configuration
-from notification import Notification
+from notice.common.notification import Notification
 
 
 def update_notification(header: str, data: Notification, session: Session, s3: BaseClient, table: Table):

notice/common/notification.py

@@ -1,18 +1,18 @@
-from sqlalchemy import Column, Integer, CHAR, ARRAY, DateTime
+from sqlalchemy import Column, Integer, CHAR, ARRAY, DateTime, JSON
 from sqlalchemy.orm import declarative_base
 
 
 # unit class for notice.notice
 class Notification(declarative_base()):
     __tablename__ = "notice"
-    __table_args__ = {"schema": "notice"}
+    __table_args__ = {"schema": "daitssu"}
     id = Column(Integer, primary_key=True)
     title = Column(CHAR(1024))
     department_id = Column(Integer)
     content = Column(CHAR(2048))
     category = Column(CHAR(32))
-    image_url = Column(ARRAY(CHAR(2048)))
-    file_url = Column(ARRAY(CHAR(2048)))
+    image_url = Column(JSON)
+    file_url = Column(JSON)
     created_at = Column(DateTime)
     updated_at = Column(DateTime)
     views = Column(Integer)

notice/computer.py

@@ -11,13 +11,13 @@
 
 from fastapi.responses import JSONResponse
 
-from common.control_db import update_notification
-from common.notification import Notification
+from notice.common.control_db import update_notification
+from notice.common.notification import Notification
 
 URL = "http://cse.ssu.ac.kr/03_sub/01_sub.htm"
 
 db_url = sqlalchemy.engine.URL.create(
-    drivername="postgresql",
+    drivername="mysql+pymysql",
     username=configuration.db_user_name,
     password=configuration.db_pw,
     host=configuration.db_host,
@@ -59,17 +59,17 @@ def __init__(self, row: bs4.element.Tag):
             for file in files:
                 link = ("http://cse.ssu.ac.kr" + file['href'])
                 file_link.append(link)
-        self.file_url = file_link
+        self.file_url = {"url": file_link}
         self.title = children[1].text.strip()
-        self.image_url = []
+        self.image_url = {"url": []}
         self.category = "UNDERGRADUATE"
 
         created_date = list(map(int, children[3].text.split(".")))
         self.created_at = date(created_date[0], created_date[1], created_date[2])
         self.updated_at = date(created_date[0], created_date[1], created_date[2])
 
         with engine.connect() as connect:
-            department_table = Table("department", metadata_obj, schema="main", autoload_with=engine)
+            department_table = Table("department", metadata_obj, schema="daitssu", autoload_with=engine)
             query = department_table.select().where(department_table.c.name == "컴퓨터학부")
             results = connect.execute(query)
             for result in results:
@@ -87,7 +87,7 @@ def __str__(self):
 
 def computer_department_crawling():
     try:
-        page = 1  # 1 ~
+        page = 2  # 1 ~
         base_url = URL + "?page={0}".format(page)
         req = requests.get(base_url)
         soup = BeautifulSoup(req.text, 'lxml')
@@ -97,13 +97,13 @@ def computer_department_crawling():
         for row in rows:
             results.append(ComputerNotification(row))
 
-        notification_table = Table("notice", metadata_obj, schema="notice", autoload_with=engine)
+        notification_table = Table("notice", metadata_obj, schema="daitssu", autoload_with=engine)
 
         with session_maker() as session:
             for result in results:
                 update_notification("CSE", result, session, s3, notification_table)
-
             session.commit()
+
     except botocore.exceptions.NoCredentialsError as e:
         return JSONResponse(content=e.args, status_code=403)
     except Exception as e:

notice/ssu_catch.py

@@ -9,16 +9,16 @@
 from fastapi.responses import JSONResponse
 import boto3
 
-from common.control_db import update_notification
-from common.notification import Notification
+from notice.common.control_db import update_notification
+from notice.common.notification import Notification
 
 URL = "https://scatch.ssu.ac.kr/%ea%b3%b5%ec%a7%80%ec%82%ac%ed%95%ad"
 
 metadata_obj = MetaData()
 Base = declarative_base()
 
 db_url = sqlalchemy.engine.URL.create(  # db연결 url 생성
-    drivername="postgresql",
+    drivername="mysql+pymysql",
     username=configuration.db_user_name,
     password=configuration.db_pw,
     host=configuration.db_host,
@@ -61,10 +61,10 @@ def __init_date(self, column):  # 생성 시각 크롤링
         self.updated_at = date(date_arr[0], date_arr[1], date_arr[2])
 
     def __init_contents(self, column):  # 본문 내용 크롤링
-        self.image_url = []
+        self.image_url = {"url": []}
         self.content = ""
         real_content = ""
-        self.file_url = []
+        self.file_url = {"url": []}
 
         target = column.find('a')
         post_url = target['href']
@@ -74,15 +74,15 @@ def __init_contents(self, column):  # 본문 내용 크롤링
         for tag in contents.findAll('p'):
             img = tag.find("img", class_=lambda css_class: css_class != "emoji")
             if img:
-                self.image_url.append(img['src'])
+                self.image_url["url"].append(img['src'])
             else:
                 real_content += BeautifulSoup(tag.text, "lxml").text
         file_urls = contents.find("ul")
 
         if file_urls:
             links = file_urls.findAll("a")
             for item in links:
-                self.file_url.append(item['href'])
+                self.file_url["url"].append(item['href'])
 
         self.content = real_content
 
@@ -109,7 +109,7 @@ def __init_title(self, column):  # 제목 크롤링
 
     def __init_department(self):  # 등록 부서 크롤링
         with engine.connect() as connect:
-            department_table = Table("department", metadata_obj, schema="main", autoload_with=engine)
+            department_table = Table("department", metadata_obj, schema="daitssu", autoload_with=engine)
             query = department_table.select().where(department_table.c.name == "슈케치")
             results = connect.execute(query)
             for result in results:
@@ -153,7 +153,7 @@ def ssu_catch_crawling():
         for (idx, item) in enumerate(content_iterator):  # 핵심 크롤링 부분
             if idx % 2 == 0:
                 content_list.append(Content(item.find('div')))
-        notification_table = Table("notice", metadata_obj, schema="notice", autoload_with=engine)
+        notification_table = Table("notice", metadata_obj, schema="daitssu", autoload_with=engine)
 
         with session_maker() as session:
             for result in content_list: