refactor: move non action files to utils

utils/__init__.py

@@ -0,0 +1,3 @@
+from utils.const import *
+from utils.helpers import *
+from utils.models import Announcement, Stock, Telegram, session

utils/const.py

@@ -8,7 +8,8 @@
 load_dotenv(find_dotenv())
 
 CATEGORIES = {
-    2: 'IPO', 3: 'FPO',
+    2: 'IPO',
+    3: 'FPO',
     5: 'Right Share',
     7: 'Mutual Fund',
     8: 'Debenture'

utils/helpers.py

@@ -4,11 +4,12 @@
 from typing import Dict, List, Optional
 
 import requests
+from bs4 import BeautifulSoup as bs
 from nepali_datetime import date as nepdate
 
 import utils.store as store
 from utils.const import CATEGORIES, current_dir
-from utils.models import Stock
+from utils.models import Stock, session
 
 
 def replace_this(substring: str, from_given_text: str) -> str:
@@ -23,8 +24,7 @@ def extract_units(sharetype: str) -> Optional[str]:
     return None
 
 
-def mark_as_published(given_item: Stock) -> None:
-    from utils.insert import session
+def mark_as_published(given_item) -> None:
     given_item.is_published = True
     return session.add(given_item)
 
@@ -76,7 +76,7 @@ def handle_response(the_url: str, payload: dict, **kwargs):
         res = req.json()
         if req.status_code == 200:
             if kwargs:
-                from utils.insert import add_chat
+                from actions.save import add_chat
                 add_chat(kwargs['stock_id'], res['result']['message_id'])
             print(req.json())
             return True
@@ -121,3 +121,18 @@ def get_sharetype(stock_id: int, raw_info: str) -> Optional[str]:
 def hashtag(given_str: str) -> str:
     # remove spaces between two words
     return f"#{''.join(given_str.split())}"
+
+
+def html_scraper(url):
+
+    parser = 'lxml'
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36\
+            (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36"
+    }
+    try:
+        response = requests.get(url, headers=headers)
+    except requests.exceptions.ConnectionError as e:
+        return print("Looks like the stock API did an oopsie:\n", e)
+    else:
+        return bs(response.text, parser)

utils/models.py

@@ -31,8 +31,10 @@ class Stock(BaseModel):
     units = Column(String(), nullable=True)
     is_published = Column(Boolean(), default=False)
     stock_added_at = Column(DateTime(), default=datetime.utcnow)
-    chat = relationship('Telegram', backref='stock', uselist=False,
-                        lazy=True, cascade='all, delete-orphan')
+    chat = relationship(
+        'Telegram', backref='stock', uselist=False,
+        lazy=True, cascade='all, delete-orphan'
+    )
 
     def __repr__(self):
         return f'{self.company_name}({self.stock_type})'
@@ -51,17 +53,15 @@ def __repr__(self):
 class Announcement(BaseModel):
     __tablename__ = 'announcement'
     id = Column(Integer(), primary_key=True)
-    message_id = Column(Integer(), default=0)
     content = Column(String(), nullable=False)
+    content_url = Column(String(), nullable=True)
     published_date = Column(Date(), nullable=True)
     is_published = Column(Boolean(), default=False)
-    announced_at = Column(DateTime(), default=datetime.utcnow)
+    scraped_at = Column(DateTime(), default=datetime.utcnow)
 
     def __repr__(self):
         return self.content
 
 
-if __name__ == '__main__':
-    # To create a new database with models specified above if db doesn't exist
-    if not path.isfile(DATABASE_URI):
-        BaseModel.metadata.create_all(engine)
+# Map the models to the database as tables
+BaseModel.metadata.create_all(engine, checkfirst=True)

utils/store.py

@@ -1,2 +1,3 @@
 image_name = None
-new_stocks = []
+new_stocks = []
+new_announcements = []