v 1.2.1 complete

create site_text table in scrape
ericnost · Dec 15, 2023 · 971699a · 971699a
1 parent abd3788
commit 971699a
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 0 deletions.
diff --git a/dist/observatory-1.2.1-py3-none-any.whl b/dist/observatory-1.2.1-py3-none-any.whl
diff --git a/dist/observatory-1.2.1.tar.gz b/dist/observatory-1.2.1.tar.gz
diff --git a/src/observatory.py b/src/observatory.py
@@ -557,7 +557,10 @@ def scrape(urls):
 
   conn = None 
   conn = psycopg2.connect("dbname="+credentials["postgres"]["db"]+" user="+credentials["postgres"]["user"]+" password="+credentials["postgres"]["password"]+" host=localhost port=5432")
+  conn.autocommit = True
+
   cur = conn.cursor()
+  cur.execute("""CREATE TABLE IF NOT EXISTS public.site_text (url varchar,  body_text varchar, date varchar, source varchar(10), domain varchar);""")
 
   def DBinsert(url, contents, date, source, domain, count):
     print("inserting: " + str(count), end='\r')