extend crawl with ordered lists

teotoplak · Mar 2, 2023 · 5b1d215 · 5b1d215
1 parent a52852e
commit 5b1d215
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 0 deletions.
diff --git a/src/crawl/__pycache__/crawl.cpython-311.pyc b/src/crawl/__pycache__/crawl.cpython-311.pyc
diff --git a/src/crawl/crawl.py b/src/crawl/crawl.py
@@ -15,6 +15,7 @@ def crawl_url(url):
     lists = soup.find_all('ul')
     # find all tables on the page
     tables = soup.find_all('table')
+    ordered_lists = soup.find_all('ol')
 
     text_chunks = []
 
@@ -28,6 +29,12 @@ def crawl_url(url):
             text_chunks.append(li.get_text())
             print(li.get_text() + "\n")
 
+    # loop through the lists and print the list items
+    for ol in ordered_lists:
+        for li in ol.find_all('li'):
+            text_chunks.append(li.get_text())
+            print(li.get_text() + "\n")
+
     # loop through the tables and print the table cells
     for table in tables:
         for row in table.find_all('tr'):

diff --git a/tst/crawl/__pycache__/test_crawl.cpython-311-pytest-7.2.1.pyc b/tst/crawl/__pycache__/test_crawl.cpython-311-pytest-7.2.1.pyc
diff --git a/tst/crawl/test_crawl.py b/tst/crawl/test_crawl.py
@@ -17,6 +17,7 @@ def test_education_fact_sheet():
     chunks = crawl_url(url)
     expected_chunks = [
         "*Üldhariduses keskmine õpilaste arv õpetaja kohta: 9,8 (HTM), regionaalselt väga suur kõikumine – mõnes koolis 2 õpilast õpetaja kohta (maakoolid), teises 20+ õpilast õpetaja kohta (Tallinn).",
+        "Töötasu kasvatamine (Eesti 200: 3000eur; Isamaa: õpetaja keskmine Eesti keskmisest 125%; Sotsdem: miinimumpalk vähemalt 130% eelneva aasta Eesti keskmisest palgast; Keskerakond: 3000eur)"
     ]
     for chunk in expected_chunks:
         assert chunk in chunks