Changes of joacmue

.gitignore

@@ -1,2 +1,4 @@
 laws
 laws-md
+.vscode
+__pycache__

README.md

@@ -1,53 +1,94 @@
-BundesGit Gesetze Tools
-=======================
+# BundesGit Gesetze Tools
 
-These scripts are used to keep the law repository up to date.
+These scripts are used to keep the [law repository](https://github.com/bundestag/gesetze) up to date.
 
 Install requirements:
-
 ```bash
 pip install -r requirements.txt
 ```
 
-For help see their docstring, command line help or source code.
+For help, see the docstring of the scripts, command line help or source code.
+
 
-## lawde.py
+## Download laws (`lawde.py`)
 
 Downloads all laws as XML files from
 [www.gesetze-im-internet.de](http://www.gesetze-im-internet.de/)
 and extracts them to a directory.
 
-Last tested: 2017-01-14 SUCCESS
+### Usage
 
-## lawdown.py
+Update your list of laws first:
+```bash
+python3 lawde.py updatelist
+```
 
-Converts all XML laws to Markdown and copies them with other files related
+You can then download all laws by calling
+```bash
+python3 lawde.py loadall
+```
+Which will take approx. 2-3hrs.
+
+Alternatively, you can find the individual law you're interested in in [./data/laws.json](./data/laws.json), which is mostly a list of laws in this form:
+```bash
+{"slug": "<shortname>", "name": "<longname>", "abbreviation": "<abbreviation>"}
+```
+You can download individual laws by calling
+```bash
+python3 lawde.py load <shortname>
+```
+
+Last tested: 2021-05-15 SUCCESS
+
+
+## Convert to Markdown (`lawdown.py`)
+
+Converts all downloaded XML laws to Markdown format and copies them with other files related
 to the law into specified working directory.
 
-Last tested: 2017-01-14 SUCCESS
+### Usage
+
+```bash
+python3 lawdown.py convert <inpath> <outpath>
+python3 lawdown.py convert ./laws ./laws-md
+```
+
+Last tested: 2021-05-15 SUCCESS
 
-## bgbl_scraper.py
+
+## Scaper Bundesgesetzblatt (`bgbl_scraper.py`)
 
 Scrapes the table of contents of all issues of the Bundesgesetzblatt and dumps
 the result to JSON.
 
-Last tested: 2021-03-30 SUCCESS
+```bash
+python3 bgbl_scraper.py data/bgbl.json
+```
+
+Last tested: 2021-05-15 SUCCESS
 
-## banz_scraper.py
+
+## Scaper Bundesanzeiger (`banz_scraper.py`)
 
 Scrapes the table of contents of all available issues of the Bundesanzeiger and
 dumps the result to JSON.
 
-Last tested: 2017-01-14 SUCCESS
+Last tested: 2020-12-23 SUCCESS
 
-## vkbl_scraper.py
+
+## Scaper Verkehrsblatt (`vkbl_scraper.py`)
 
 Scrapes the table of contents of all available issues of the Verkehrsblatt and
 dumps the result to JSON.
 
-Last tested: 2017-01-14 SUCCESS
+```bash
+python3 vkbl_scraper.py data/vkbl.json
+```
+
+Last tested: 2021-05-15 SUCCESS
+
 
-## lawgit.py
+## Commit changes (`lawgit.py`)
 
 Checks the repositories working directory for changes, tries to find relations
 to table of content entries in BGBl and BAnz data, commits the changes to a branch

banz_scraper.py

@@ -1,23 +1,18 @@
 #!/usr/bin/env python3
 
 """BAnz-Scraper.
-
 Usage:
   banz_scaper.py <outputfile> [<minyear> [<maxyear>]]
   banz_scaper.py -h | --help
   banz_scaper.py --version
-
 Options:
   -h --help     Show this screen.
   --version     Show version.
-
 Duration Estimates:
   2-5 minutes per year
   30-75 minutes in total
-
 Examples:
   banz_scaper.py data/banz.json
-
 """
 from pathlib import Path
 import re
@@ -114,7 +109,7 @@ def get_items(self, year, date: Tuple[str, str]):
             title_result = row.find(class_="title_result")
 
             orig_date: Optional[str] = None
-            match = re.search(r'[Vv]om: (\d+)\. ([\wä]+) (\d{4})', str(title_result), re.U)
+            match = re.search(r'[Vv]om: (\d+)\. ([\wä]+) (\d{4})', str(title_result), re.U)
             if match:
                 day = int(match.group(1))
                 month = self.MONTHS.index(match.group(2)) + 1
@@ -151,17 +146,20 @@ def main(arguments):
     maxyear = arguments['<maxyear>'] or 10000
     minyear = int(minyear)
     maxyear = int(maxyear)
+    print(f"This will scrape information from the Bundesanzeiger between {minyear} and {maxyear}.")
+    print(f"Results will be stored in {arguments['<outputfile>']}")
+    print("You will see all dates with publications appear below as they are parsed.")
     banz = BAnzScraper()
     data = {}
     if Path(arguments['<outputfile>']).exists():
         with open(arguments['<outputfile>']) as f:
             data = json.load(f)
     data.update(banz.scrape(minyear, maxyear))
-    with open(arguments['<outputfile>'], 'w') as f:
-        json.dump(data, f, indent=4)
+    with open(arguments['<outputfile>'], 'w', encoding='utf8') as f:
+        json.dump(data, f, indent=4, sort_keys=True, ensure_ascii=False)
 
 
 if __name__ == '__main__':
     from docopt import docopt
     arguments = docopt(__doc__, version='BAnz-Scraper 0.0.1')
-    main(arguments)
+    main(arguments)

bgbl_scraper.py

@@ -30,7 +30,6 @@
 
 from typing import List
 
-
 class BGBLScraper:
     BASE_URL = 'http://www.bgbl.de/xaver/bgbl/'
 
@@ -151,6 +150,7 @@ def get_number_toc(self, number_id, number_did):
             toc.append(d)
         return toc
 
+
 def main(arguments):
     minyear = arguments['<minyear>'] or 0
     maxyear = arguments['<maxyear>'] or 10000
@@ -162,8 +162,8 @@ def main(arguments):
         with open(arguments['<outputfile>']) as f:
             data = json.load(f)
     data.update(bgbl.scrape(minyear, maxyear))
-    with open(arguments['<outputfile>'], 'w') as f:
-        json.dump(data, f, indent=4)
+    with open(arguments['<outputfile>'], 'w+', encoding='utf8') as f:
+        json.dump(data, f, indent=2, sort_keys=True, ensure_ascii=False)
 
 
 if __name__ == '__main__':