Merge pull request #255 from edgi-govdata-archiving/202-be-nicer-to-i…

…nternet-archive-and-also-be-more-strict-when-loading-mementos Be nicer to Internet Archive and also be more strict when loading mementos
edgi-govdata-archiving · Sep 14, 2018 · 4bb08f8 · 4bb08f8
2 parents c0061fd + e63da2d
commit 4bb08f8
Show file tree

Hide file tree

Showing 33 changed files with 1,563 additions and 260 deletions.
diff --git a/.gitignore b/.gitignore
@@ -51,8 +51,8 @@ cover/*
 *.log
 
 # Sphinx documentation
-doc/build/
-doc/source/generated/*.rst
+docs/build/
+docs/source/generated/*.rst
 
 # PyBuilder
 target/

diff --git a/docs/source/cdx_api.rst b/docs/source/cdx_api.rst
@@ -1,12 +1,15 @@
 .. currentmodule:: web_monitoring.internetarchive
 
-******************************************************
-Python API to Internet Archive Wayback Machine CDX API
-******************************************************
+**********************************************
+Python API to Internet Archive Wayback Machine
+**********************************************
 
 Search for historical snapshots of a URL. Download metadata about the snapshots
 and/or the snapshot content itself.
 
+We implement Python clients for the CDX and Memento APIs provided by Wayback
+Machine.
+
 Tutorial
 ========
 
@@ -15,9 +18,8 @@ TO DO
 API Documentation
 =================
 
-.. autosummary::
-    :toctree: generated
+.. autoclass:: WaybackClient
 
-    list_versions
-    search_cdx
-    timestamped_uri_to_version
+    .. automethod:: search
+    .. automethod:: list_versions
+    .. automethod:: timestamped_uri_to_version
diff --git a/docs/source/generated/web_monitoring.internetarchive.list_versions.rst b/docs/source/generated/web_monitoring.internetarchive.list_versions.rst
diff --git a/docs/source/generated/web_monitoring.internetarchive.search_cdx.rst b/docs/source/generated/web_monitoring.internetarchive.search_cdx.rst
diff --git a/.../source/generated/web_monitoring.internetarchive.timestamped_uri_to_version.rst b/.../source/generated/web_monitoring.internetarchive.timestamped_uri_to_version.rst
diff --git a/web_monitoring/cli.py b/web_monitoring/cli.py
@@ -28,22 +28,24 @@ def _add_and_monitor(versions):
 
 def import_ia(url, *, from_date=None, to_date=None, maintainers=None,
               tags=None, skip_unchanged='resolved-response'):
-    # Pulling on this generator does the work.
     skip_responses = skip_unchanged == 'response'
-    versions = (ia.timestamped_uri_to_version(version.date, version.raw_url,
-                                              url=version.url,
-                                              maintainers=maintainers,
-                                              tags=tags,
-                                              view_url=version.view_url)
-                for version in ia.list_versions(url,
-                                                from_date=from_date,
-                                                to_date=to_date,
-                                                skip_repeats=skip_responses))
-
-    if skip_unchanged == 'resolved-response':
-        versions = _filter_unchanged_versions(versions)
-
-    _add_and_monitor(versions)
+    with ia.WaybackClient() as wayback:
+        # Pulling on this generator does the work.
+        versions = (wayback.timestamped_uri_to_version(version.date,
+                                                       version.raw_url,
+                                                       url=version.url,
+                                                       maintainers=maintainers,
+                                                       tags=tags,
+                                                       view_url=version.view_url)
+                    for version in wayback.list_versions(url,
+                                                         from_date=from_date,
+                                                         to_date=to_date,
+                                                         skip_repeats=skip_responses))
+
+        if skip_unchanged == 'resolved-response':
+            versions = _filter_unchanged_versions(versions)
+
+        _add_and_monitor(versions)
 
 
 def _filter_unchanged_versions(versions):