Clean up data import and logging

data/domains/aa.yaml

@@ -9,19 +9,19 @@ aa1:
 aa1-1:
   title: Shoals Area (N.W. Alabama)
   target: https://www.shoalsaa.org
-  feed: tsml^https://www.shoalsaa.org/meetings
+  feed: tsml^https://www.shoalsaa.org/meetings/
   keywords: Florence, Lexington, Moulton, Red Bay, Rogersville, Russellville & Sheffield, Colbert, Franklin, Lauderdale & Lawrence Counties, Alabama and Belmont, Mississippi
 
 aa1-2:
   title: Hartselle, Cullman, and Blountsville, Alabama
   target: https://the12traditions.com
-  feed: tsml^https://the12traditions.com/meetings
+  feed: tsml^https://the12traditions.com/meetings/
   keywords: Allgood, Blountsville, Cullman, Eva, Hartselle
 
 aa1-3:
   title: Dekalb, Jackson and Marshall Counties, (Northeast) Alabama
   target: https://aaarea1dist3.org
-  feed: tsml^https://aaarea1dist3.org/meetings
+  feed: tsml^https://aaarea1dist3.org/meetings/
   keywords: Albertville, Arab, Douglas, Guntersville, Henagar, Rainsville, & Scottsboro. Dekalb County, Jackson County, Marshall County
 
 aa1-4:
@@ -33,13 +33,13 @@ aa1-4:
 aa1-5:
   title: West Alabama
   target: https://westalaa.org
-  feed: tsml^https://westalaa.org/meetings
+  feed: tsml^https://westalaa.org/meetings/
   keywords: Fayette, Haleyville, Hamilton, Jasper, Tuscaloosa, Vernon, Winfield
 
 aa1-6:
   title: Birmingham, Alabama
   target: https://birminghamaa.org
-  feed: tsml^https://birminghamaa.org/wp/meetings
+  feed: tsml^https://birminghamaa.org/wp/meetings/
   keywords: Adamsville, Alabaster, Bessemer, Birmingham, Calera, Center Point, Dolomite, Homewood, Hoover, Hueytown, Leeds, Montevallo, Morris, Mountain Brook, Pelham, Trussville, Vestavia Hills, & Warrior
 
 aa1-7:  # alias
@@ -48,19 +48,19 @@ aa1-7:  # alias
 aa1-8:
   title: Central Alabama
   target: https://centralalaa.org
-  feed: tsml^https://centralalaa.org/meetings
+  feed: tsml^https://centralalaa.org/meetings/
   keywords: Clanton, Greenville, Maplesville, Millbrook, Montgomery, Prattville, Selma, Slapout, Tallassee, Thorsby, Union Springs, Wetumpka, Montgomery
 
 aa1-9:
   title: East Central Alabama
   target: https://aaauburn.org
-  feed: tsml^https://aaauburn.org/meetings
+  feed: tsml^https://aaauburn.org/meetings/
   keywords: Alexander City, Auburn, Dadeville, Lanett, Opelika, Phenix City, Roanoke, Shorter, Tuskegee
 
 aa1-10:
   title: Dothan Area, Alabama
   target: http://www.aadothan.org
-  feed: tsml^https://aadothan.org/meetings
+  feed: tsml^https://aadothan.org/meetings/
   keywords: Ashford, Dothan, Eufaula, Headland
 
 aa1-11:
@@ -72,13 +72,14 @@ aa1-11:
 aa1-12:
   title: Mobile, Washington and Choctaw Counties, Alabama
   target: https://mobileaa.org
-  feed: tsml^https://mobileaa.org/meetings
+  feed: tsml^https://mobileaa.org/meetings/
   keywords: Butler, Chatom, Chickasaw, Citronelle, Dauphin Island, Frisco City, Grove Hill, Jackson, Mobile, Monroeville, Saraland, Semmes, Seton, Theodore. Choctaw County, Clarke County, Mobile County, Monroe County, Washington County
 
 aa1-13:
   title: Northwest Florida Tri-District Area
   target: https://aapensacola.org
-  feed: tsml^https://aapensacola.org/meetings
+  # Disabled: JSON.parse()??
+  #feed: tsml^https://aapensacola.org/meetings/
   keywords: Tri-District Intergroup, Navarre, Florida, Alabama state line, Escambia County, Santa Rosa County
 
 aa1-14:
@@ -112,13 +113,13 @@ aa1-16:
 aa1-19:
   title: Gulf Coast, Florida
   target: https://gulfcoastaa.org
-  feed: tsml^https://gulfcoastaa.org/meetings
+  feed: tsml^https://gulfcoastaa.org/meetings/
   keywords: Central and South Baldwin Counties, Gulf Shores, Orange Beach and Foley
 
 aa1-20:
   title: Huntsville, Florida
   target: https://aahuntsvilleal.com
-  feed: tsml^https://aahuntsvilleal.com/meetings
+  feed: tsml^https://aahuntsvilleal.com/meetings/
   keywords: Athens, Decatur, Huntsville, Madison and Morgan City
 
 aa1-21:  # alias
@@ -153,7 +154,8 @@ aa2:
 aa2-a:
   title: Anchorage Area Intergroup
   target: https://anchorageaa.org
-  feed: tsml^https://anchorageaa.org/?post_type=tsml_meeting
+  # Disabled: JSON.parse()??
+  #feed: tsml^https://anchorageaa.org/archives/meetings
 
 aa2-f:
   title: Fairbanks Intergroup
@@ -163,32 +165,33 @@ aa2-f:
 aa2-j:
   title: Juneau Intergroup
   target: https://aajuneauak.org
-  feed: tsml^https://aajuneauak.org/meetings
+  feed: tsml^https://aajuneauak.org/meetings/
 
 aa2-k:
   title: Kenai Peninsula Intergroup
   target: https://aakenaipeninsula.org
-  feed: tsml^https://aakenaipeninsula.org/meetings
+  feed: tsml^https://aakenaipeninsula.org/meetings/
 
 aa2-m:
   title: Mat-Su Valley Intergroup
   target: https://www.alaskamatsuaa.org/
-  #feed: CUSTOM^https://www.alaskamatsuaa.org/meetings
+  #feed: CUSTOM^https://www.alaskamatsuaa.org/meetings/
 
 aa2-s:
   title: Sitka Intergroup
   target: https://www.sitkaintergroup.org
-  #feed: CUSTOM^https://www.sitkaintergroup.org/meetings
+  #feed: CUSTOM^https://www.sitkaintergroup.org/meetings/
 
 aa3:
   title: Arizona
   target: http://www.area03.org
-  feed: tsml^https://area03.com/?post_type=tsml_meeting
+  # Disabled for various issues
+  #feed: tsml^https://area03.com/?post_type=tsml_meeting
 
 aa4:
   title: Arkansas
   target: http://www.arkansasaa.org
-  feed: tsml^https://arkansascentraloffice.org/meetings
+  feed: tsml^https://arkansascentraloffice.org/meetings/
 
 aa4-1:
   title: Northwest Arkansas
@@ -224,7 +227,7 @@ aa5:
 aa5-la:
   title: Los Angeles
   target: https://lacoaa.org
-  feed: tsml^https://lacoaa.org/meetings.php
+  #feed: UNKNOWN^https://lacoaa.org/meetings.php
 
 aa5-es-la:
   title: Hispano de Los Angeles

sync/__main__.py

@@ -51,4 +51,5 @@ def main():
 
 
 if __name__ == '__main__':
+    logging.basicConfig(format='%(levelname)s: %(message)s')
     main()

sync/collect/__init__.py

@@ -8,21 +8,11 @@
 
 class CollectionException(Exception):
     '''
-    Fatal (non-recoverable) feed import error
+    Fatal (semi-recoverable) feed import error
     '''
     pass
 
 
-def refresh(subdomain, source_url, source_type):
-    '''
-    Refresh meeting information from a source_url
-    '''
-    collector = importlib.import_module('.' + source_type, __name__)
-    if not collector:
-        raise Exception(f'Unable to load "{source_type}" collector.')
-    return collector.refresh(subdomain, source_url)
-
-
 def fetch_all(source_data):
     '''
     Attempt to refresh all feeds found in source_data
@@ -44,7 +34,17 @@ def fetch_all(source_data):
         try:
             refresh(subdomain, src_url, src_type)
         except CollectionException as e:
-            logging.error(f'Feed import failed [{src_url}]: {e}')
+            logging.warning(f'Feed import failed [{src_url}]: {e}')
+
+
+def refresh(subdomain, source_url, source_type):
+    '''
+    Refresh meeting information from a source_url
+    '''
+    collector = importlib.import_module('.' + source_type, __name__)
+    if not collector:
+        raise Exception(f'Unable to load "{source_type}" collector.')
+    return collector.grab(subdomain, source_url)
 
 
 def validate(
@@ -75,62 +75,65 @@ def validate(
             'time': time,
             }
 
+    # Short alias for heavily used exception
+    Invalid = CollectionException
+
     # Validate shortname
     if not re.match(r'^[a-z0-9\-\_]+$', shortname):
-        raise Exception(f'{shortname} must match [a-z0-9\\-_]+')
+        raise Invalid(f'{shortname} must match [a-z0-9\\-_]+')
 
     # Validate name
     if not isinstance(name, str):
-        raise Exception(f'Invalid: {shortname}[name] must be a string')
+        raise Invalid(f'Invalid: {shortname}[name] must be a string')
 
     # Validate longitude
     if not isinstance(longitude, float):
-        raise Exception(f'Invalid: {shortname}[longitude] must be a float')
+        raise Invalid(f'Invalid: {shortname}[longitude] must be a float')
 
     # Validate latitude
     if not isinstance(latitude, float):
-        raise Exception(f'Invalid: {shortname}[latitude] must be a float')
+        raise Invalid(f'Invalid: {shortname}[latitude] must be a float')
 
     # Validate timezone
     if not isinstance(tz, str):
-        raise Exception(f'Invalid: {shortname}[timezone] must be a string')
+        raise Invalid(f'Invalid: {shortname}[timezone] must be a string')
 
     # Validate time
     for day, hours in time.items():
         if day not in ['Monday', 'Tuesday', 'Wednesday',
                        'Thursday', 'Friday', 'Saturday', 'Sunday']:
-            raise Exception(f'Invalid day in {shortname}[time]')
+            raise Invalid(f'Invalid day in {shortname}[time]')
         if not isinstance(hours, list):
-            raise Exception(f'Invalid: {shortname}[time] must be dict/lists')
+            raise Invalid(f'Invalid: {shortname}[time] must be dict/lists')
 
     # Validate place
     if place:
         if not isinstance(place, str):
-            raise Exception(f'Invalid: {shortname}[place] must be a string')
+            raise Invalid(f'Invalid: {shortname}[place] must be a string')
         nf['place'] = place
 
     # Validate address
     if address:
         if not isinstance(address, str):
-            raise Exception(f'Invalid: {shortname}[address] must be a string')
+            raise Invalid(f'Invalid: {shortname}[address] must be a string')
         nf['address'] = address
 
     # Validate note
     if note:
         if not isinstance(note, str):
-            raise Exception(f'Invalid: {shortname}[note] must be a string')
+            raise Invalid(f'Invalid: {shortname}[note] must be a string')
         nf['note'] = note
 
     # Validate types
     if types:
         if not isinstance(types, list):
-            raise Exception(f'Invalid: {shortname}[types] must be a list')
+            raise Invalid(f'Invalid: {shortname}[types] must be a list')
         nf['types'] = types
 
     # Validate keywords
     if keywords:
         if not isinstance(keywords, str):
-            raise Exception(f'Invalid: {shortname}[keywords] must be a string')
+            raise Invalid(f'Invalid: {shortname}[keywords] must be a string')
         nf['keywords'] = keywords
 
     # Return normalized and validated data