Implemented goko log scraping, puts it into .all.tar.bz2 and .bz2.tar…

… files like the former iso ones.

scrape.py

@@ -6,6 +6,7 @@
 import datetime
 import glob
 import logging
+import shutil
 import os
 import os.path
 import subprocess
@@ -14,19 +15,27 @@
 import time
 import urllib
 import utils
+import re
 
 # if the size of the game log is less than this assume we got an error page
 SMALL_FILE_SIZE = 5000 
 
-DEBUG = False
+DEBUG = True
 
 GOOD = 0
 MISSING = 1
 ERROR = 2
 DOWNLOADED = 3
 REPACKAGED = 4
 
-# make I should just adopt the isotropic format for consistency?
+CR_SOURCE = 5
+GOKO_SOURCE = 6
+ISO_SOURCE = 7
+
+GOKO_LOG_RE = re.compile('"(log.\w+.\w+.txt)"', re.MULTILINE)
+
+# Councilroom format is more similar to old isotropic format. 
+GOKO_FORMAT = '%(year)d%(month)02d%(day)02d/'
 ISOTROPIC_FORMAT =   '%(year)d%(month)02d/%(day)02d/all.tar.bz2'
 COUNCILROOM_FORMAT = '%(year)d%(month)02d%(day)02d/%(year)d%(month)02d%(day)02d.all.tar.bz2'
 
@@ -39,6 +48,13 @@ def IsotropicGamesCollectionUrl(cur_date):
     host = 'http://dominion.isotropic.org/gamelog/'
     return host + FormatDate(ISOTROPIC_FORMAT, cur_date)
 
+def GokoGamesCollectionUrl(cur_date):
+    host = 'http://dominionlogs.goko.com/'
+    return host+FormatDate(GOKO_FORMAT, cur_date)
+
+def GokoSingleGameUrl(cur_date, cur_game):
+    return GokoGamesCollectionUrl(cur_date)+cur_game
+
 def CouncilroomGamesCollectionUrl(cur_date):
     host = 'http://councilroom.com/static/scrape_data/'
     return host + FormatDate(COUNCILROOM_FORMAT, cur_date)
@@ -51,25 +67,50 @@ def RemoveSmallFileIfExists(fn):
 
 def download_date(str_date, cur_date, saved_games_bundle):
     urls_by_priority = [
-                        CouncilroomGamesCollectionUrl(cur_date),
-                        IsotropicGamesCollectionUrl(cur_date),
+                        (CR_SOURCE, CouncilroomGamesCollectionUrl(cur_date)),
+                        (GOKO_SOURCE, GokoGamesCollectionUrl(cur_date)),
+                        (ISO_SOURCE, IsotropicGamesCollectionUrl(cur_date))
                         ] 
 
-    for url in urls_by_priority:
+    for (source, url) in urls_by_priority:
         if DEBUG:
             print 'getting', saved_games_bundle, 'at', url
 
-        contents = urllib.urlopen(url).read()
+        try:
+            contents = urllib.urlopen(url).read()
+        except IOError:
+            contents = "0"
+
         if len(contents) > SMALL_FILE_SIZE:
             if DEBUG:
                 print 'yay, success from', url, 'no more requests for', \
                     str_date, 'needed'
-            open(saved_games_bundle, 'w').write(contents)
+            if source == CR_SOURCE or source == ISO_SOURCE:
+                open(saved_games_bundle, 'w').write(contents)
+            elif source == GOKO_SOURCE:
+                games = re.findall(GOKO_LOG_RE, contents)
+                bundle_goko_games(cur_date, games, saved_games_bundle)
             return True
         elif DEBUG:
             print 'request to', url, 'failed to find large file'
     return False
 
+def bundle_goko_games(cur_date, games, saved_games_bundle):
+    directory_name = tempfile.mkdtemp()
+    for cur_game in games:
+        url = GokoSingleGameUrl(cur_date, cur_game)
+        game_text = urllib.urlopen(url).read()
+        open(os.path.join(directory_name,cur_game),'w').write(game_text)
+
+    try:
+        subprocess.check_call(["tar", "-cjf", saved_games_bundle, "-C" ,
+                               directory_name] + games)
+    except subprocess.CalledProcessError, e:  
+        # Not handling this, just re-raise
+        logging.warning("Unexpected return from tar compressing goko output >>{msg}<<".format(msg=e.output))
+        raise
+    shutil.rmtree(directory_name)
+
 def unzip_date(directory, filename):
     os.chdir(directory)
     cmd = 'tar -xjvf %s >/dev/null 2>/dev/null'%filename
@@ -95,12 +136,12 @@ def repackage_archive(filename):
 
     Game archives are distributed as .tar.bz2 (a bzip2-compressed tar
     archive). For speed of serving, we repackage them as .bz2.tar (a
-    tar archive of bzip2-compressed HTML files). The .bz2.tar file is
+    tar archive of bzip2-compressed HTML or text files). The .bz2.tar file is
     a good bit larger, but an individual file can be extracted,
     decompressed, and served to a client in tenths of a second instead
     of tens of seconds. At the same time, storage space is still
     dramatically smaller than a raw folder of uncompressed (or even
-    compressed) HTML files.
+    compressed) HTML or text files.
     """
 
     orig_dir = os.getcwd()
@@ -118,7 +159,7 @@ def repackage_archive(filename):
 
     # Compress all the game*.html files
     os.chdir(directory_name)
-    game_files = glob.glob("game*.html")
+    game_files = glob.glob("game*.html")+glob.glob("log*.txt")
     if len(game_files) > 0:
         try:
             subprocess.check_call(["bzip2"] + game_files)
@@ -134,7 +175,7 @@ def repackage_archive(filename):
     # Tar the results back to the directory where the original file
     # came from
     dest_filename = repackage_filename(source_filename)
-    game_files = glob.glob("game*.html.bz2")
+    game_files = glob.glob("game*.html.bz2")+glob.glob("log*.txt.bz2")
     try:
         subprocess.check_call(["tar", "--remove", "-cf", dest_filename+".part"] + game_files)
     except subprocess.CalledProcessError, e:  #(retcode, cmd, output=output)
@@ -163,11 +204,10 @@ def scrape_date(str_date, cur_date, passive=False):
 
         if passive:
             return_code = MISSING
-
         elif not download_date(str_date, cur_date, saved_games_bundle):
             return_code = ERROR
-
-        return_code = DOWNLOADED
+        else:
+            return_code = DOWNLOADED
 
     # Repackage an existing file, if found
     if utils.at_least_as_big_as(saved_games_bundle, SMALL_FILE_SIZE) and \