Added option to override file names (fix for brtmr/podfox#20)

podfox/__init__.py

@@ -7,7 +7,7 @@
     podfox.py update [<shortname>] [-c=<path>]
     podfox.py feeds [-c=<path>]
     podfox.py episodes <shortname> [-c=<path>]
-    podfox.py download [<shortname> --how-many=<n>] [-c=<path>]
+    podfox.py download [<shortname> --how-many=<n>] [--rename-files] [-c=<path>]
     podfox.py rename <shortname> <newname> [-c=<path>]
 
 Options:
@@ -20,7 +20,9 @@
 from colorama import Fore, Back, Style
 from docopt import docopt
 from os.path import expanduser
+from urllib.parse import urlparse
 from sys import exit
+from tqdm import tqdm
 import colorama
 import feedparser
 import json
@@ -29,6 +31,10 @@
 import requests
 import sys
 import re
+import concurrent.futures
+import threading
+import logging
+logging.basicConfig(level=logging.WARNING)
 
 # RSS datetimes follow RFC 2822, same as email headers.
 # this is the chain of stackoverflow posts that led me to believe this is true.
@@ -38,7 +44,7 @@
 # how-to-parse-a-rfc-2822-date-time-into-a-python-datetime
 
 from email.utils import parsedate
-from time import mktime
+from time import mktime, localtime, strftime
 
 CONFIGURATION_DEFAULTS = {
     "podcast-directory": "~/Podcasts",
@@ -202,31 +208,60 @@ def episodes_from_feed(d):
     return episodes
 
 
-def download_multiple(feed, maxnum):
-    for episode in feed['episodes']:
-        if maxnum == 0:
-            break
-        if not episode['downloaded']:
-            download_single(feed['shortname'], episode['url'])
-            episode['downloaded'] = True
-            maxnum -= 1
+def download_multiple(feed, maxnum, rename):
+    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+        # parse up to maxnum of the not downloaded episodes
+        future_to_episodes = {}
+        for episode in list(filter(lambda ep: not ep['downloaded'], feed['episodes']))[:maxnum]:
+            filename = ""
+
+            if rename:
+                title = episode['title']
+                for c in '<>\"|*%?\\/': 
+                    title = title.replace(c, "")
+                title = title.replace(" ", "_").replace("’", "'").replace("—", "-").replace(":", ".")
+                # Shorten the title to max 120 characters
+                title = title[:120]
+                extension = os.path.splitext(urlparse(episode['url'])[2])[1]
+                filename = "{}_{}{}".format(strftime('%Y-%m-%d', localtime(episode['published'])),
+                                            title, extension)
+
+
+            future_to_episodes[executor.submit(download_single, feed['shortname'], episode['url'], filename)]=episode
+
+        for future in concurrent.futures.as_completed(future_to_episodes):
+            episode = future_to_episodes[future]
+            try:
+                episode['downloaded'] = future.result()    
+            except Exception as exc:
+                print('%r generated an exception: %s' % (episode['title'], exc))
     overwrite_config(feed)
 
 
-def download_single(folder, url):
-    print(url)
+def download_single(folder, url, filename=""):
+    logging.info("{}: Parsing URL {}".format(threading.current_thread().name, url))
     base = CONFIGURATION['podcast-directory']
     r = requests.get(url.strip(), stream=True)
+    if not filename:
+        try:
+            filename=re.findall('filename="([^"]+)',r.headers['content-disposition'])[0]
+        except:
+            filename = url.split('/')[-1]
+            filename = filename.split('?')[0]
+    logging.info("{}: {:s} downloading".format(threading.current_thread().name, filename))
+
     try:
-        filename=re.findall('filename="([^"]+)',r.headers['content-disposition'])[0]
-    except:
-        filename = url.split('/')[-1]
-        filename = filename.split('?')[0]
-    print_green("{:s} downloading".format(filename))
-    with open(os.path.join(base, folder, filename), 'wb') as f:
-        for chunk in r.iter_content(chunk_size=1024**2):
-            f.write(chunk)
-    print("done.")
+        with open(os.path.join(base, folder, filename), 'wb') as f:
+            pbar = tqdm(total=int(r.headers['Content-Length']), unit='B', unit_scale=True, unit_divisor=1024)
+            pbar.set_description(filename if len(filename)<20 else filename[:20])
+            for chunk in r.iter_content(chunk_size=1024**2):
+                f.write(chunk)
+                pbar.update(len(chunk))
+    except EnvironmentError:
+        print_err("{}: Error while writing {}".format(threading.current_thread().name, filename))
+        return False
+    logging.info("{}: done.".format(threading.current_thread().name))
+    return True
 
 
 def available_feeds():
@@ -317,6 +352,9 @@ def main():
     CONFIGURATION.update(userconf)
     CONFIGURATION['podcast-directory'] = os.path.expanduser(CONFIGURATION['podcast-directory'])
 
+    # Check if we should use the progress bar
+
+
     #handle the commands
     if arguments['import']:
         if arguments['<shortname>'] is None:
@@ -356,19 +394,20 @@ def main():
             maxnum = int(arguments['--how-many'])
         else:
             maxnum = CONFIGURATION['maxnum']
+        rename_files = bool(arguments['--rename-files'])
         #download episodes for a specific feed
         if arguments['<shortname>']:
             feed = find_feed(arguments['<shortname>'])
             if feed:
-                download_multiple(feed, maxnum)
+                download_multiple(feed, maxnum, rename_files)
                 exit(0)
             else:
                 print_err("feed {} not found".format(arguments['<shortname>']))
                 exit(-1)
         #download episodes for all feeds.
         else:
             for feed in available_feeds():
-                download_multiple(feed,  maxnum)
+                download_multiple(feed, maxnum, rename_files)
             exit(0)
     if arguments['rename']:
         rename(arguments['<shortname>'], arguments['<newname>'])

requirements.txt

@@ -2,3 +2,4 @@ colorama==0.3.7
 docopt==0.6.2
 feedparser==5.2.1
 requests==2.20.0
+tqdm=4.48.2

setup.py

@@ -4,7 +4,7 @@
     required = f.read().splitlines()
 
 setup(name='podfox',
-    version='0.1.1',
+    version='0.1.2',
     description='Podcatcher for the terminal',
     url='http://github.com/brtmr/podfox',
     author='Bastian Reitemeier',