diff --git a/wayslack.py b/wayslack.py index 8bbed25..a4638b1 100755 --- a/wayslack.py +++ b/wayslack.py @@ -282,7 +282,7 @@ def iter_incomplete(self): class Downloader(object): - def __init__(self, token, path): + def __init__(self, token, path, no_download=False): self.counter = 0 self.token = token self.path = path @@ -299,6 +299,7 @@ def __init__(self, token, path): for item in pending: self.pool.put(item) atexit.register(self._write_pending) + self.no_download = no_download def _write_pending(self): try: @@ -374,6 +375,8 @@ def _download_path(self, url): return self.path / url_to_filename(url) def add(self, urls): + if self.no_download: + return for _, url in urls: download_path = self._download_path(url) if not download_path.exists(): @@ -893,6 +896,7 @@ class SlackArchive(object): def __init__(self, slack, opts): self.opts = opts self.dir = opts["dir"] + self.download_files = opts.get("download_files", True) self.slack = slack self.path = pathlib.Path(self.dir) self.emoji = ArchiveEmoji(self, self.path / "_emoji") @@ -914,7 +918,11 @@ def __init__(self, slack, opts): ] def __enter__(self): - self.downloader = Downloader(self.slack.api.token, self.path / "_files" / "storage") + self.downloader = Downloader( + self.slack.api.token, + self.path / "_files" / "storage", + no_download=not self.download_files, + ) return self def __exit__(self, *a): @@ -1030,6 +1038,8 @@ def main(argv=None): delete_old_files: 60 days - dir: second-export token: xoxp-9876-wxyz + # Do not download any files; only download conversation text. + download_files: false """ parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=""" diff --git a/wayslack2sql.py b/wayslack2sql.py index 33b6bc9..35e77fb 100755 --- a/wayslack2sql.py +++ b/wayslack2sql.py @@ -62,11 +62,14 @@ def insert(table, defaults, values=None): defaults.pop('_original', None) for v in values: - engine.execute(table.insert(), fix_timestamps_inplace(extend( - {'_original': v}, - defaults, - v, - ))) + try: + engine.execute(table.insert(), fix_timestamps_inplace(extend( + {'_original': v}, + defaults, + v, + ))) + except Exception as e: + print e metadata = sa.MetaData()