From 4926dc14c4f16c7702950494b7efbab48021514a Mon Sep 17 00:00:00 2001
From: Rafael Steil <rafaelsteil@gmail.com>
Date: Mon, 17 May 2021 23:35:46 -0400
Subject: [PATCH] Add command to export log files to CSV

---
 src/plotman/csv_exporter.py | 98 +++++++++++++++++++++++++++++++++++++
 src/plotman/plotinfo.py     |  4 ++
 src/plotman/plotman.py      | 15 +++++-
 3 files changed, 115 insertions(+), 2 deletions(-)
 create mode 100644 src/plotman/csv_exporter.py

diff --git a/src/plotman/csv_exporter.py b/src/plotman/csv_exporter.py
new file mode 100644
index 00000000..9babe0de
--- /dev/null
+++ b/src/plotman/csv_exporter.py
@@ -0,0 +1,98 @@
+import csv
+import sys
+from plotman.log_parser import PlotLogParser
+
+def export(logfilenames, save_to = None):
+    if save_to is None:
+        __send_to_stdout(logfilenames)
+    else:
+        __save_to_file(logfilenames, save_to)
+
+def __save_to_file(logfilenames, filename: str):
+    with open(filename, 'w') as file:
+        __generate(logfilenames, file)
+
+def __send_to_stdout(logfilenames):
+    __generate(logfilenames, sys.stdout)
+
+def __generate(logfilenames, out):
+    writer = csv.writer(out)
+    writer.writerow([
+        'Plot ID', 
+        'Start date',
+        'Size',
+        'Buffer',
+        'Buckets',
+        'Threads',
+        'Tmp dir 1',
+        'Tmp dir 2',
+        'Phase 1 duration (raw)',
+        'Phase 1 duration',
+        'Phase 1 duration (minutes)',
+        'Phase 1 duration (hours)',
+        'Phase 2 duration (raw)',
+        'Phase 2 duration',
+        'Phase 2 duration (minutes)',
+        'Phase 2 duration (hours)',
+        'Phase 3 duration (raw)',
+        'Phase 3 duration',
+        'Phase 3 duration (minutes)',
+        'Phase 3 duration (hours)',
+        'Phase 4 duration (raw)',
+        'Phase 4 duration',
+        'Phase 4 duration (minutes)',
+        'Phase 4 duration (hours)',
+        'Total time (raw)',
+        'Total time',
+        'Total time (minutes)',
+        'Total time (hours)',
+        'Copy time (raw)',
+        'Copy time',
+        'Copy time (minutes)',
+        'Copy time (hours)',
+        'Filename'
+    ])
+
+    parser = PlotLogParser()
+
+    for filename in logfilenames:
+        info = parser.parse(filename)
+        
+        if info.is_empty():
+            continue
+
+        writer.writerow([
+            info.plot_id,
+            info.started_at,
+            info.plot_size,
+            info.buffer,
+            info.buckets,
+            info.threads,
+            info.tmp_dir1,
+            info.tmp_dir2,
+            info.phase1_duration_raw,
+            info.phase1_duration,
+            info.phase1_duration_minutes,
+            info.phase1_duration_hours,
+            info.phase2_duration_raw,
+            info.phase2_duration,
+            info.phase2_duration_minutes,
+            info.phase2_duration_hours,
+            info.phase3_duration_raw,
+            info.phase3_duration,
+            info.phase3_duration_minutes,
+            info.phase3_duration_hours,
+            info.phase4_duration_raw,
+            info.phase4_duration,
+            info.phase4_duration_minutes,
+            info.phase4_duration_hours,
+            info.total_time_raw,
+            info.total_time,
+            info.total_time_minutes,
+            info.total_time_hours,
+            info.copy_time_raw,
+            info.copy_time,
+            info.copy_time_minutes,
+            info.copy_time_hours,
+            info.filename
+        ])
diff --git a/src/plotman/plotinfo.py b/src/plotman/plotinfo.py
index 1ebc5700..96d21250 100644
--- a/src/plotman/plotinfo.py
+++ b/src/plotman/plotinfo.py
@@ -19,6 +19,10 @@ class PlotInfo:
     copy_time_raw: float = 0
     filename: str = ""
 
+    def is_empty(self) -> bool:
+      """Data is considered empty if no copy_time was found"""
+      return self.copy_time == 0
+
     # Phase 1 duration
     @property
     def phase1_duration(self) -> int:
diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py
index 35e1616f..326fb0c8 100755
--- a/src/plotman/plotman.py
+++ b/src/plotman/plotman.py
@@ -2,13 +2,14 @@
 import importlib
 import importlib.resources
 import os
+import glob
 import random
 from shutil import copyfile
 import time
 import datetime
 
 # Plotman libraries
-from plotman import analyzer, archive, configuration, interactive, manager, plot_util, reporting
+from plotman import analyzer, archive, configuration, interactive, manager, plot_util, reporting, csv_exporter
 from plotman import resources as plotman_resources
 from plotman.job import Job
 
@@ -39,6 +40,9 @@ def parse_args(self):
 
         sp.add_parser('archive', help='move completed plots to farming location')
 
+        p_export_csv = sp.add_parser('export_csv', help='exports metadata from the plot logs as CSV')
+        p_export_csv.add_argument('-o', dest='save_to', default=None, type=str, help='save to file. Optional, prints to stdout by default')
+
         p_config = sp.add_parser('config', help='display or generate plotman.yaml configuration')
         sp_config = p_config.add_subparsers(dest='config_subcommand')
         sp_config.add_parser('generate', help='generate a default plotman.yaml file and print path')
@@ -159,8 +163,15 @@ def main():
         analyzer.analyze(args.logfile, args.clipterminals,
                 args.bytmp, args.bybitfield)
 
+    # 
+    # Exports log metadata to CSV
+    #
+    elif args.cmd == 'export_csv':
+        logfilenames = glob.glob(os.path.join(cfg.directories.log, '*'))
+        csv_exporter.export(logfilenames, args.save_to)
+
     else:
-        jobs = Job.get_running_jobs(cfg.directories.log)
+        jobs = Job.get_running_jobs(cfg.directories.log())
 
         # Status report
         if args.cmd == 'status':