Upgrades

CIPP/Makefile

@@ -0,0 +1,2 @@
+test:
+	python -m unittest

CIPP/README.rst

@@ -6,12 +6,61 @@ CIPPtools
 
 ---------------------------------
 
-These are a very loose collection of Python programs for helping someone 
+These are a very loose collection of Python 3 programs for helping someone 
 serving as a HiRISE CIPP perform various tasks.
 
+
 * Free software: Apache Software License 2.0
 
+Conversion
+----------
+The ``ptf2csv.py`` and ``csv2ptf.py`` function mostly like the Perl
+equivalents that have been in use by HiRISE for a decade.
+
+The difference is that ``ptf2csv.py`` has an option to add a
+'HiReport Link' column to the end of the output CSV file.  This
+column contains a formula that when read in by most spreadsheet
+programs will result in a clickable link in that cell of your
+spreadsheet to allow easy checking of HiReport.
+
+And ``csv2ptf.py`` basically ignores any non-PTF columns in your
+.csv file (like maybe that HiReport column that you had ``ptf2csv.py``
+put in, or any other columns that you might have added).
+
+
+Working with the HiTList
+------------------------
+``prioritize_by_orbit.py`` can be used on the HiTList you get from
+your HiTS to clearly flag (by changing their existing priority from
+positive to negative) which lower-priority observations in each
+orbit are 'excluded' by the latitude-exclusion zone (defaults to
+40 degrees in latitude on either side of an observation) of higher
+priority observations.
+
+``priority_rewrite.py`` can be used near the end of your process when you
+have a bunch of observations that all have the same priority that each need
+a unique priority.  This program takes that block of entries, and assigns unique
+priorities based on latitude.
+
+``orbit_count.py`` again, a program of the same name as a Perl program that we have.
+The difference here is that this one is 'aware' of the possible negative priorities
+given by ``prioritize_by_orbit.py``, prints out an observation count histogram (how many 
+orbits have 3 observations, etc.) Although it doesn't report data volume like
+the Perl verison does (but it could).
+
+
+TOS
+---
+What made it through TOS?  Did my WTH make it through TOS?  Did my WTH even make
+it into the HiTLIST?  Did you elminiate my WTH from the HiTList, you monster?
+
+Or replace 'WTH' with HiKERs or CaSSIS targets or really any list of suggestions
+that you want to know are contained in a PTF.
+
+All of these questions can be answered with a PTF, text copied from the WTH list
+wiki page, and ``tos_success.py``.
+
 
 WARNING
 -------
-**You'll notice there aren't any tests, be warned!**
+**There are some tests, but user beware.**

CIPP/orbit_count.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python
+"""Scans a PTF evaluating the state of priorities and performing counts."""
+
+# Copyright 2019, Ross A. Beyer ([email protected])
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import argparse
+import logging
+from collections import Counter
+from itertools import groupby
+
+import priority_rewrite as pr
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('in_file', help="a .ptf or .csv file")
+
+    args = parser.parse_args()
+
+    logging.basicConfig(format='%(levelname)s: %(message)s')
+
+    ptf_in = pr.get_input(args.in_file)
+
+    print('\n'.join(format_report(orbit_count(ptf_in))))
+
+
+def orbit_count(records: list) -> str:
+    reports = list()
+
+    sorted_by_o = sorted(records,
+                         key=lambda x: int(x['Orbit Number'][:-1]))
+    for orbit, g in groupby(sorted_by_o,
+                            key=lambda x: int(x['Orbit Number'][:-1])):
+        report_dict = {'orbit': orbit, 'pos': 0, 'neg': 0}
+
+        for rec in g:
+            if int(rec['Request Priority']) > 0:
+                report_dict['pos'] += 1
+            else:
+                report_dict['neg'] += 1
+
+        reports.append(report_dict)
+
+    return reports
+
+
+def format_report(records: list) -> list:
+    formatted_lines = list()
+
+    # Prepare header and set widths
+    header = {'orbit': 'Orbit ({})'.format(len(records)),
+              'pos': '# obs', 'neg': '# negative obs'}
+    o_width = len(header['orbit'])
+    p_width = len(header['pos'])
+    n_width = len(header['neg'])
+    rules = {'orbit': '-' * o_width,
+             'pos': '-' * p_width,
+             'neg': '-' * n_width}
+
+    # Accumulate counts and fuss with removing zeros from output.
+    pos_counts = Counter()
+    neg_count = 0
+    str_reports = list()
+    for r in records:
+        str_d = {'orbit': str(r['orbit']), 'pos': '', 'neg': ''}
+        pos_counts.update((r['pos'],))
+        neg_count += r['neg']
+        for pn in ('pos', 'neg'):
+            if r[pn] == 0:
+                str_d[pn] = ''
+            else:
+                str_d[pn] = str(r[pn])
+        str_reports.append(str_d)
+
+    # The meat of formatting each line of the report:
+    lines_to_format = [header, rules] + str_reports
+    for d in lines_to_format:
+        o_str = '{orb:<{width}}'.format(orb=d['orbit'], width=o_width)
+        p_str = '{pos:^{width}}'.format(pos=d['pos'], width=p_width)
+        n_str = '{neg:<{width}}'.format(neg=d['neg'], width=n_width)
+        formatted_lines.append(f'{o_str} {p_str} {n_str}')
+
+    # Summary line at the bottom:
+    count_summ = '{label:-^{width}}'.format(label='Counts', width=o_width)
+    count_summ += ' {} {}'.format(rules['pos'], rules['neg'])
+    formatted_lines.append(count_summ)
+    pos_count = sum(k * v for k, v in pos_counts.items())
+    t_sum = '{sum:^#{o_width}}'.format(sum=pos_count + neg_count,
+                                       o_width=o_width)
+    t_pos = f'{pos_count:^#{p_width}}'
+    t_neg = f'{neg_count:<#{n_width}}'
+    formatted_lines.append('{}={}+ {}'.format(t_sum, t_pos, t_neg))
+    formatted_lines.append('')
+
+    num_of_obs = list()
+    num_of_orbs = list()
+    for k in sorted(list(pos_counts.keys()), reverse=True):
+        width = len(str(pos_counts[k]))
+        if width < 3:
+            width = 3
+        num_of_obs.append(f'{k:^#{width}}')
+        num_of_orbs.append(f'{pos_counts[k]:^#{width}}')
+
+    # formatted_lines.append('Orbit Count Histogram {}'.format(' '.join()))
+    formatted_lines.append('# of Observations: {}'.format(' '.join(num_of_obs)))
+    formatted_lines.append('# of Orbits      : {}'.format(' '.join(num_of_orbs)))
+    formatted_lines.append('')
+
+    # Set up the empty orbit report
+    empty_orbits = find_empty_orbits(records)
+    formatted_lines.append('Empty Orbits')
+    formatted_lines.append('------------')
+
+    return formatted_lines + list(map(str, empty_orbits))
+
+
+def find_empty_orbits(records: list) -> list:
+    orbs = list(map(lambda x: x['orbit'], records))
+    return sorted(set(range(orbs[0], orbs[-1])) - set(orbs))
+
+
+if __name__ == "__main__":
+    main()

CIPP/prioritize_by_orbit.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python
+"""Scans a PTF, grouping records by orbit, and attempts to deconflict
+   observations in that orbit.
+
+   This is NOT a substitute for reviewing the PTF yourself.
+
+   The algorithm will examine each orbit (based on the PTF Orbit Number
+   field, ignoring any Orbit Alternatives), and will guarantee that
+   at most, only the user-specified number of observations will remain
+   as positive priorities after it runs, all other records will be set
+   to negative priorities (allowing inspection afterwards).
+
+   For each orbit, the alrgorithm will begin with the highest
+   prioritized observation, and deprioritize any observations within
+   the latitude exclusion range in either direction, then find the
+   next highest, and so on.  When faced with observations that have
+   the same priority, it will give preference to the observation
+   that is closest to the equator.
+"""
+
+# Copyright 2020, Ross A. Beyer ([email protected])
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# TODO: write in a mechanism that allows the user to optimize for more
+# observations per orbit, rather than strictly giving the highest priority
+# plus the lowest latitude the top spot.  Doing so may 'exclude' two other
+# observations, and if one had the same priority, but didn't 'exclude' the
+# other, you might be able to fit more observations on an orbit.
+#
+# TODO: may also want something other than abs(latitude) to be a driver,
+# possibly time since coming out of eclipse or something.  Lots of
+# possibilities.
+
+import argparse
+import logging
+
+from itertools import groupby
+
+import priority_rewrite as pr
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('-o', '--output', required=False)
+    parser.add_argument('--per_orbit', required=False, default=4,
+                        help="The number of observations to keep in"
+                             "an orbit.")
+    parser.add_argument('--latitude_exclude', required=False, default=40,
+                        help="The amount of latitude on either side "
+                             "(so it is a half-width) of an observation "
+                             "that will be excluded for consideration.")
+    parser.add_argument('-n', '--dry_run', required=False,
+                        action='store_true', help='Perform the rearranging '
+                        'but do not write out results.')
+    parser.add_argument('in_file', help="a .ptf or .csv file")
+
+    args = parser.parse_args()
+
+    logging.basicConfig(format='%(levelname)s: %(message)s')
+
+    ptf_in = pr.get_input(args.in_file)
+
+    new_ptf_records = prioritize_by_orbit(ptf_in,
+                                          args.per_orbit,
+                                          args.latitude_exclude)
+
+    # This sorting ignores the 'a' or 'd' markers on Orbits.
+    new_ptf_records.sort(key=lambda x: int(x['Orbit Number'][:-1]))
+
+    if args.dry_run:
+        pass
+    else:
+        out_str = pr.write_output(ptf_in, new_ptf_records, args.output)
+        if out_str:
+            print(out_str)
+
+
+class intervals(object):
+
+    def __init__(self, half_width: float):
+        self.intervals = list()  # a list of two-tuples
+        self.half_width = float(half_width)
+
+    def add(self, point: float):
+        p = float(point)
+        new_interval = ((p - self.half_width), (p + self.half_width))
+        intervals = self.intervals + [new_interval]
+
+        # This interval merging logic is from
+        # https://codereview.stackexchange.com/questions/69242/merging-overlapping-intervals
+        sorted_by_lower_bound = sorted(intervals, key=lambda x: x[0])
+        merged = list()
+
+        for higher in sorted_by_lower_bound:
+            if not merged:
+                merged.append(higher)
+            else:
+                lower = merged[-1]
+                # test for intersection between lower and higher:
+                # we know via sorting that lower[0] <= higher[0]
+                if higher[0] <= lower[1]:
+                    upper_bound = max(lower[1], higher[1])
+                    # replace by merged interval:
+                    merged[-1] = (lower[0], upper_bound)
+        else:
+            merged.append(higher)
+
+        self.intervals = merged
+        return
+
+    def is_in(self, point: float):
+        p = float(point)
+        for i in self.intervals:
+            if p >= i[0] and p <= i[1]:
+                return True
+        else:
+            return False
+
+
+def prioritize_by_orbit(records: list, observations=4,
+                        latitude_exclude=40) -> list:
+    '''Rewrites priorities by orbit.'''
+
+    new_records = list()
+    sorted_by_o = sorted(records,
+                         key=lambda x: int(x['Orbit Number'][:-1]))
+    for orbit, g in groupby(sorted_by_o,
+                            key=lambda x: int(x['Orbit Number'][:-1])):
+        exclude = intervals(latitude_exclude)
+        obs_count = 0
+        by_orbit = list(g)
+        by_orbit.sort(key=lambda x: int(x['Request Priority']), reverse=True)
+        for pri, pri_g in groupby(by_orbit,
+                                  key=lambda x: int(x['Request Priority'])):
+            recs = list(pri_g)
+            if len(recs) != 1:
+                # need to prioritize these by latitude
+                recs = pr.priority_rewrite(recs, keepzero=True)
+
+            for r in sorted(recs, key=lambda x: int(x['Request Priority']),
+                            reverse=True):
+                if(obs_count < observations and
+                   not exclude.is_in(r['Latitude'])):
+                    exclude.add(r['Latitude'])
+                    obs_count += 1
+                    r['Request Priority'] = pri
+                else:
+                    r['Request Priority'] = -1 * pri
+
+                new_records.append(r)
+
+    return new_records
+
+
+if __name__ == "__main__":
+    main()