diff --git a/antarctica_today/__init__.py b/antarctica_today/__init__.py index e69de29..e869860 100644 --- a/antarctica_today/__init__.py +++ b/antarctica_today/__init__.py @@ -0,0 +1,12 @@ +import os + +# IMPORTANT: If we don't specify this setting, then the projection we want to use will +# be replaced with another (and this warning will be printed)! +# +# Warning 1: CRS EPSG:3411 is deprecated. Its non-deprecated replacement EPSG:3413 will be +# used instead. To use the original CRS, set the OSR_USE_NON_DEPRECATED configuration +# option to NO. +# Warning 1: CRS EPSG:3412 is deprecated. Its non-deprecated replacement EPSG:3976 will be +# used instead. To use the original CRS, set the OSR_USE_NON_DEPRECATED configuration +# option to NO. +os.environ["OSR_USE_NON_DEPRECATED"] = "NO" diff --git a/antarctica_today/__main__.py b/antarctica_today/__main__.py new file mode 100644 index 0000000..eaee86b --- /dev/null +++ b/antarctica_today/__main__.py @@ -0,0 +1,87 @@ +import click + +from antarctica_today.generate_daily_melt_file import generate_new_daily_melt_files +from antarctica_today.generate_gap_filled_melt_picklefile import ( + save_gap_filled_picklefile, +) +from antarctica_today.main import ( + generate_all_plots_and_maps_main, + preprocessing_main, +) +from antarctica_today.melt_array_picklefile import save_model_array_picklefile +from antarctica_today.nsidc_download_Tb_data import download_new_files + + +@click.group() +def cli(): + """Antarctica Today.""" + pass + + +@click.option( + "--start-date", + default="2022-01-10", + help="TODO", +) +@cli.command() +def download_tb(start_date: str): + """Download NSIDC-0080 brightness temperature granules. + + The default start date is the day after the end of the .bin data available in + `/data/daily_melt_bin_files/` directory in this repo. + """ + download_new_files(time_start=start_date) + + +@click.option( + "--start-date", + # TODO: Is this default right? The actual function we're calling has a default of + # 2021-10-01. Why? + default="2022-01-10", + help="TODO", +) +@cli.command() +def generate_daily_melt(start_date: str): + """Generate daily melt file from brightness temperature granules.""" + generate_new_daily_melt_files( + start_date=start_date, + overwrite=False, + ) + + +@cli.command +def melt_array_picklefile(): + """Is this needed operationally?""" + save_model_array_picklefile() + + +@cli.command +def gap_filled_melt_picklefile(): + """Is this needed operationally?""" + save_gap_filled_picklefile() + + +@cli.command() +def preprocess(): + """Perform pre-processing steps for Antarctica Today data. + + This includes: + - ... + - ... + """ + preprocessing_main() + + +@cli.command() +def process(): + """Perform processing steps for Antarctica Today data. + + This includes: + - ... + - ... + """ + generate_all_plots_and_maps_main() + + +if __name__ == "__main__": + cli() diff --git a/antarctica_today/compute_mean_climatology.py b/antarctica_today/compute_mean_climatology.py index 6e4d1cd..bc6957c 100644 --- a/antarctica_today/compute_mean_climatology.py +++ b/antarctica_today/compute_mean_climatology.py @@ -167,9 +167,10 @@ def compute_daily_climatology_pixel_averages( # Save out to a picklefile array. if output_picklefile != None: - f = open(output_picklefile, "wb") - pickle.dump((average_melt_array, baseline_dates_mmdd_dict), f) - f.close() + output_picklefile.parent.mkdir(exist_ok=True, parents=True) + with open(output_picklefile, "wb") as f: + pickle.dump((average_melt_array, baseline_dates_mmdd_dict), f) + if verbose: print(output_picklefile, "written.") diff --git a/antarctica_today/generate_daily_melt_file.py b/antarctica_today/generate_daily_melt_file.py index 4c23bf8..51d82fd 100644 --- a/antarctica_today/generate_daily_melt_file.py +++ b/antarctica_today/generate_daily_melt_file.py @@ -378,6 +378,3 @@ def create_daily_melt_array( # output_gtif_filename=gtif_name, # verbose=args.verbose, # ) - -if __name__ == "__main__": - generate_new_daily_melt_files(overwrite=False) diff --git a/antarctica_today/generate_gap_filled_melt_picklefile.py b/antarctica_today/generate_gap_filled_melt_picklefile.py index bbdeec4..673fca0 100644 --- a/antarctica_today/generate_gap_filled_melt_picklefile.py +++ b/antarctica_today/generate_gap_filled_melt_picklefile.py @@ -106,7 +106,3 @@ def fill_melt_array_with_interpolations(array=None, datetimes_dict=None, verbose # print("\t", "Day missing, fill with mean. Average:", numpy.sum(day_slice[day_slice!=-1])) return gap_filled_array, gap_filled_dt_dict - - -if __name__ == "__main__": - save_gap_filled_picklefile() diff --git a/antarctica_today/main.py b/antarctica_today/main.py index 81a1dd4..8428de3 100644 --- a/antarctica_today/main.py +++ b/antarctica_today/main.py @@ -110,35 +110,3 @@ def generate_all_plots_and_maps_main(): # outfile = fname_svg) # # TODO: Finish if you want. - - -@click.group() -def cli(): - """Antarctica Today.""" - pass - - -@cli.command() -def preprocess(): - """Perform pre-processing steps for Antarctica Today data. - - This includes: - - ... - - ... - """ - preprocessing_main() - - -@cli.command() -def process(): - """Perform processing steps for Antarctica Today data. - - This includes: - - ... - - ... - """ - generate_all_plots_and_maps_main() - - -if __name__ == "__main__": - cli() diff --git a/antarctica_today/melt_array_picklefile.py b/antarctica_today/melt_array_picklefile.py index f6219d9..d5b6018 100755 --- a/antarctica_today/melt_array_picklefile.py +++ b/antarctica_today/melt_array_picklefile.py @@ -334,8 +334,3 @@ def read_gap_filled_melt_picklefile( array, dt_dict = pickle.load(f) return array, dt_dict - - -if __name__ == "__main__": - # Let's save the v2.5 data from Tom's stuff. - array, dt_dict = save_model_array_picklefile() diff --git a/antarctica_today/nsidc_download_Tb_data.py b/antarctica_today/nsidc_download_Tb_data.py index ea9a2dd..ef5501b 100644 --- a/antarctica_today/nsidc_download_Tb_data.py +++ b/antarctica_today/nsidc_download_Tb_data.py @@ -318,8 +318,3 @@ def download_new_files( quit() return files_saved - - -if __name__ == "__main__": - # By default, start the following day after the .bin files end in the /data/daily_melt_bin_files/ directory. - download_new_files(time_start="2022-01-10") diff --git a/doc/operation.md b/doc/operation.md index 86fc4a3..805f82c 100644 --- a/doc/operation.md +++ b/doc/operation.md @@ -1,7 +1,4 @@ -# Operation - -> :warning: Warning -> +> [!WARNING] > This software requires a large amount of memory (min. 8GB), increasingly more as time > goes on, because the entire climatology is sometimes read in to memory. An error may > look like: @@ -12,122 +9,180 @@ > We should move to a tool like SQLite or XArray for managing this data without needing > so much memory. +# Operation + The one set of files that is required but not generated by this code are the 37h threshold binaries provided by Tom Mote. These are checked in to this repository. -> NOTE: +
🛠️ _TODO_ + +- [ ] Simpler commands/CLI, shouldn't have to know to set PYTHONPATH. +- [ ] Simplify steps; either consolidate or express step order in code, e.g. in CLI + with clear order. +- [ ] Convert from storage via picklefiles to NetCDF. See issue: + https://github.com/nsidc/Antarctica_Today/issues/19 + +
+ +> [!NOTE] > The path commands are for users working in a bash environment! For csh users, > the commands are: - setenv PYTHONPATH /location where antarctica_today lives - python antactica_today/program.py - -> ‼️ Important > -> Steps must be performed in order. +> ``` +> setenv PYTHONPATH /path/to/repository/Antarctica_Today +> python antactica_today/program.py +> ``` + +> [!IMPORTANT] +> :bangbang: Steps must be performed in order :bangbang: ## 1. Download NSIDC-0080 -Download all NSIDC-0080 granules: +Download NSIDC-0080 granules into the `Tb/` directory: -``` +```bash PYTHONPATH=. -python antarctica_today/nsidc_download_Tb_data.py +python antarctica_today download-tb ``` -> :memo: Note -> -> For data before 2016, `NSIDC-0001` and `NSIDC-0007` are used. These previous data have already been processed and are available as binary ".bin" files in the **/data/daily_melt_bin_files** directory. All new data are be calculated from the NSIDC-0080 "Near Real-Time DMSP SSM/SSMIS Daily Polar Gridded Brightness Temperature" product (https://nsidc.org/data/nsidc-0080/versions/2). +> [!IMPORTANT] +> For data before 2022-01-10, data have already been processed through step 2 and are +> available as binary ".bin" files in this repo's `/data/daily_melt_bin_files` +> directory. This data was generated from `NSIDC-0001` and `NSIDC-0007` datasets. > -> 🛠️ _TODO_ -> -> - [ ] Add functionality to CLI if needed. +> Data newer than that date are calculated freshly from the `NSIDC-0080` v2 product +> (https://nsidc.org/data/nsidc-0080/versions/2), which is in NetCDF format. This step +> downloads that raw data. ## 2. Generate all the daily melt binary files -``` +Generate new data in `data/daily_melt_bin_files/`: + +```bash PYTHONPATH=. -python antarctica_today/generate_daily_melt_file.py +python antarctica_today generate-daily-melt ``` +> [!IMPORTANT] +> Binaries provided in this repo's `data/daily_melt_bin_files/` dir with pre-2016 dates +> are already calibrated by Tom Mote and don't need to be generated. Remember from the +> note in the previous step: pre-generated data goes through to 2022-01-10. -> :memo: Note -> -> Binaries provided in `data/daily_melt_bin_files` already calibrated by -> Tom Mote for pre 2016 dates. This generates new daily melt files from the NSIDC-0080 data downloaded in the previous step. -> -> 🛠️ _TODO_ -> -> - [ ] Add functionality to CLI if needed. + +
🛠️ _TODO_ +I receive a large number of warnings like: + +``` +UserWarning: Warning: At least one NSIDC Tb file on date '20230909' is missing. Skipping +that date. +``` + +Why? +
## 3. Generate the database -This software manages a database covering the full climatology in the form of a pickle file. +This step creates, primarily, four pickle files: +* `daily_cumulative_melt_averages.pickle` +* `daily_melt_pixel_averages.pickle` +* `database/v3_1979-present_gap_filled.pickle` +* `database/v3_1979-present_raw.pickle` -> TODO: This this as simple as: -> -> ``` -> PYTHONPATH=. -> python antarctica_today/main.py preprocess -> ``` -> -> 🛠️ _TODO_ -> -> - [ ] Convert from storage via picklefiles to NetCDF. See issue https://github.com/nsidc/Antarctica_Today/issues/19 +Additionally: +* `.csv` files will be created in `database/` directory +* `.tif` files will be created in `data/mean_climatology/` directory +* `.tif` files will be created in `data/annual_*_geotifs` directories -### Initializing -> 🛠️ _TODO_ -> -> - [ ] Simpler command, shouldn't have to know to set PYTHONPATH. -> - [ ] Add functionality to CLI if needed. +```bash +PYTHONPATH=. +python antarctica_today preprocess +``` + +> [!NOTE] +> This command may take up to tens of minutes. + +
🛠️ _TODO_ + +- [ ] Why is the next section called "Initializing"? Are there multiple pickle files? + Does each command initialize one? Can we combine them all into one command? +- [ ] After this command, `git status` shows untracked files. Which should ignored? + Which should be committed? + + Untracked: + + ``` + database/baseline_percentiles_1990-2020.csv + database/baseline_percentiles_1990-2020_gap_filled.csv + database/daily_melt_totals.csv + database/daily_melt_totals_gap_filled.csv + ``` +
+ + +### Database initialization (?) + + +
🛠️ _TODO_ +Is this step necessary? It seems like new files aren't being created when this step is +run. +
Create the melt array picklefile, a file containing a 2d grid for each day: -``` +```bash PYTHONPATH=. -python antarctica_today/melt_array_picklefile.py +python antarctica_today melt-array-picklefile ``` -Create a gap-filled melt picklefile, This "fills the gaps" of missing data or missing days in the historical record with climatological averages. This is especially prevalent in the 1980s when composites are only tallied every other day. In the small 'pole hole' orbital gap, "no melt" (1) is filled. (NOTE: This assumption may need to be changed if melt ever reaches South Pole.) -``` +Create a gap-filled melt picklefile, This "fills the gaps" of missing data or missing +days in the historical record with climatological averages. This is especially prevalent +in the 1980s when composites are only tallied every other day. In the small 'pole hole' +orbital gap, "no melt" (1) is filled. (NOTE: This assumption may need to be changed if +melt ever reaches South Pole.) + +```bash PYTHONPATH=. -python antarctica_today/generate_gap_filled_melt_picklefile.py +python antarctica_today gap-filled-melt-picklefile ``` ### Daily updates + +> [!WARNING] +> All initialization steps above must be completed first. + This step will download any new Tb data files from NSIDC since its last run, and generate new plots from the last day's data (for all of Antartica and for each individual region), including: 1) A "daily melt" map of the most recent day's melt extent 2) A "sum" map of that season's total melt days 3) An "anomaly" map of that season's total melt days in comparison to baseline average values to-that-day-of-year 4) A line plot of melt extent up do that date, compared to historical baseline averages. It will copy these plots into a sub-directory /plots/daily_plots_gathered/[date]/ for easy collection. -> 🛠️ _TODO_ -> -> - [ ] Simpler command, shouldn't have to know to set PYTHONPATH. -> - [ ] Add functionality to CLI if needed. - -> :warning: Warning -> -> All initialization steps above must be completed first. -``` +```bash PYTHONPATH=. python antarctica_today/update_data.py ``` -## 4. Generate outputs -(Optional.) +## 4. Generate outputs (optional) -This will go through the entire database and produce summary maps and plots for every year on record. -Run the main CLI's `process` command. +> [!NOTE] +> This command may take up to tens of minutes. -``` +This will go through the entire database and produce summary maps and plots for every year on record in the `plots/` directory. + +```bash PYTHONPATH=. -python antarctica_today/main.py process +python antarctica_today process ``` + +
🛠️ _TODO_ + +- [ ] After this step, `git status` shows changed files. Should they be committed? + +