From 264881395583ea8341b3a827624ea76b47e3b99b Mon Sep 17 00:00:00 2001 From: Greg Sleap Date: Wed, 26 Jul 2023 16:11:25 +1000 Subject: [PATCH] Added voltage download submission option. Fixes #12 --- CHANGELOG.md | 4 ++ Cargo.lock | 2 +- Cargo.toml | 3 +- README.md | 26 ++++++++++++- src/asvo/error.rs | 2 +- src/asvo/mod.rs | 56 +++++++++++++++++++++++++--- src/bin/giant-squid.rs | 85 ++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 167 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b98b88..4c96237 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.7.0 - 2023-07-26 + +* support submission of voltage download jobs + ## 0.6.0 - 2023-07-04 * enable hash validation by default diff --git a/Cargo.lock b/Cargo.lock index 1c21535..7102ded 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -783,7 +783,7 @@ dependencies = [ [[package]] name = "mwa_giant_squid" -version = "0.6.0" +version = "0.7.0" dependencies = [ "anyhow", "backoff", diff --git a/Cargo.toml b/Cargo.toml index abce168..41d82e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,11 @@ [package] name = "mwa_giant_squid" -version = "0.6.0" +version = "0.7.0" authors = [ "Christopher H. Jordan ", "Harrison Barlow ", "Dev Null ", + "Greg Sleap ", ] edition = "2018" rust-version = "1.63" diff --git a/README.md b/README.md index 9b0efe1..104d16b 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ suit users for a few reasons: 4. One can ask `giant-squid` to print their ASVO queue as JSON; this makes parsing the state of your jobs in another programming language much simpler. -5. By default, `giant-squid` will validate the hash of the archive. You can skip +5. By default, `giant-squid` will validate the hash of the archive. You can skip this check with `--skip-hash` ## Usage @@ -193,7 +193,7 @@ Text files containing job IDs or obsids may be used too. By default, `giant-squid` will perform stream unzipping. Disable this with `-k` (or `--keep-zip`). -The MWA ASVO provides a SHA-1 of its downloads. `giant-squid` will verify the integrity +The MWA ASVO provides a SHA-1 of its downloads. `giant-squid` will verify the integrity of your download by default. Give a `--skip-hash` to the `download` command to skip. Jobs which were submitted with the /astro data delivery option behave differently @@ -291,6 +291,28 @@ or you can request that the files be left on Pawsey's /astro filesystem. The sec that your Pawsey group be set in your ASVO account, please contact an admin to request this. To submit a job with the /astro option, set the environment variable GIANT_SQUID_DELIVERY=astro. +#### Voltage downloads + +A "voltage download job" refers to a job which provides the raw voltages for one or more obsids. + +To submit a voltage download job for the obsid 1065880128: + +```bash +giant-squid submit-volt --delivery astro --offset 0 --duration 8 1065880128 +# or +giant-squid sv -d astro -o 0 -u 8 1065880128 +``` + +Text files containing obsids may be used too. + +If you want to check that your command works without actually submitting the +obsids, then you can use the `--dry-run` option (short version `-n`). + +Unlike other jobs, you cannot choose to have your files tarred up and uploaded to Pawsey's Acacia for remote +download, as the data is generally too large. If you are in the `mwaops` or `mwavcs` Pawsey groups and you have asked an MWA ASVO admin to +set the pawsey group in your MWA ASVO profile, you can request that the files be left on Pawsey's /astro filesystem. To submit +a job with the /astro option, set the environment variable GIANT_SQUID_DELIVERY=astro or pass `-d astro`. + ## Download performance By default, when downloading, `giant-squid` will store 100 MiB of the download diff --git a/src/asvo/error.rs b/src/asvo/error.rs index 1d1c462..9b5b5f9 100644 --- a/src/asvo/error.rs +++ b/src/asvo/error.rs @@ -123,7 +123,7 @@ pub enum AsvoError { #[error("Could not determine path for job {job_id:?}")] NoPath { job_id: u32 }, - // Error determining path for Astro job + // file type error for job #[error("Invalid file type for job {job_id:?}")] InvalidFileType { job_id: u32 }, } diff --git a/src/asvo/mod.rs b/src/asvo/mod.rs index b7ff050..721d111 100644 --- a/src/asvo/mod.rs +++ b/src/asvo/mod.rs @@ -183,15 +183,14 @@ impl AsvoClient { let op = || { self.try_download(url, keep_tar, hash, f, job) - .map_err(|e| { - match &e { - &AsvoError::IO(_) => Error::permanent(e), - _ => Error::transient(e), - } + .map_err(|e| match &e { + &AsvoError::IO(_) => Error::permanent(e), + _ => Error::transient(e), }) }; - if let Err(Error::Permanent(err)) = retry(ExponentialBackoff::default(), op) { + if let Err(Error::Permanent(err)) = retry(ExponentialBackoff::default(), op) + { return Err(err); } @@ -345,6 +344,30 @@ impl AsvoClient { self.submit_asvo_job(&AsvoJobType::DownloadVisibilities, form) } + /// Submit an ASVO job for voltage download. + pub fn submit_volt( + &self, + obsid: Obsid, + delivery: Delivery, + offset: i32, + duration: i32, + ) -> Result { + debug!("Submitting a voltage job to ASVO"); + + let obsid_str = format!("{}", obsid); + let d_str = format!("{}", delivery); + let offset_str: String = format!("{}", offset); + let duration_str: String = format!("{}", duration); + + let mut form = BTreeMap::new(); + form.insert("obs_id", obsid_str.as_str()); + form.insert("delivery", &d_str); + form.insert("offset", &offset_str); + form.insert("duration", &duration_str); + form.insert("download_type", "volt"); + self.submit_asvo_job(&AsvoJobType::DownloadVoltage, form) + } + /// Submit an ASVO job for conversion. pub fn submit_conv( &self, @@ -400,6 +423,7 @@ impl AsvoClient { let api_path = match job_type { AsvoJobType::Conversion => "conversion_job", AsvoJobType::DownloadVisibilities | AsvoJobType::DownloadMetadata => "download_vis_job", + AsvoJobType::DownloadVoltage => "voltage_job", jt => return Err(AsvoError::UnsupportedType(jt.clone())), }; @@ -525,4 +549,24 @@ mod tests { }, } } + + #[test] + fn test_submit_volt() { + let client = AsvoClient::new().unwrap(); + // NOTE: this obs_id is a voltage observation, however for this test to pass, + // You must have your pawsey_group set in your MWA ASVO profile to mwaops or mwavcs (contact an Admin to have this done). + let obs_id = Obsid::validate(1290094336).unwrap(); + let offset: i32 = 0; // This will attempt to get data from GPS TIME: 1290094336 + let duration: i32 = 1; // This will attempt to get data up to GPS TIME: 1290094336 + let delivery = Delivery::Astro; + + let volt_job = client.submit_volt(obs_id, delivery, offset, duration); + match volt_job { + Ok(_) => (), + Err(error) => match error { + AsvoError::BadStatus { code, message: _ } => println!("Got return code {}", code), + _ => panic!("Unexpected error has occured."), + }, + } + } } diff --git a/src/bin/giant-squid.rs b/src/bin/giant-squid.rs index d9bf6b8..95284bf 100644 --- a/src/bin/giant-squid.rs +++ b/src/bin/giant-squid.rs @@ -189,6 +189,43 @@ enum Args { obsids: Vec, }, + /// Submit ASVO jobs to download MWA voltages + #[clap(alias = "sv")] + SubmitVolt { + /// Tell the ASVO where to deliver the job. The only valid value for a voltage + /// job is "astro". + #[clap(short, long)] + delivery: Option, + + /// The offset in seconds from the start GPS time of the observation. + #[clap(short, long)] + offset: i32, + + /// The duration (in seconds) to download. + #[clap(short = 'u', long)] + duration: i32, + + /// Do not exit giant-squid until the specified obsids are ready for + /// download. + #[clap(short, long)] + wait: bool, + + /// Don't actually submit; print information on what would've happened + /// instead. + #[clap(short = 'n', long)] + dry_run: bool, + + /// The verbosity of the program. The default is to print high-level + /// information. + #[clap(short, long, parse(from_occurrences))] + verbosity: u8, + + /// The obsids to be submitted. Files containing obsids are also + /// accepted. + #[clap(name = "OBSID")] + obsids: Vec, + }, + /// Wait for ASVO jobs to complete, return the urls #[clap(alias = "w")] Wait { @@ -514,6 +551,54 @@ fn main() -> Result<(), anyhow::Error> { } } + Args::SubmitVolt { + delivery, + offset, + duration, + wait, + dry_run, + verbosity, + obsids, + } => { + let (parsed_jobids, parsed_obsids) = parse_many_jobids_or_obsids(&obsids)?; + // There shouldn't be any job IDs here. + if !parsed_jobids.is_empty() { + bail!( + "Expected only obsids, but found these exceptions: {:?}", + parsed_jobids + ); + } + if parsed_obsids.is_empty() { + bail!("No obsids specified!"); + } + init_logger(verbosity); + + let delivery = Delivery::validate(delivery)?; + debug!("Using {} for delivery", delivery); + + if dry_run { + info!( + "Would have submitted {} obsids for voltage download.", + obsids.len() + ); + } else { + let client = AsvoClient::new()?; + let mut jobids: Vec = Vec::with_capacity(obsids.len()); + for o in parsed_obsids { + let j = client.submit_volt(o, delivery, offset, duration)?; + info!("Submitted {} as ASVO job ID {}", o, j); + jobids.push(j); + } + info!("Submitted {} obsids for voltage download.", obsids.len()); + + if wait { + // Endlessly loop over the newly-supplied job IDs until + // they're all ready. + wait_loop(&client, &jobids)?; + } + } + } + Args::Wait { verbosity, jobs,