Merge pull request #13 from MWATelescope/voltage_job_support

Added voltage download submission option. Fixes #12

CHANGELOG.md

@@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic
 Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 0.7.0 - 2023-07-26
+
+* support submission of voltage download jobs
+
 ## 0.6.0 - 2023-07-04
 
 * enable hash validation by default

Cargo.toml

@@ -1,10 +1,11 @@
 [package]
 name = "mwa_giant_squid"
-version = "0.6.0"
+version = "0.7.0"
 authors = [
     "Christopher H. Jordan <[email protected]>",
     "Harrison Barlow <[email protected]>",
     "Dev Null <[email protected]>",
+    "Greg Sleap <[email protected]>",
 ]
 edition = "2018"
 rust-version = "1.63"

README.md

@@ -32,7 +32,7 @@ suit users for a few reasons:
 4. One can ask `giant-squid` to print their ASVO queue as JSON; this makes
    parsing the state of your jobs in another programming language much simpler.
 
-5. By default, `giant-squid` will validate the hash of the archive. You can skip 
+5. By default, `giant-squid` will validate the hash of the archive. You can skip
    this check with `--skip-hash`
 
 ## Usage
@@ -193,7 +193,7 @@ Text files containing job IDs or obsids may be used too.
 By default, `giant-squid` will perform stream unzipping. Disable this with `-k`
 (or `--keep-zip`).
 
-The MWA ASVO provides a SHA-1 of its downloads. `giant-squid` will verify the integrity 
+The MWA ASVO provides a SHA-1 of its downloads. `giant-squid` will verify the integrity
 of your download by default. Give a `--skip-hash` to the `download` command to skip.
 
 Jobs which were submitted with the /astro data delivery option behave differently
@@ -291,6 +291,28 @@ or you can request that the files be left on Pawsey's /astro filesystem. The sec
 that your Pawsey group be set in your ASVO account, please contact an admin to request this. To submit
 a job with the /astro option, set the environment variable GIANT_SQUID_DELIVERY=astro.
 
+#### Voltage downloads
+
+A "voltage download job" refers to a job which provides the raw voltages for one or more obsids.
+
+To submit a voltage download job for the obsid 1065880128:
+
+```bash
+giant-squid submit-volt --delivery astro --offset 0 --duration 8 1065880128
+# or
+giant-squid sv -d astro -o 0 -u 8 1065880128
+```
+
+Text files containing obsids may be used too.
+
+If you want to check that your command works without actually submitting the
+obsids, then you can use the `--dry-run` option (short version `-n`).
+
+Unlike other jobs, you cannot choose to have your files tarred up and uploaded to Pawsey's Acacia for remote
+download, as the data is generally too large. If you are in the `mwaops` or `mwavcs` Pawsey groups and you have asked an MWA ASVO admin to
+set the pawsey group in your MWA ASVO profile, you can request that the files be left on Pawsey's /astro filesystem. To submit
+a job with the /astro option, set the environment variable GIANT_SQUID_DELIVERY=astro or pass `-d astro`.
+
 ## Download performance
 
 By default, when downloading, `giant-squid` will store 100 MiB of the download

src/asvo/error.rs

@@ -123,7 +123,7 @@ pub enum AsvoError {
     #[error("Could not determine path for job {job_id:?}")]
     NoPath { job_id: u32 },
 
-    // Error determining path for Astro job
+    // file type error for job
     #[error("Invalid file type for job {job_id:?}")]
     InvalidFileType { job_id: u32 },
 }

src/asvo/mod.rs

@@ -183,15 +183,14 @@ impl AsvoClient {
 
                         let op = || {
                             self.try_download(url, keep_tar, hash, f, job)
-                                .map_err(|e| {
-                                    match &e {
-                                        &AsvoError::IO(_) => Error::permanent(e),
-                                        _ => Error::transient(e),
-                                    }
+                                .map_err(|e| match &e {
+                                    &AsvoError::IO(_) => Error::permanent(e),
+                                    _ => Error::transient(e),
                                 })
                         };
 
-                        if let Err(Error::Permanent(err)) = retry(ExponentialBackoff::default(), op) {
+                        if let Err(Error::Permanent(err)) = retry(ExponentialBackoff::default(), op)
+                        {
                             return Err(err);
                         }
 
@@ -345,6 +344,30 @@ impl AsvoClient {
         self.submit_asvo_job(&AsvoJobType::DownloadVisibilities, form)
     }
 
+    /// Submit an ASVO job for voltage download.
+    pub fn submit_volt(
+        &self,
+        obsid: Obsid,
+        delivery: Delivery,
+        offset: i32,
+        duration: i32,
+    ) -> Result<AsvoJobID, AsvoError> {
+        debug!("Submitting a voltage job to ASVO");
+
+        let obsid_str = format!("{}", obsid);
+        let d_str = format!("{}", delivery);
+        let offset_str: String = format!("{}", offset);
+        let duration_str: String = format!("{}", duration);
+
+        let mut form = BTreeMap::new();
+        form.insert("obs_id", obsid_str.as_str());
+        form.insert("delivery", &d_str);
+        form.insert("offset", &offset_str);
+        form.insert("duration", &duration_str);
+        form.insert("download_type", "volt");
+        self.submit_asvo_job(&AsvoJobType::DownloadVoltage, form)
+    }
+
     /// Submit an ASVO job for conversion.
     pub fn submit_conv(
         &self,
@@ -400,6 +423,7 @@ impl AsvoClient {
         let api_path = match job_type {
             AsvoJobType::Conversion => "conversion_job",
             AsvoJobType::DownloadVisibilities | AsvoJobType::DownloadMetadata => "download_vis_job",
+            AsvoJobType::DownloadVoltage => "voltage_job",
             jt => return Err(AsvoError::UnsupportedType(jt.clone())),
         };
 
@@ -525,4 +549,24 @@ mod tests {
             },
         }
     }
+
+    #[test]
+    fn test_submit_volt() {
+        let client = AsvoClient::new().unwrap();
+        // NOTE: this obs_id is a voltage observation, however for this test to pass,
+        // You must have your pawsey_group set in your MWA ASVO profile to mwaops or mwavcs (contact an Admin to have this done).
+        let obs_id = Obsid::validate(1290094336).unwrap();
+        let offset: i32 = 0; // This will attempt to get data from GPS TIME: 1290094336
+        let duration: i32 = 1; // This will attempt to get data up to GPS TIME: 1290094336
+        let delivery = Delivery::Astro;
+
+        let volt_job = client.submit_volt(obs_id, delivery, offset, duration);
+        match volt_job {
+            Ok(_) => (),
+            Err(error) => match error {
+                AsvoError::BadStatus { code, message: _ } => println!("Got return code {}", code),
+                _ => panic!("Unexpected error has occured."),
+            },
+        }
+    }
 }

src/bin/giant-squid.rs

@@ -189,6 +189,43 @@ enum Args {
         obsids: Vec<String>,
     },
 
+    /// Submit ASVO jobs to download MWA voltages
+    #[clap(alias = "sv")]
+    SubmitVolt {
+        /// Tell the ASVO where to deliver the job. The only valid value for a voltage
+        /// job is "astro".
+        #[clap(short, long)]
+        delivery: Option<String>,
+
+        /// The offset in seconds from the start GPS time of the observation.
+        #[clap(short, long)]
+        offset: i32,
+
+        /// The duration (in seconds) to download.
+        #[clap(short = 'u', long)]
+        duration: i32,
+
+        /// Do not exit giant-squid until the specified obsids are ready for
+        /// download.
+        #[clap(short, long)]
+        wait: bool,
+
+        /// Don't actually submit; print information on what would've happened
+        /// instead.
+        #[clap(short = 'n', long)]
+        dry_run: bool,
+
+        /// The verbosity of the program. The default is to print high-level
+        /// information.
+        #[clap(short, long, parse(from_occurrences))]
+        verbosity: u8,
+
+        /// The obsids to be submitted. Files containing obsids are also
+        /// accepted.
+        #[clap(name = "OBSID")]
+        obsids: Vec<String>,
+    },
+
     /// Wait for ASVO jobs to complete, return the urls
     #[clap(alias = "w")]
     Wait {
@@ -514,6 +551,54 @@ fn main() -> Result<(), anyhow::Error> {
             }
         }
 
+        Args::SubmitVolt {
+            delivery,
+            offset,
+            duration,
+            wait,
+            dry_run,
+            verbosity,
+            obsids,
+        } => {
+            let (parsed_jobids, parsed_obsids) = parse_many_jobids_or_obsids(&obsids)?;
+            // There shouldn't be any job IDs here.
+            if !parsed_jobids.is_empty() {
+                bail!(
+                    "Expected only obsids, but found these exceptions: {:?}",
+                    parsed_jobids
+                );
+            }
+            if parsed_obsids.is_empty() {
+                bail!("No obsids specified!");
+            }
+            init_logger(verbosity);
+
+            let delivery = Delivery::validate(delivery)?;
+            debug!("Using {} for delivery", delivery);
+
+            if dry_run {
+                info!(
+                    "Would have submitted {} obsids for voltage download.",
+                    obsids.len()
+                );
+            } else {
+                let client = AsvoClient::new()?;
+                let mut jobids: Vec<AsvoJobID> = Vec::with_capacity(obsids.len());
+                for o in parsed_obsids {
+                    let j = client.submit_volt(o, delivery, offset, duration)?;
+                    info!("Submitted {} as ASVO job ID {}", o, j);
+                    jobids.push(j);
+                }
+                info!("Submitted {} obsids for voltage download.", obsids.len());
+
+                if wait {
+                    // Endlessly loop over the newly-supplied job IDs until
+                    // they're all ready.
+                    wait_loop(&client, &jobids)?;
+                }
+            }
+        }
+
         Args::Wait {
             verbosity,
             jobs,