From fc74da862143641eb69adac897928ff4013bc00d Mon Sep 17 00:00:00 2001 From: Steuteville Date: Tue, 19 Dec 2023 14:58:12 -0700 Subject: [PATCH 01/25] adding new from_github vehicle method --- rust/fastsim-core/src/vehicle.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/rust/fastsim-core/src/vehicle.rs b/rust/fastsim-core/src/vehicle.rs index adcf98f3..9af5755d 100644 --- a/rust/fastsim-core/src/vehicle.rs +++ b/rust/fastsim-core/src/vehicle.rs @@ -6,6 +6,7 @@ use crate::params::*; use crate::proc_macros::{add_pyo3_api, doc_field, ApproxEq}; #[cfg(feature = "pyo3")] use crate::pyo3imports::*; +use crate::utils::create_project_subdir; use lazy_static::lazy_static; use regex::Regex; @@ -1033,6 +1034,23 @@ impl RustVehicle { v.set_derived().unwrap(); v } + /// Downloads specified vehicle from vehicle repo into fastsim/python/fastsim/resources/vehdb. + /// + /// Arguments: + /// ---------- + /// vehicle (string): name of the vehicle to be downloaded + + // make separate function that gets list of yaml files off of github, then user can choose from that for input -- Rust function + // exposed to python, put in utilities + pub fn from_github(vehicle_file_name: String) { + let mut vehicle_url: String = "https://github.com/NREL/fastsim-vehicles/blob/main/public/".to_string(); + vehicle_url.push_str(&vehicle_file_name); + // find or create fastsim data directory with subdirectory "vehicles" and return path + let data_directory: Option = create_project_subdir("vehicles"); + // see if you can find file_name in vehicle data + // if file doesn't already exist in directory, and once we have file path to directory: + // download_file_from_url(vehicle_url, data_directory) + } } impl Default for RustVehicle { From 5d2c1d4b331c58a87c9bd390e1ff6005bc22e8ca Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Thu, 21 Dec 2023 15:06:48 -0700 Subject: [PATCH 02/25] adding from_url and to_cache methods to SerdeAPI --- rust/fastsim-core/src/cycle.rs | 13 +++++++++++ rust/fastsim-core/src/imports.rs | 1 + rust/fastsim-core/src/lib.rs | 1 + rust/fastsim-core/src/traits.rs | 39 ++++++++++++++++++++++++++++++++ rust/fastsim-core/src/vehicle.rs | 12 ++++++++++ 5 files changed, 66 insertions(+) diff --git a/rust/fastsim-core/src/cycle.rs b/rust/fastsim-core/src/cycle.rs index 6f3f32d5..cf50eedc 100644 --- a/rust/fastsim-core/src/cycle.rs +++ b/rust/fastsim-core/src/cycle.rs @@ -726,6 +726,19 @@ impl SerdeAPI for RustCycle { ), } } + + /// takes an object from a url and saves it in the fastsim data directory in a rust_objects folder + /// WARNING: if there is a file already in the data subdirectory with the same name, it will be replaced by the new file + fn to_cache>(url: S) { + let url = url.as_ref(); + let url_parts: Vec<&str> = url.split("/").collect(); + let file_name = url_parts.last().unwrap_or_else(||panic!("Could not determine file name/type.")); + let data_subdirectory = create_project_subdir("cycles").unwrap_or_else(|_|panic!("Could not find or create Fastsim data subdirectory.")); + let file_path = data_subdirectory.join(file_name); + // I believe this will overwrite any existing files with the same name -- is this preferable, or should we add + // a bool argument so user can determine whether the file should overwrite an existing file or not + download_file_from_url(url, &file_path); + } } impl TryFrom>> for RustCycle { diff --git a/rust/fastsim-core/src/imports.rs b/rust/fastsim-core/src/imports.rs index f938a8cb..0b7bbf6b 100644 --- a/rust/fastsim-core/src/imports.rs +++ b/rust/fastsim-core/src/imports.rs @@ -13,3 +13,4 @@ pub(crate) use std::path::PathBuf; pub(crate) use crate::traits::*; pub(crate) use crate::utils::*; +pub(crate) use crate::vehicle_utils::*; diff --git a/rust/fastsim-core/src/lib.rs b/rust/fastsim-core/src/lib.rs index e6e6bd8d..67812580 100644 --- a/rust/fastsim-core/src/lib.rs +++ b/rust/fastsim-core/src/lib.rs @@ -29,6 +29,7 @@ //! ``` extern crate ndarray; +extern crate tempfile; #[macro_use] pub mod macros; diff --git a/rust/fastsim-core/src/traits.rs b/rust/fastsim-core/src/traits.rs index 5bdfaede..076dc45e 100644 --- a/rust/fastsim-core/src/traits.rs +++ b/rust/fastsim-core/src/traits.rs @@ -1,5 +1,6 @@ use crate::imports::*; use std::collections::HashMap; +use tempfile::tempdir; pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { const ACCEPTED_BYTE_FORMATS: &'static [&'static str] = &["yaml", "json", "bin"]; @@ -154,6 +155,44 @@ pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { fn from_bincode(encoded: &[u8]) -> anyhow::Result { Ok(bincode::deserialize(encoded)?) } + + /// instantiates an object from a url + fn from_url>(url: S) -> anyhow::Result { + let url = url.as_ref(); + let temp_dir = tempdir()?; + let mut file_path = PathBuf::new(); + // do these file types need to be specific to the object? + // TODO: either make funciton work for csv files, or remove from supported file list + if url.ends_with("yaml"){ + let file_path = temp_dir.path().join("temporary_object.yaml"); + } else if url.ends_with("csv"){ + let file_path = temp_dir.path().join("temporary_object.csv"); + } else if url.ends_with("json"){ + let file_path = temp_dir.path().join("temporary_object.json"); + } else { + bail!("Unsupported file type, must be a yaml, json, or csv file."); + } + download_file_from_url(url, &file_path); + // only works for json and yaml + // seems like I might also be able to use from_reader instead -- which one is preferable? + Self::from_file(file_path) + } + + /// takes an object from a url and saves it in the fastsim data directory in a rust_objects folder + /// WARNING: if there is a file already in the data subdirectory with the same name, it will be replaced by the new file + /// to save to a folder other than rust_objects for a specific object type, override this default + /// implementation for the Rust object, and in the object-specific implementation, replace + /// "rust_objects" with your choice of folder name + fn to_cache>(url: S) { + let url = url.as_ref(); + let url_parts: Vec<&str> = url.split("/").collect(); + let file_name = url_parts.last().unwrap_or_else(||panic!("Could not determine file name/type.")); + let data_subdirectory = create_project_subdir("rust_objects").unwrap_or_else(|_|panic!("Could not find or create Fastsim data subdirectory.")); + let file_path = data_subdirectory.join(file_name); + // I believe this will overwrite any existing files with the same name -- is this preferable, or should we add + // a bool argument so user can determine whether the file should overwrite an existing file or not + download_file_from_url(url, &file_path); + } } pub trait ApproxEq { diff --git a/rust/fastsim-core/src/vehicle.rs b/rust/fastsim-core/src/vehicle.rs index 9af5755d..acc74ece 100644 --- a/rust/fastsim-core/src/vehicle.rs +++ b/rust/fastsim-core/src/vehicle.rs @@ -1082,6 +1082,18 @@ impl SerdeAPI for RustVehicle { fn init(&mut self) -> anyhow::Result<()> { self.set_derived() } + /// takes an object from a url and saves it in the fastsim data directory in a rust_objects folder + /// WARNING: if there is a file already in the data subdirectory with the same name, it will be replaced by the new file + fn to_cache>(url: S) { + let url = url.as_ref(); + let url_parts: Vec<&str> = url.split("/").collect(); + let file_name = url_parts.last().unwrap_or_else(||panic!("Could not determine file name/type.")); + let data_subdirectory = create_project_subdir("vehicles").unwrap_or_else(|_|panic!("Could not find or create Fastsim data subdirectory.")); + let file_path = data_subdirectory.join(file_name); + // I believe this will overwrite any existing files with the same name -- is this preferable, or should we add + // a bool argument so user can determine whether the file should overwrite an existing file or not + download_file_from_url(url, &file_path); + } } #[cfg(test)] From 84754c3fff2e4b24dd7693f1f3924c6c956d4f90 Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Sun, 24 Dec 2023 22:51:10 -0500 Subject: [PATCH 03/25] fixing caching functions --- rust/fastsim-core/Cargo.toml | 1 + rust/fastsim-core/src/cycle.rs | 15 +------ rust/fastsim-core/src/lib.rs | 1 - rust/fastsim-core/src/traits.rs | 72 +++++++++++++++++++------------- rust/fastsim-core/src/utils.rs | 24 +++++++++++ rust/fastsim-core/src/vehicle.rs | 43 +++++++------------ 6 files changed, 87 insertions(+), 69 deletions(-) diff --git a/rust/fastsim-core/Cargo.toml b/rust/fastsim-core/Cargo.toml index 82d30c20..4678c5c5 100644 --- a/rust/fastsim-core/Cargo.toml +++ b/rust/fastsim-core/Cargo.toml @@ -40,6 +40,7 @@ include_dir = "0.7.3" itertools = "0.12.0" ndarray-stats = "0.5.1" tempfile = "3.8.1" +url = "2.5.0" [package.metadata] include = [ diff --git a/rust/fastsim-core/src/cycle.rs b/rust/fastsim-core/src/cycle.rs index cf50eedc..f87133ce 100644 --- a/rust/fastsim-core/src/cycle.rs +++ b/rust/fastsim-core/src/cycle.rs @@ -629,6 +629,8 @@ pub struct RustCycle { impl SerdeAPI for RustCycle { const ACCEPTED_BYTE_FORMATS: &'static [&'static str] = &["yaml", "json", "bin", "csv"]; const ACCEPTED_STR_FORMATS: &'static [&'static str] = &["yaml", "json", "csv"]; + // is this enough, or do I have to copy paste in the whole to_cache mathod? + const CACHE_FOLDER: &'static str = &"cycles"; // TODO: make this get called somewhere fn init(&mut self) -> anyhow::Result<()> { @@ -726,19 +728,6 @@ impl SerdeAPI for RustCycle { ), } } - - /// takes an object from a url and saves it in the fastsim data directory in a rust_objects folder - /// WARNING: if there is a file already in the data subdirectory with the same name, it will be replaced by the new file - fn to_cache>(url: S) { - let url = url.as_ref(); - let url_parts: Vec<&str> = url.split("/").collect(); - let file_name = url_parts.last().unwrap_or_else(||panic!("Could not determine file name/type.")); - let data_subdirectory = create_project_subdir("cycles").unwrap_or_else(|_|panic!("Could not find or create Fastsim data subdirectory.")); - let file_path = data_subdirectory.join(file_name); - // I believe this will overwrite any existing files with the same name -- is this preferable, or should we add - // a bool argument so user can determine whether the file should overwrite an existing file or not - download_file_from_url(url, &file_path); - } } impl TryFrom>> for RustCycle { diff --git a/rust/fastsim-core/src/lib.rs b/rust/fastsim-core/src/lib.rs index 67812580..e6e6bd8d 100644 --- a/rust/fastsim-core/src/lib.rs +++ b/rust/fastsim-core/src/lib.rs @@ -29,7 +29,6 @@ //! ``` extern crate ndarray; -extern crate tempfile; #[macro_use] pub mod macros; diff --git a/rust/fastsim-core/src/traits.rs b/rust/fastsim-core/src/traits.rs index 076dc45e..79d0db9d 100644 --- a/rust/fastsim-core/src/traits.rs +++ b/rust/fastsim-core/src/traits.rs @@ -5,6 +5,7 @@ use tempfile::tempdir; pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { const ACCEPTED_BYTE_FORMATS: &'static [&'static str] = &["yaml", "json", "bin"]; const ACCEPTED_STR_FORMATS: &'static [&'static str] = &["yaml", "json"]; + const CACHE_FOLDER: &'static str = &"rust_objects"; /// Runs any initialization steps that might be needed fn init(&mut self) -> anyhow::Result<()> { @@ -156,42 +157,57 @@ pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { Ok(bincode::deserialize(encoded)?) } - /// instantiates an object from a url + /// instantiates an object from a url + /// accepts yaml and json file types + /// # Arguments + /// - url: url (either as a string or url type) to object fn from_url>(url: S) -> anyhow::Result { let url = url.as_ref(); + let url_parts: Vec<&str> = url.split(".").collect(); + let file_extension = url_parts.last().unwrap_or_else(||panic!("Could not determine file type.")).to_lowercase().as_str(); + ensure!( + Self::ACCEPTED_STR_FORMATS.contains(&file_extension) || file_extension == "yml", + "Unsupported format {file_extension:?}, must be one of {:?}", + Self::ACCEPTED_STR_FORMATS + ); + let file_name = "temporary_object.".to_string() + file_extension; let temp_dir = tempdir()?; - let mut file_path = PathBuf::new(); - // do these file types need to be specific to the object? - // TODO: either make funciton work for csv files, or remove from supported file list - if url.ends_with("yaml"){ - let file_path = temp_dir.path().join("temporary_object.yaml"); - } else if url.ends_with("csv"){ - let file_path = temp_dir.path().join("temporary_object.csv"); - } else if url.ends_with("json"){ - let file_path = temp_dir.path().join("temporary_object.json"); - } else { - bail!("Unsupported file type, must be a yaml, json, or csv file."); - } + let file_path = temp_dir.path().join(file_name); download_file_from_url(url, &file_path); - // only works for json and yaml - // seems like I might also be able to use from_reader instead -- which one is preferable? + // seems like I might also be able to use from_reader instead -- which + // one is preferable? + // try switching over to from_reader(), this will require finding + // a crate which takes a url and spits out a Rust object + // that implements std::io::Read Self::from_file(file_path) } - /// takes an object from a url and saves it in the fastsim data directory in a rust_objects folder - /// WARNING: if there is a file already in the data subdirectory with the same name, it will be replaced by the new file - /// to save to a folder other than rust_objects for a specific object type, override this default - /// implementation for the Rust object, and in the object-specific implementation, replace - /// "rust_objects" with your choice of folder name - fn to_cache>(url: S) { - let url = url.as_ref(); - let url_parts: Vec<&str> = url.split("/").collect(); - let file_name = url_parts.last().unwrap_or_else(||panic!("Could not determine file name/type.")); - let data_subdirectory = create_project_subdir("rust_objects").unwrap_or_else(|_|panic!("Could not find or create Fastsim data subdirectory.")); + /// takes an instantiated Rust object and saves it in the FASTSim data directory in + /// a rust_objects folder + /// WARNING: if there is a file already in the data subdirectory with the + /// same name, it will be replaced by the new file + /// to save to a folder other than rust_objects for a specific object type, + /// in the object-specific SerdeAPI implementation, redefine the + /// CACHE_FOLDER constant to be your choice of folder name + /// # Arguments + /// - self (rust object) + /// - file_path: path to file within subdirectory. If only the file name is + /// listed, file will sit directly within the subdirectory of + /// the FASTSim data directory. If a path is given, the file will live + /// within the path specified, within the subdirectory CACHE_FOLDER of the + /// FASTSim data directory. + fn to_cache>(&self, file_path: P) -> anyhow::Result<()> { + let file_name = file_path.as_ref().file_name().with_context(||"Could not determine file name")?.to_str().context("Could not determine file name.")?; + let mut subpath = PathBuf::new(); + let file_path_internal = file_path.as_ref().to_str().context("Could not determine file name.")?; + if file_name == file_path_internal { + subpath = PathBuf::from(Self::CACHE_FOLDER); + } else { + subpath = Path::new(Self::CACHE_FOLDER).join(file_path_internal.strip_suffix(file_name).context("Could not determine path to subdirectory.")?); + } + let data_subdirectory = create_project_subdir(subpath).with_context(||"Could not find or build Fastsim data subdirectory.")?; let file_path = data_subdirectory.join(file_name); - // I believe this will overwrite any existing files with the same name -- is this preferable, or should we add - // a bool argument so user can determine whether the file should overwrite an existing file or not - download_file_from_url(url, &file_path); + self.to_file(file_path) } } diff --git a/rust/fastsim-core/src/utils.rs b/rust/fastsim-core/src/utils.rs index 80681efa..98cb560c 100644 --- a/rust/fastsim-core/src/utils.rs +++ b/rust/fastsim-core/src/utils.rs @@ -7,6 +7,7 @@ use ndarray::*; use ndarray_stats::QuantileExt; use regex::Regex; use std::collections::HashSet; +use url::{Url, Host, Position}; use crate::imports::*; #[cfg(feature = "pyo3")] @@ -525,6 +526,29 @@ pub fn create_project_subdir>(subpath: P) -> anyhow::Result, P: AsRef>(url: S, subpath: P) -> anyhow::Result<()> { + let url = Url::parse(url.as_ref())?; + let file_name = url + .path_segments() + .and_then(|segments| segments.last()) + .with_context(|| "Could not parse filename from URL: {url:?}")?; + let data_subdirectory = create_project_subdir(subpath).with_context(||"Could not find or build Fastsim data subdirectory.")?; + let file_path = data_subdirectory.join(file_name); + download_file_from_url(url.as_ref(), &file_path)?; + Ok(()) +} + #[cfg(feature = "pyo3")] pub mod array_wrappers { use crate::proc_macros::add_pyo3_api; diff --git a/rust/fastsim-core/src/vehicle.rs b/rust/fastsim-core/src/vehicle.rs index acc74ece..ee0092d0 100644 --- a/rust/fastsim-core/src/vehicle.rs +++ b/rust/fastsim-core/src/vehicle.rs @@ -546,6 +546,7 @@ pub struct RustVehicle { /// RustVehicle rust methods impl RustVehicle { + const VEHICLE_DIRECTORY_URL: &'static str = &"https://github.com/NREL/fastsim-vehicles/blob/main/public/"; /// Sets the following parameters: /// - `ess_mass_kg` /// - `mc_mass_kg` @@ -1034,22 +1035,19 @@ impl RustVehicle { v.set_derived().unwrap(); v } - /// Downloads specified vehicle from vehicle repo into fastsim/python/fastsim/resources/vehdb. - /// - /// Arguments: - /// ---------- - /// vehicle (string): name of the vehicle to be downloaded - // make separate function that gets list of yaml files off of github, then user can choose from that for input -- Rust function - // exposed to python, put in utilities - pub fn from_github(vehicle_file_name: String) { - let mut vehicle_url: String = "https://github.com/NREL/fastsim-vehicles/blob/main/public/".to_string(); - vehicle_url.push_str(&vehicle_file_name); - // find or create fastsim data directory with subdirectory "vehicles" and return path - let data_directory: Option = create_project_subdir("vehicles"); - // see if you can find file_name in vehicle data - // if file doesn't already exist in directory, and once we have file path to directory: - // download_file_from_url(vehicle_url, data_directory) + /// Downloads specified vehicle from vehicle repo into + /// fastsim/python/fastsim/resources/vehdb. + /// # Arguments + /// - vehicle_file_name: file name for vehicle to be downloaded + /// - url: url for vehicle to be downloaded, if None, assumed to be + /// downloaded from vehicle FASTSim repo + /// - cache: If True, function will first check for vehicle in FASTSim data + /// directory, and if vehicle is already there, will use local version. If + /// vehicle is not stored locally, will download and store vehicle for later + /// use. If False, will not check for vehicle locally or store vehicle + /// locally for later use + pub fn from_github_or_url>(vehicle_file_name: S, url: Option, cache: bool) { } } @@ -1079,21 +1077,12 @@ impl Default for RustVehicle { } impl SerdeAPI for RustVehicle { + // is this enough, or do I have to copy paste in the whole to_cache mathod? + const CACHE_FOLDER: &'static str = &"vehicles"; + fn init(&mut self) -> anyhow::Result<()> { self.set_derived() } - /// takes an object from a url and saves it in the fastsim data directory in a rust_objects folder - /// WARNING: if there is a file already in the data subdirectory with the same name, it will be replaced by the new file - fn to_cache>(url: S) { - let url = url.as_ref(); - let url_parts: Vec<&str> = url.split("/").collect(); - let file_name = url_parts.last().unwrap_or_else(||panic!("Could not determine file name/type.")); - let data_subdirectory = create_project_subdir("vehicles").unwrap_or_else(|_|panic!("Could not find or create Fastsim data subdirectory.")); - let file_path = data_subdirectory.join(file_name); - // I believe this will overwrite any existing files with the same name -- is this preferable, or should we add - // a bool argument so user can determine whether the file should overwrite an existing file or not - download_file_from_url(url, &file_path); - } } #[cfg(test)] From c81eca640e63bbda7eee938e774456198a46993c Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Tue, 26 Dec 2023 15:31:44 -0500 Subject: [PATCH 04/25] fixing from_url function --- rust/fastsim-core/Cargo.toml | 1 + rust/fastsim-core/src/traits.rs | 154 ++++++++++++++++++++++++++++---- 2 files changed, 139 insertions(+), 16 deletions(-) diff --git a/rust/fastsim-core/Cargo.toml b/rust/fastsim-core/Cargo.toml index 4678c5c5..9dd19f08 100644 --- a/rust/fastsim-core/Cargo.toml +++ b/rust/fastsim-core/Cargo.toml @@ -41,6 +41,7 @@ itertools = "0.12.0" ndarray-stats = "0.5.1" tempfile = "3.8.1" url = "2.5.0" +ureq = "2.9.1" [package.metadata] include = [ diff --git a/rust/fastsim-core/src/traits.rs b/rust/fastsim-core/src/traits.rs index 79d0db9d..db8f26ea 100644 --- a/rust/fastsim-core/src/traits.rs +++ b/rust/fastsim-core/src/traits.rs @@ -1,6 +1,7 @@ use crate::imports::*; use std::collections::HashMap; -use tempfile::tempdir; +use ureq; +use url::Url; pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { const ACCEPTED_BYTE_FORMATS: &'static [&'static str] = &["yaml", "json", "bin"]; @@ -162,24 +163,145 @@ pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { /// # Arguments /// - url: url (either as a string or url type) to object fn from_url>(url: S) -> anyhow::Result { - let url = url.as_ref(); - let url_parts: Vec<&str> = url.split(".").collect(); - let file_extension = url_parts.last().unwrap_or_else(||panic!("Could not determine file type.")).to_lowercase().as_str(); + let url = Url::parse(url.as_ref())?; + let format = url + .path_segments() + .and_then(|segments| segments.last()) + .and_then(|filename| Path::new(filename).extension()) + .and_then(OsStr::to_str) + .with_context(|| "Could not parse file format from URL: {url:?}")?; ensure!( - Self::ACCEPTED_STR_FORMATS.contains(&file_extension) || file_extension == "yml", - "Unsupported format {file_extension:?}, must be one of {:?}", + Self::ACCEPTED_STR_FORMATS.contains(&format) || format == "yml", + "Unsupported format {format:?}, must be one of {:?}", Self::ACCEPTED_STR_FORMATS ); - let file_name = "temporary_object.".to_string() + file_extension; - let temp_dir = tempdir()?; - let file_path = temp_dir.path().join(file_name); - download_file_from_url(url, &file_path); - // seems like I might also be able to use from_reader instead -- which - // one is preferable? - // try switching over to from_reader(), this will require finding - // a crate which takes a url and spits out a Rust object - // that implements std::io::Read - Self::from_file(file_path) + let body_ureq = ureq::get(url.as_ref()).call()?.into_string()?; + let body = body_ureq.as_str(); + SerdeAPI::from_str(body, format) + // if (format == "yaml") | (format == "yml") { + // // let file = serde_yaml::from_str::(body).with_context(||"Could not access url content."); + // // Ok(serde_yaml::from_str::(&body)) + // self.from_str(body, format) + // } else if format == "json" { + // // let file = serde_json::from_str::(body).with_context(||"Could not access url content."); + // // Ok(serde_json::from_str::(&body)) + // } else { + // return Err(anyhow!("Incorrect file format.")) + // } + + + // let client = reqwest::Client::builder() + // .redirect(reqwest::redirect::Policy::none()) + // .build() + // .unwrap(); + // let res = client.get(url) + // .text()?; + // let file = match file_extension { + // "yaml" | "yml" => serde_yaml::from_str(&body).with_context(||"Could not access url content."), + // "json" => serde_json::from_str(&body).with_context(||"Could not access url content."), + // _ => return Err(anyhow!("Incorrect file format.")) + // }; + + + // let page = http::handle().get(url).exec().with_context(||"Could not access url content.")?; + // let body = std::str::from_utf8(page.get_body()).with_context(|| { + // "Failed to parse response from url." + // })?; + // let file = match file_extension { + // "yaml" | "yml" => serde_yaml::from_str(body).with_context("Could not access url content."), + // "json" => serde_json::from_str(body).with_context("Could not access url content."), + // _ => return Err(anyhow!("Incorrect file format.")) + // }; + // file.from_reader(); + // let client = reqwest::blocking::Client::builder() + // .redirect(reqwest::redirect::Policy::none()) + // .build() + // .unwrap(); + // let res = client.get("http://example.com") + // .send()? + // .text()?; + // println!("{}", res); + + // if (file_extension == "yaml") | (file_extension == "yml") { + // let file: serde_yaml::Value = reqwest::Client::new() + // .get(url) + // .header("User-Agent", "Reqwest Rust Test") + // .send() + // .await? + // .json() + // .await? + // .from_reader(); + // } else if file_extension == "json" { + // let file: serde_json::Value = reqwest::Client::new() + // .get(url) + // .header("User-Agent", "Reqwest Rust Test") + // .send() + // .await? + // .json() + // .await?; + // } else { + // return Err(anyhow!("Incorrect file format.")) + // } + // file.from_reader() + + + // let echo_json: serde_json::Value = reqwest::Client::new() + // .get(url) + // .header("User-Agent", "Reqwest Rust Test") + // .send() + // .await? + // .json() + // .await?; + + + // let url = url.as_ref(); + // let url_parts: Vec<&str> = url.split(".").collect(); + // let file_extension = url_parts.last().unwrap_or_else(||panic!("Could not determine file type.")).to_lowercase().as_str(); + // ensure!( + // Self::ACCEPTED_STR_FORMATS.contains(&file_extension) || file_extension == "yml", + // "Unsupported format {file_extension:?}, must be one of {:?}", + // Self::ACCEPTED_STR_FORMATS + // ); + // let page = http::handle().get(url).exec().with_context(||"Could not access url content.")?; + // let body = std::str::from_utf8(page.get_body()).with_context(|| { + // "Failed to parse response from url." + // })?; + // let file = match file_extension { + // "yaml" | "yml" => serde_yaml::from_str(body).with_context("Could not access url content."), + // "json" => serde_json::from_str(body).with_context("Could not access url content."), + // _ => return Err(anyhow!("Incorrect file format.")) + // }; + // file.from_reader(); + + // let url = Url::parse(url.as_ref())?; + // let scheme = url.scheme(); + // let port = match scheme { + // "http" => 80, + // "https" => 443, + // _ => return Err(anyhow!("Could not determine port.")) + // }; + // let ips = lookup_host(url.as_ref()).with_context(||"Could not determine ips.")?; + + + + // let url = url.as_ref(); + // let url_parts: Vec<&str> = url.split(".").collect(); + // let file_extension = url_parts.last().unwrap_or_else(||panic!("Could not determine file type.")).to_lowercase().as_str(); + // ensure!( + // Self::ACCEPTED_STR_FORMATS.contains(&file_extension) || file_extension == "yml", + // "Unsupported format {file_extension:?}, must be one of {:?}", + // Self::ACCEPTED_STR_FORMATS + // ); + // let file_name = "temporary_object.".to_string() + file_extension; + // let temp_dir = tempdir()?; + // let file_path = temp_dir.path().join(file_name); + // download_file_from_url(url, &file_path); + // // seems like I might also be able to use from_reader instead -- which + // // one is preferable? + // // try switching over to from_reader(), this will require finding + // // a crate which takes a url and spits out a Rust object + // // that implements std::io::Read + // Self::from_file(file_path) } /// takes an instantiated Rust object and saves it in the FASTSim data directory in From f68d05545be6b64c3d4fa62a7f357db7596a9c4e Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Tue, 26 Dec 2023 16:14:13 -0500 Subject: [PATCH 05/25] updating from_url and from_str --- rust/fastsim-core/src/cycle.rs | 3 +- rust/fastsim-core/src/traits.rs | 140 ++------------------------------ 2 files changed, 7 insertions(+), 136 deletions(-) diff --git a/rust/fastsim-core/src/cycle.rs b/rust/fastsim-core/src/cycle.rs index f87133ce..e78d3c14 100644 --- a/rust/fastsim-core/src/cycle.rs +++ b/rust/fastsim-core/src/cycle.rs @@ -717,7 +717,8 @@ impl SerdeAPI for RustCycle { /// Note that using this method to instantiate a RustCycle from CSV, rather /// than the `from_csv_str` method, sets the cycle name to an empty string - fn from_str(contents: &str, format: &str) -> anyhow::Result { + fn from_str>(contents: S, format: &str) -> anyhow::Result { + let contents = contents.as_ref(); match format.trim_start_matches('.').to_lowercase().as_str() { "yaml" | "yml" => Self::from_yaml(contents), "json" => Self::from_json(contents), diff --git a/rust/fastsim-core/src/traits.rs b/rust/fastsim-core/src/traits.rs index db8f26ea..cd1838c3 100644 --- a/rust/fastsim-core/src/traits.rs +++ b/rust/fastsim-core/src/traits.rs @@ -1,7 +1,6 @@ use crate::imports::*; use std::collections::HashMap; use ureq; -use url::Url; pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { const ACCEPTED_BYTE_FORMATS: &'static [&'static str] = &["yaml", "json", "bin"]; @@ -117,7 +116,8 @@ pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { } } - fn from_str(contents: &str, format: &str) -> anyhow::Result { + fn from_str>(contents: S, format: &str) -> anyhow::Result { + let contents = contents.as_ref(); match format.trim_start_matches('.').to_lowercase().as_str() { "yaml" | "yml" => Self::from_yaml(contents), "json" => Self::from_json(contents), @@ -163,145 +163,15 @@ pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { /// # Arguments /// - url: url (either as a string or url type) to object fn from_url>(url: S) -> anyhow::Result { - let url = Url::parse(url.as_ref())?; + let url = url::Url::parse(url.as_ref())?; let format = url .path_segments() .and_then(|segments| segments.last()) .and_then(|filename| Path::new(filename).extension()) .and_then(OsStr::to_str) .with_context(|| "Could not parse file format from URL: {url:?}")?; - ensure!( - Self::ACCEPTED_STR_FORMATS.contains(&format) || format == "yml", - "Unsupported format {format:?}, must be one of {:?}", - Self::ACCEPTED_STR_FORMATS - ); - let body_ureq = ureq::get(url.as_ref()).call()?.into_string()?; - let body = body_ureq.as_str(); - SerdeAPI::from_str(body, format) - // if (format == "yaml") | (format == "yml") { - // // let file = serde_yaml::from_str::(body).with_context(||"Could not access url content."); - // // Ok(serde_yaml::from_str::(&body)) - // self.from_str(body, format) - // } else if format == "json" { - // // let file = serde_json::from_str::(body).with_context(||"Could not access url content."); - // // Ok(serde_json::from_str::(&body)) - // } else { - // return Err(anyhow!("Incorrect file format.")) - // } - - - // let client = reqwest::Client::builder() - // .redirect(reqwest::redirect::Policy::none()) - // .build() - // .unwrap(); - // let res = client.get(url) - // .text()?; - // let file = match file_extension { - // "yaml" | "yml" => serde_yaml::from_str(&body).with_context(||"Could not access url content."), - // "json" => serde_json::from_str(&body).with_context(||"Could not access url content."), - // _ => return Err(anyhow!("Incorrect file format.")) - // }; - - - // let page = http::handle().get(url).exec().with_context(||"Could not access url content.")?; - // let body = std::str::from_utf8(page.get_body()).with_context(|| { - // "Failed to parse response from url." - // })?; - // let file = match file_extension { - // "yaml" | "yml" => serde_yaml::from_str(body).with_context("Could not access url content."), - // "json" => serde_json::from_str(body).with_context("Could not access url content."), - // _ => return Err(anyhow!("Incorrect file format.")) - // }; - // file.from_reader(); - // let client = reqwest::blocking::Client::builder() - // .redirect(reqwest::redirect::Policy::none()) - // .build() - // .unwrap(); - // let res = client.get("http://example.com") - // .send()? - // .text()?; - // println!("{}", res); - - // if (file_extension == "yaml") | (file_extension == "yml") { - // let file: serde_yaml::Value = reqwest::Client::new() - // .get(url) - // .header("User-Agent", "Reqwest Rust Test") - // .send() - // .await? - // .json() - // .await? - // .from_reader(); - // } else if file_extension == "json" { - // let file: serde_json::Value = reqwest::Client::new() - // .get(url) - // .header("User-Agent", "Reqwest Rust Test") - // .send() - // .await? - // .json() - // .await?; - // } else { - // return Err(anyhow!("Incorrect file format.")) - // } - // file.from_reader() - - - // let echo_json: serde_json::Value = reqwest::Client::new() - // .get(url) - // .header("User-Agent", "Reqwest Rust Test") - // .send() - // .await? - // .json() - // .await?; - - - // let url = url.as_ref(); - // let url_parts: Vec<&str> = url.split(".").collect(); - // let file_extension = url_parts.last().unwrap_or_else(||panic!("Could not determine file type.")).to_lowercase().as_str(); - // ensure!( - // Self::ACCEPTED_STR_FORMATS.contains(&file_extension) || file_extension == "yml", - // "Unsupported format {file_extension:?}, must be one of {:?}", - // Self::ACCEPTED_STR_FORMATS - // ); - // let page = http::handle().get(url).exec().with_context(||"Could not access url content.")?; - // let body = std::str::from_utf8(page.get_body()).with_context(|| { - // "Failed to parse response from url." - // })?; - // let file = match file_extension { - // "yaml" | "yml" => serde_yaml::from_str(body).with_context("Could not access url content."), - // "json" => serde_json::from_str(body).with_context("Could not access url content."), - // _ => return Err(anyhow!("Incorrect file format.")) - // }; - // file.from_reader(); - - // let url = Url::parse(url.as_ref())?; - // let scheme = url.scheme(); - // let port = match scheme { - // "http" => 80, - // "https" => 443, - // _ => return Err(anyhow!("Could not determine port.")) - // }; - // let ips = lookup_host(url.as_ref()).with_context(||"Could not determine ips.")?; - - - - // let url = url.as_ref(); - // let url_parts: Vec<&str> = url.split(".").collect(); - // let file_extension = url_parts.last().unwrap_or_else(||panic!("Could not determine file type.")).to_lowercase().as_str(); - // ensure!( - // Self::ACCEPTED_STR_FORMATS.contains(&file_extension) || file_extension == "yml", - // "Unsupported format {file_extension:?}, must be one of {:?}", - // Self::ACCEPTED_STR_FORMATS - // ); - // let file_name = "temporary_object.".to_string() + file_extension; - // let temp_dir = tempdir()?; - // let file_path = temp_dir.path().join(file_name); - // download_file_from_url(url, &file_path); - // // seems like I might also be able to use from_reader instead -- which - // // one is preferable? - // // try switching over to from_reader(), this will require finding - // // a crate which takes a url and spits out a Rust object - // // that implements std::io::Read - // Self::from_file(file_path) + let response = ureq::get(url.as_ref()).call()?.into_string()?; + Self::from_str(response, format) } /// takes an instantiated Rust object and saves it in the FASTSim data directory in From 8d556f3557dbdd5ea79b3ccc15b9c555a19ca0e4 Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Tue, 2 Jan 2024 13:22:37 -0700 Subject: [PATCH 06/25] adding from_github_or_url method --- rust/fastsim-core/src/traits.rs | 8 ++++--- rust/fastsim-core/src/utils.rs | 2 +- rust/fastsim-core/src/vehicle.rs | 40 +++++++++++++++++++++++++++++--- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/rust/fastsim-core/src/traits.rs b/rust/fastsim-core/src/traits.rs index cd1838c3..684be745 100644 --- a/rust/fastsim-core/src/traits.rs +++ b/rust/fastsim-core/src/traits.rs @@ -161,7 +161,9 @@ pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { /// instantiates an object from a url /// accepts yaml and json file types /// # Arguments - /// - url: url (either as a string or url type) to object + /// - url: URL (either as a string or url type) to object + /// Note: The URL needs to be a URL pointing directly to a file, for example + /// a raw github URL. fn from_url>(url: S) -> anyhow::Result { let url = url::Url::parse(url.as_ref())?; let format = url @@ -170,8 +172,8 @@ pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { .and_then(|filename| Path::new(filename).extension()) .and_then(OsStr::to_str) .with_context(|| "Could not parse file format from URL: {url:?}")?; - let response = ureq::get(url.as_ref()).call()?.into_string()?; - Self::from_str(response, format) + let response = ureq::get(url.as_ref()).call()?.into_reader(); + Self::from_reader(response, format) } /// takes an instantiated Rust object and saves it in the FASTSim data directory in diff --git a/rust/fastsim-core/src/utils.rs b/rust/fastsim-core/src/utils.rs index 98cb560c..645a9c49 100644 --- a/rust/fastsim-core/src/utils.rs +++ b/rust/fastsim-core/src/utils.rs @@ -7,7 +7,7 @@ use ndarray::*; use ndarray_stats::QuantileExt; use regex::Regex; use std::collections::HashSet; -use url::{Url, Host, Position}; +use url::Url; use crate::imports::*; #[cfg(feature = "pyo3")] diff --git a/rust/fastsim-core/src/vehicle.rs b/rust/fastsim-core/src/vehicle.rs index ee0092d0..56a82ce4 100644 --- a/rust/fastsim-core/src/vehicle.rs +++ b/rust/fastsim-core/src/vehicle.rs @@ -7,6 +7,7 @@ use crate::proc_macros::{add_pyo3_api, doc_field, ApproxEq}; #[cfg(feature = "pyo3")] use crate::pyo3imports::*; use crate::utils::create_project_subdir; +use std::fs; use lazy_static::lazy_static; use regex::Regex; @@ -1036,8 +1037,8 @@ impl RustVehicle { v } - /// Downloads specified vehicle from vehicle repo into - /// fastsim/python/fastsim/resources/vehdb. + /// Downloads specified vehicle from vehicle repo or url into + /// VEHICLE_DIRECTORY_URL, if not already downloaded. Returns vehicle. /// # Arguments /// - vehicle_file_name: file name for vehicle to be downloaded /// - url: url for vehicle to be downloaded, if None, assumed to be @@ -1047,7 +1048,40 @@ impl RustVehicle { /// vehicle is not stored locally, will download and store vehicle for later /// use. If False, will not check for vehicle locally or store vehicle /// locally for later use - pub fn from_github_or_url>(vehicle_file_name: S, url: Option, cache: bool) { + /// Note: The URL needs to be a URL pointing directly to a file, for example + /// a raw github URL. + pub fn from_github_or_url>(vehicle_file_name: S, url: Option, cache: bool) -> anyhow::Result { + if cache { + let dir = create_project_subdir("vehicles").with_context(||"Could not determine FASTSim data directory path.")?; + let mut file_already_downloaded = false; + let mut file_path = PathBuf::new(); + for entry in fs::read_dir(dir)? { + let entry = entry?; + file_path = entry.path(); + let entry_name = file_path.file_name().with_context(||"Could not parse file name from directory file: {entry:?}")?.to_str().with_context(||"Could not parse file name from directory file: {entry:?}")?; + if entry_name == vehicle_file_name.as_ref() { + file_already_downloaded = true; + break + } else { + continue + } + } + if file_already_downloaded { + Self::from_resource(file_path) + } else { + let url_internal = match url { + Some(s) => s.as_ref().to_owned(), + None => Self::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name.as_ref() + }; + Self::from_url(url_internal) + } + } else { + let url_internal = match url { + Some(s) => s.as_ref().to_owned(), + None => Self::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name.as_ref() + }; + Self::from_url(url_internal) + } } } From f6896f6f4beeee40ebc29974cd2bed7618c225b0 Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Tue, 2 Jan 2024 17:25:41 -0700 Subject: [PATCH 07/25] updating error messages --- rust/fastsim-core/src/vehicle.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/rust/fastsim-core/src/vehicle.rs b/rust/fastsim-core/src/vehicle.rs index 56a82ce4..e50c270a 100644 --- a/rust/fastsim-core/src/vehicle.rs +++ b/rust/fastsim-core/src/vehicle.rs @@ -7,11 +7,11 @@ use crate::proc_macros::{add_pyo3_api, doc_field, ApproxEq}; #[cfg(feature = "pyo3")] use crate::pyo3imports::*; use crate::utils::create_project_subdir; -use std::fs; use lazy_static::lazy_static; use regex::Regex; use validator::Validate; +use std::fs; // veh_pt_type options pub const CONV: &str = "Conv"; @@ -547,7 +547,7 @@ pub struct RustVehicle { /// RustVehicle rust methods impl RustVehicle { - const VEHICLE_DIRECTORY_URL: &'static str = &"https://github.com/NREL/fastsim-vehicles/blob/main/public/"; + const VEHICLE_DIRECTORY_URL: &'static str = &"https://github.com/NREL/fastsim-vehicles/tree/main/public"; /// Sets the following parameters: /// - `ess_mass_kg` /// - `mc_mass_kg` @@ -1039,6 +1039,8 @@ impl RustVehicle { /// Downloads specified vehicle from vehicle repo or url into /// VEHICLE_DIRECTORY_URL, if not already downloaded. Returns vehicle. + /// WARNING: if there is a file already in the data subdirectory with the + /// same name, it will be replaced by the new file /// # Arguments /// - vehicle_file_name: file name for vehicle to be downloaded /// - url: url for vehicle to be downloaded, if None, assumed to be @@ -1055,8 +1057,8 @@ impl RustVehicle { let dir = create_project_subdir("vehicles").with_context(||"Could not determine FASTSim data directory path.")?; let mut file_already_downloaded = false; let mut file_path = PathBuf::new(); - for entry in fs::read_dir(dir)? { - let entry = entry?; + for entry in fs::read_dir(dir).with_context(||"Could not read data directory.")? { + let entry = entry.with_context(||"Could not parse entry: {entry:?}")?; file_path = entry.path(); let entry_name = file_path.file_name().with_context(||"Could not parse file name from directory file: {entry:?}")?.to_str().with_context(||"Could not parse file name from directory file: {entry:?}")?; if entry_name == vehicle_file_name.as_ref() { From 474df42ef7edbad0e564f15d01d0b8d25e5c0a6c Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Wed, 3 Jan 2024 15:36:48 -0700 Subject: [PATCH 08/25] adding fetch_github_list function --- rust/fastsim-core/src/vehicle_utils.rs | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/rust/fastsim-core/src/vehicle_utils.rs b/rust/fastsim-core/src/vehicle_utils.rs index a34b99a8..67758b97 100644 --- a/rust/fastsim-core/src/vehicle_utils.rs +++ b/rust/fastsim-core/src/vehicle_utils.rs @@ -1771,6 +1771,33 @@ pub fn import_and_save_all_vehicles( Ok(()) } +#[derive(Deserialize)] +pub struct GitVehicleInfo { + pub name: String, + pub path: String, + pub sha: String, + pub size: i64, + pub url: String, + pub html_url: String, + pub git_url: String, + pub download_url: String, + #[serde(rename = "type")] + pub url_type: String, + pub links: Vec, +} + +const VEHICLE_REPO_LIST_URL: &'static str = &"https://api.github.com/repos/NREL/fastsim-vehicles/contents/public"; + +pub fn fetch_github_list() -> anyhow::Result> { + let response = ureq::get(VEHICLE_REPO_LIST_URL).call()?.into_reader(); + let github_list: HashMap = serde_json::from_reader(response).with_context(||"Cannot parse github vehicle list.")?; + let mut vehicle_name_list: Vec = Vec::new(); + for (key, value) in github_list.iter() { + vehicle_name_list.push(value.name.to_owned()) + } + Ok(vehicle_name_list) +} + #[cfg(test)] mod vehicle_utils_tests { use super::*; From ea631fdd72a1366b4c64e0bf05ab58b5db4d3632 Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Wed, 3 Jan 2024 15:50:20 -0700 Subject: [PATCH 09/25] updating fetch_github_list --- rust/fastsim-core/src/vehicle_utils.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/rust/fastsim-core/src/vehicle_utils.rs b/rust/fastsim-core/src/vehicle_utils.rs index 67758b97..c5694a20 100644 --- a/rust/fastsim-core/src/vehicle_utils.rs +++ b/rust/fastsim-core/src/vehicle_utils.rs @@ -1771,6 +1771,14 @@ pub fn import_and_save_all_vehicles( Ok(()) } +#[derive(Deserialize)] +pub struct VehicleLinks { + #[serde(rename = "self")] + pub self_url: String, + pub git: String, + pub html: String, +} + #[derive(Deserialize)] pub struct GitVehicleInfo { pub name: String, @@ -1783,7 +1791,8 @@ pub struct GitVehicleInfo { pub download_url: String, #[serde(rename = "type")] pub url_type: String, - pub links: Vec, + #[serde(rename = "_links")] + pub links: VehicleLinks, } const VEHICLE_REPO_LIST_URL: &'static str = &"https://api.github.com/repos/NREL/fastsim-vehicles/contents/public"; @@ -1792,7 +1801,7 @@ pub fn fetch_github_list() -> anyhow::Result> { let response = ureq::get(VEHICLE_REPO_LIST_URL).call()?.into_reader(); let github_list: HashMap = serde_json::from_reader(response).with_context(||"Cannot parse github vehicle list.")?; let mut vehicle_name_list: Vec = Vec::new(); - for (key, value) in github_list.iter() { + for (_key, value) in github_list.iter() { vehicle_name_list.push(value.name.to_owned()) } Ok(vehicle_name_list) From 5275708633e06a1b0f6a19f36737e00d699cb909 Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Thu, 4 Jan 2024 12:36:23 -0700 Subject: [PATCH 10/25] adding caching of vehicle if not downloaded to from_github_or_url --- rust/fastsim-core/src/vehicle.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/fastsim-core/src/vehicle.rs b/rust/fastsim-core/src/vehicle.rs index e50c270a..1659ea00 100644 --- a/rust/fastsim-core/src/vehicle.rs +++ b/rust/fastsim-core/src/vehicle.rs @@ -1082,6 +1082,7 @@ impl RustVehicle { Some(s) => s.as_ref().to_owned(), None => Self::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name.as_ref() }; + url_to_cache(&url_internal, "vehicles").with_context(||"Unable to cache vehicle from url")?; Self::from_url(url_internal) } } From 24a11ac6b7a4ce8c7cd8950608d37d26a1884d98 Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Thu, 4 Jan 2024 12:54:11 -0700 Subject: [PATCH 11/25] updating from_github_or_url doc string --- rust/fastsim-core/src/vehicle.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/rust/fastsim-core/src/vehicle.rs b/rust/fastsim-core/src/vehicle.rs index 1659ea00..0fa94280 100644 --- a/rust/fastsim-core/src/vehicle.rs +++ b/rust/fastsim-core/src/vehicle.rs @@ -1039,8 +1039,6 @@ impl RustVehicle { /// Downloads specified vehicle from vehicle repo or url into /// VEHICLE_DIRECTORY_URL, if not already downloaded. Returns vehicle. - /// WARNING: if there is a file already in the data subdirectory with the - /// same name, it will be replaced by the new file /// # Arguments /// - vehicle_file_name: file name for vehicle to be downloaded /// - url: url for vehicle to be downloaded, if None, assumed to be From 7e9a51c9fede422f1f198b412f532eea1a73210d Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Thu, 4 Jan 2024 12:57:20 -0700 Subject: [PATCH 12/25] updating doc strings --- rust/fastsim-core/src/vehicle.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rust/fastsim-core/src/vehicle.rs b/rust/fastsim-core/src/vehicle.rs index 0fa94280..20845326 100644 --- a/rust/fastsim-core/src/vehicle.rs +++ b/rust/fastsim-core/src/vehicle.rs @@ -1038,7 +1038,9 @@ impl RustVehicle { } /// Downloads specified vehicle from vehicle repo or url into - /// VEHICLE_DIRECTORY_URL, if not already downloaded. Returns vehicle. + /// VEHICLE_DIRECTORY_URL, if not already downloaded. Returns vehicle. + /// Optionally checks if vehicle is already downloaded and uses downloaded + /// version if so, and downloads and stores for later use if not. /// # Arguments /// - vehicle_file_name: file name for vehicle to be downloaded /// - url: url for vehicle to be downloaded, if None, assumed to be From 05fea86cd79f44aae0ef9e00eaff08291b105492 Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Fri, 5 Jan 2024 14:48:27 -0700 Subject: [PATCH 13/25] adding temporary tests/file to support the tests --- rust/fastsim-core/Cargo.toml | 1 + .../1110_2022_Tesla_Model_Y_RWD_opt45017.yaml | 226 ++++++++++++++++++ rust/fastsim-core/src/traits.rs | 31 +++ rust/fastsim-core/src/utils.rs | 16 +- rust/fastsim-core/src/vehicle.rs | 167 ++++++++++++- rust/fastsim-core/src/vehicle_utils.rs | 9 + 6 files changed, 446 insertions(+), 4 deletions(-) create mode 100644 rust/fastsim-core/resources/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml diff --git a/rust/fastsim-core/Cargo.toml b/rust/fastsim-core/Cargo.toml index ae0c8a84..261739e9 100644 --- a/rust/fastsim-core/Cargo.toml +++ b/rust/fastsim-core/Cargo.toml @@ -41,6 +41,7 @@ ndarray-stats = "0.5.1" tempfile = "3.8.1" url = "2.5.0" ureq = "2.9.1" +isahc = "1.7.2" [package.metadata] include = [ diff --git a/rust/fastsim-core/resources/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml b/rust/fastsim-core/resources/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml new file mode 100644 index 00000000..0f6fdc3a --- /dev/null +++ b/rust/fastsim-core/resources/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml @@ -0,0 +1,226 @@ +--- +scenario_name: 2022 Tesla Model Y RWD +veh_year: 2022 +veh_pt_type: BEV +drag_coef: 0.21716617209403616 +frontal_area_m2: 3.121538772737215 +glider_kg: 306.4316324129186 +veh_cg_m: -0.53 +drive_axle_weight_frac: 0.59 +wheel_base_m: 2.6 +cargo_kg: 136.0 +veh_override_kg: ~ +comp_mass_multiplier: 1.4 +fs_max_kw: 0.0 +fs_secs_to_peak_pwr: 1.0 +fs_kwh: 0.0 +fs_kwh_per_kg: 9.89 +fc_max_kw: 0.0 +fc_pwr_out_perc: + v: 1 + dim: + - 12 + data: + - 0.0 + - 0.005 + - 0.015 + - 0.04 + - 0.06 + - 0.1 + - 0.14 + - 0.2 + - 0.4 + - 0.6 + - 0.8 + - 1.0 +fc_eff_map: + v: 1 + dim: + - 12 + data: + - 0.1 + - 0.12 + - 0.28 + - 0.35 + - 0.38 + - 0.39 + - 0.4 + - 0.4 + - 0.38 + - 0.37 + - 0.36 + - 0.35 +fc_eff_type: SI +fc_sec_to_peak_pwr: 6.0 +fc_base_kg: 61.0 +fc_kw_per_kg: 2.13 +min_fc_time_on: 30.0 +idle_fc_kw: 0.0 +mc_max_kw: 208.79628939165707 +mc_pwr_out_perc: + v: 1 + dim: + - 11 + data: + - 0.0 + - 0.02 + - 0.04 + - 0.06 + - 0.08 + - 0.1 + - 0.2 + - 0.4 + - 0.6 + - 0.8 + - 1.0 +mc_eff_map: + v: 1 + dim: + - 11 + data: + - 0.41 + - 0.45 + - 0.48 + - 0.54 + - 0.58 + - 0.62 + - 0.83 + - 0.93 + - 0.94 + - 0.93 + - 0.92 +mc_sec_to_peak_pwr: 4.0 +mc_pe_kg_per_kw: 0.833 +mc_pe_base_kg: 21.6 +ess_max_kw: 219.23610386123994 +ess_max_kwh: 81.0 +ess_kg_per_kwh: 8.0 +ess_base_kg: 75.0 +ess_round_trip_eff: 0.97 +ess_life_coef_a: 110.0 +ess_life_coef_b: -0.6811 +min_soc: 0.05 +max_soc: 0.98 +ess_dischg_to_fc_max_eff_perc: 0.0 +ess_chg_to_fc_max_eff_perc: 0.0 +wheel_inertia_kg_m2: 0.815 +num_wheels: 4.0 +wheel_rr_coef: 0.007752538840328322 +wheel_radius_m: 0.336 +wheel_coef_of_fric: 0.7 +max_accel_buffer_mph: 60.0 +max_accel_buffer_perc_of_useable_soc: 0.2 +perc_high_acc_buf: 0.0 +mph_fc_on: 1.0 +kw_demand_fc_on: 100.0 +max_regen: 0.98 +stop_start: false +force_aux_on_fc: false +alt_eff: 1.0 +chg_eff: 0.86 +aux_kw: 0.25 +trans_kg: 114.0 +trans_eff: 0.98 +ess_to_fuel_ok_error: 0.005 +fc_eff_array: + - 0.1 + - 0.10400000000000001 + - 0.108 + - 0.112 + - 0.11599999999999999 + - 0.12 + - 0.124 + - 0.128 + - 0.132 + - 0.136 + - 0.14 + - 0.14400000000000002 + - 0.14800000000000002 + - 0.15200000000000002 + - 0.15600000000000003 + - 0.16 + - 0.16240000000000002 + - 0.1648 + - 0.16720000000000002 + - 0.1696 + - 0.17200000000000001 + - 0.1744 + - 0.1768 + - 0.1792 + - 0.1816 + - 0.184 + - 0.1864 + - 0.1888 + - 0.1912 + - 0.1936 + - 0.196 + - 0.20800000000000002 + - 0.22 + - 0.23500000000000001 + - 0.25 + - 0.265 + - 0.28 + - 0.28625 + - 0.29250000000000004 + - 0.30500000000000005 + - 0.3175 + - 0.33 + - 0.335 + - 0.33999999999999997 + - 0.345 + - 0.35 + - 0.3516666666666666 + - 0.35333333333333333 + - 0.355 + - 0.35666666666666663 + - 0.35833333333333334 + - 0.36 + - 0.3595 + - 0.359 + - 0.3585 + - 0.358 + - 0.3575 + - 0.357 + - 0.3565 + - 0.356 + - 0.3555 + - 0.355 + - 0.3545 + - 0.354 + - 0.3535 + - 0.353 + - 0.3525 + - 0.352 + - 0.3515 + - 0.351 + - 0.3505 + - 0.35 + - 0.3495 + - 0.349 + - 0.3485 + - 0.348 + - 0.3475 + - 0.347 + - 0.3465 + - 0.346 + - 0.3455 + - 0.345 + - 0.34450000000000003 + - 0.34400000000000003 + - 0.3435 + - 0.343 + - 0.3425 + - 0.342 + - 0.3415 + - 0.341 + - 0.3405 + - 0.34 + - 0.335 + - 0.33 + - 0.325 + - 0.32 + - 0.315 + - 0.31 + - 0.305 + - 0.3 +veh_kg: 1887.969865101469 \ No newline at end of file diff --git a/rust/fastsim-core/src/traits.rs b/rust/fastsim-core/src/traits.rs index 5c455b24..2b0b7674 100644 --- a/rust/fastsim-core/src/traits.rs +++ b/rust/fastsim-core/src/traits.rs @@ -1,6 +1,7 @@ use crate::imports::*; use std::collections::HashMap; use ureq; +use isahc::prelude::*; pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { const ACCEPTED_BYTE_FORMATS: &'static [&'static str] = &["yaml", "json", "bin"]; @@ -159,6 +160,8 @@ pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { .and_then(|filename| Path::new(filename).extension()) .and_then(OsStr::to_str) .with_context(|| "Could not parse file format from URL: {url:?}")?; + // let mut response = isahc::get(url.as_ref())?; + // Self::from_str(response.text()?, format) let response = ureq::get(url.as_ref()).call()?.into_reader(); Self::from_reader(response, format) } @@ -292,3 +295,31 @@ where .all(|(key, value)| other.get(key).map_or(false, |v| value.approx_eq(v, tol))); } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_from_url() { + let vehicle = crate::vehicle::RustVehicle::from_url("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); + let comparison_vehicle = crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); + // let comparison_vehicle = crate::vehicle::RustVehicle::from_file("/Users/rsteutev/Documents/GitHub/fastsim/rust/fastsim-core/src/test_vehicle.json").unwrap(); + println!("{}", vehicle.to_yaml().unwrap()); + assert_eq! (vehicle, comparison_vehicle); + } + + #[test] + fn test_to_cache() { + let vehicle_a = crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); + crate::vehicle::RustVehicle::to_cache(&vehicle_a, "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); + let data_subdirectory = create_project_subdir("vehicles").unwrap(); + let file_path = data_subdirectory.join("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"); + println!("{}", file_path.to_string_lossy()); + println!("{}", crate::vehicle::RustVehicle::CACHE_FOLDER); + let vehicle_b = crate::vehicle::RustVehicle::from_file(&file_path).unwrap(); + assert_eq! (vehicle_a, vehicle_b); + std::fs::remove_file(file_path).unwrap(); + } +} + diff --git a/rust/fastsim-core/src/utils.rs b/rust/fastsim-core/src/utils.rs index 645a9c49..f2ddbf8c 100644 --- a/rust/fastsim-core/src/utils.rs +++ b/rust/fastsim-core/src/utils.rs @@ -536,7 +536,9 @@ pub fn create_project_subdir>(subpath: P) -> anyhow::Result, P: AsRef>(url: S, subpath: P) -> anyhow::Result<()> { let url = Url::parse(url.as_ref())?; let file_name = url @@ -756,4 +758,16 @@ mod tests { let y_lookup = interpolate_vectors(&x, &xs.to_vec(), &ys.to_vec(), false); assert_eq!(expected_y_lookup, y_lookup); } + + #[test] + fn test_url_to_cache() { + url_to_cache("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", "vehicles").unwrap(); + let data_subdirectory = create_project_subdir("vehicles").unwrap(); + let file_path = data_subdirectory.join("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"); + println!("{}", file_path.to_string_lossy()); + let vehicle = crate::vehicle::RustVehicle::from_file(&file_path).unwrap(); + let comparison_vehicle = crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); + assert_eq! (vehicle, comparison_vehicle); + std::fs::remove_file(file_path).unwrap(); + } } diff --git a/rust/fastsim-core/src/vehicle.rs b/rust/fastsim-core/src/vehicle.rs index 20845326..2e3bc993 100644 --- a/rust/fastsim-core/src/vehicle.rs +++ b/rust/fastsim-core/src/vehicle.rs @@ -547,7 +547,7 @@ pub struct RustVehicle { /// RustVehicle rust methods impl RustVehicle { - const VEHICLE_DIRECTORY_URL: &'static str = &"https://github.com/NREL/fastsim-vehicles/tree/main/public"; + const VEHICLE_DIRECTORY_URL: &'static str = &"https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/"; /// Sets the following parameters: /// - `ess_mass_kg` /// - `mc_mass_kg` @@ -1069,12 +1069,13 @@ impl RustVehicle { } } if file_already_downloaded { - Self::from_resource(file_path) + Self::from_file(file_path) } else { let url_internal = match url { Some(s) => s.as_ref().to_owned(), None => Self::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name.as_ref() }; + url_to_cache(&url_internal, "vehicles").with_context(||"Unable to cache vehicle from url")?; Self::from_url(url_internal) } } else { @@ -1082,7 +1083,6 @@ impl RustVehicle { Some(s) => s.as_ref().to_owned(), None => Self::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name.as_ref() }; - url_to_cache(&url_internal, "vehicles").with_context(||"Unable to cache vehicle from url")?; Self::from_url(url_internal) } } @@ -1428,4 +1428,165 @@ mod tests { .all(|key| bad_fields.contains(key))); assert!(validation_errs_hashmap.len() == bad_fields.len()); } + + #[test] + fn test_from_github_or_url() { + let comparison_vehicle = crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); + let data_subdirectory = create_project_subdir("vehicles").unwrap(); + let file_path_test = data_subdirectory.join("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"); + // // test when cache = false and no url provided + // let vehicle = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", None, false).unwrap(); + // assert_eq! (vehicle, comparison_vehicle); + // // test when cache = false and url provided + // let vehicle_1 = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"), false).unwrap(); + // assert_eq! (vehicle_1, comparison_vehicle); + // // test when cache = true and no url provided and no file downloaded -- general + // let vehicle_2 = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", None, true).unwrap(); + // assert_eq! (vehicle_2, comparison_vehicle); + // std::fs::remove_file(&file_path_test).unwrap(); + // // test when cache = true and no url provided and no file downloaded -- specific + // let vehicle_file_name = "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"; + // let url: Option = None; + // let dir = create_project_subdir("vehicles").with_context(||"Could not determine FASTSim data directory path.").unwrap(); + // let mut file_already_downloaded = false; + // let mut file_path = PathBuf::new(); + // for entry in fs::read_dir(dir).with_context(||"Could not read data directory.").unwrap() { + // let entry = entry.with_context(||"Could not parse entry: {entry:?}").unwrap(); + // file_path = entry.path(); + // let entry_name = file_path.file_name().with_context(||"Could not parse file name from directory file: {entry:?}").unwrap().to_str().with_context(||"Could not parse file name from directory file: {entry:?}").unwrap(); + // if entry_name == vehicle_file_name { + // file_already_downloaded = true; + // break + // } else { + // continue + // } + // } + // assert_eq!(file_already_downloaded, false); + // if file_already_downloaded { + // let vehicle_3 = RustVehicle::from_resource(file_path).unwrap(); + // println!("Accessed vehicle from cache in no file downloaded scenario -- BAD."); + // assert_eq! (vehicle_3, comparison_vehicle); + // } else { + // let url_internal = match url { + // Some(s) => s.to_owned(), + // None => RustVehicle::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name + // }; + // url_to_cache(&url_internal, "vehicles").with_context(||"Unable to cache vehicle from url").unwrap(); + // let vehicle_3 = RustVehicle::from_url(url_internal).unwrap(); + // println!("Downloaded vehicle from url and saved it to cache in no file downloaded scenario."); + // assert_eq! (vehicle_3, comparison_vehicle); + // } + // testing when cache = true and no url provided and file downloaded -- general + url_to_cache("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", "vehicles").unwrap(); + let vehicle_4 = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", None, true).unwrap(); + assert_eq! (vehicle_4, comparison_vehicle); + // testing when cache = true and no url provided and file downloaded -- specific + let vehicle_file_name_2 = "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"; + let url_2: Option = None; + let dir_2 = create_project_subdir("vehicles").with_context(||"Could not determine FASTSim data directory path.").unwrap(); + let mut file_already_downloaded_2 = false; + let mut file_path_2 = PathBuf::new(); + for entry_2 in fs::read_dir(dir_2).with_context(||"Could not read data directory.").unwrap() { + let entry_2 = entry_2.with_context(||"Could not parse entry: {entry_2:?}").unwrap(); + file_path_2 = entry_2.path(); + let entry_name_2 = file_path_2.file_name().with_context(||"Could not parse file name from directory file: {entry_2:?}").unwrap().to_str().with_context(||"Could not parse file name from directory file: {entry_2:?}").unwrap(); + if entry_name_2 == vehicle_file_name_2 { + file_already_downloaded_2 = true; + break + } else { + continue + } + } + assert_eq!(file_already_downloaded_2, true); + if file_already_downloaded_2 { + let vehicle_5 = RustVehicle::from_file(file_path_2).unwrap(); + println!("Accessed vehicle from cache in vehicle already downloaded scenario."); + assert_eq! (vehicle_5, comparison_vehicle); + } else { + let url_internal_2 = match url_2 { + Some(s) => s.to_owned(), + None => RustVehicle::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name_2 + }; + url_to_cache(&url_internal_2, "vehicles").with_context(||"Unable to cache vehicle from url").unwrap(); + let vehicle_5 = RustVehicle::from_url(url_internal_2).unwrap(); + println!("Downloaded vehicle from url and saved it to cache in vehicle already downloaded scenario -- BAD."); + assert_eq! (vehicle_5, comparison_vehicle); + } + std::fs::remove_file(&file_path_test).unwrap(); + + + // // test when cache = true and url provided and no file downloaded -- general + // let vehicle_6 = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"), true).unwrap(); + // assert_eq! (vehicle_6, comparison_vehicle); + // std::fs::remove_file(&file_path_test).unwrap(); + // // test when cache = true and url provided and no file downloaded -- specific + // let vehicle_file_name_3 = "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"; + // let url_3: Option = Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml".to_string()); + // let dir_3 = create_project_subdir("vehicles").with_context(||"Could not determine FASTSim data directory path.").unwrap(); + // let mut file_already_downloaded_3 = false; + // let mut file_path_3 = PathBuf::new(); + // for entry_3 in fs::read_dir(dir_3).with_context(||"Could not read data directory.").unwrap() { + // let entry_3 = entry_3.with_context(||"Could not parse entry: {entry_3:?}").unwrap(); + // file_path_3 = entry_3.path(); + // let entry_name_3 = file_path_3.file_name().with_context(||"Could not parse file name from directory file: {entry_3:?}").unwrap().to_str().with_context(||"Could not parse file name from directory file: {entry_3:?}").unwrap(); + // if entry_name_3 == vehicle_file_name_3 { + // file_already_downloaded_3 = true; + // break + // } else { + // continue + // } + // } + // assert_eq!(file_already_downloaded_3, false); + // if file_already_downloaded_3 { + // let vehicle_7 = RustVehicle::from_resource(file_path_3).unwrap(); + // println!("Accessed vehicle from cache in no file downloaded scenario -- BAD."); + // assert_eq! (vehicle_7, comparison_vehicle); + // } else { + // let url_internal_3 = match url_3 { + // Some(s) => s.to_owned(), + // None => RustVehicle::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name_3 + // }; + // url_to_cache(&url_internal_3, "vehicles").with_context(||"Unable to cache vehicle from url").unwrap(); + // let vehicle_7 = RustVehicle::from_url(url_internal_3).unwrap(); + // println!("Downloaded vehicle from url and saved it to cache in no file downloaded scenario."); + // assert_eq! (vehicle_7, comparison_vehicle); + // } + // testing when cache = true and url provided and file downloaded -- general + url_to_cache("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", "vehicles").unwrap(); + let vehicle_8 = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"), true).unwrap(); + assert_eq! (vehicle_8, comparison_vehicle); + // testing when cache = true and url provided and file downloaded -- specific + let vehicle_file_name_4 = "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"; + let url_4: Option = Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml".to_string()); + let dir_4 = create_project_subdir("vehicles").with_context(||"Could not determine FASTSim data directory path.").unwrap(); + let mut file_already_downloaded_4 = false; + let mut file_path_4 = PathBuf::new(); + for entry_4 in fs::read_dir(dir_4).with_context(||"Could not read data directory.").unwrap() { + let entry_4 = entry_4.with_context(||"Could not parse entry: {entry_4:?}").unwrap(); + file_path_4 = entry_4.path(); + let entry_name_4 = file_path_4.file_name().with_context(||"Could not parse file name from directory file: {entry_4:?}").unwrap().to_str().with_context(||"Could not parse file name from directory file: {entry_4:?}").unwrap(); + if entry_name_4 == vehicle_file_name_4 { + file_already_downloaded_4 = true; + break + } else { + continue + } + } + assert_eq!(file_already_downloaded_4, true); + if file_already_downloaded_4 { + let vehicle_9 = RustVehicle::from_file(file_path_4).unwrap(); + println!("Accessed vehicle from cache in vehicle already downloaded scenario."); + assert_eq! (vehicle_9, comparison_vehicle); + } else { + let url_internal_4 = match url_4 { + Some(s) => s.to_owned(), + None => RustVehicle::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name_4 + }; + url_to_cache(&url_internal_4, "vehicles").with_context(||"Unable to cache vehicle from url").unwrap(); + let vehicle_9 = RustVehicle::from_url(url_internal_4).unwrap(); + println!("Downloaded vehicle from url and saved it to cache in vehicle already downloaded scenario -- BAD."); + assert_eq! (vehicle_9, comparison_vehicle); + } + std::fs::remove_file(&file_path_test).unwrap(); + } } diff --git a/rust/fastsim-core/src/vehicle_utils.rs b/rust/fastsim-core/src/vehicle_utils.rs index c5694a20..58e9d723 100644 --- a/rust/fastsim-core/src/vehicle_utils.rs +++ b/rust/fastsim-core/src/vehicle_utils.rs @@ -16,6 +16,7 @@ use std::num::ParseIntError; use std::option::Option; use std::path::PathBuf; use zip::ZipArchive; +use isahc::prelude::*; use crate::air::*; use crate::cycle::RustCycle; @@ -1798,6 +1799,8 @@ pub struct GitVehicleInfo { const VEHICLE_REPO_LIST_URL: &'static str = &"https://api.github.com/repos/NREL/fastsim-vehicles/contents/public"; pub fn fetch_github_list() -> anyhow::Result> { + // let mut response = isahc::get(VEHICLE_REPO_LIST_URL)?; + // let github_list: HashMap = serde_json::from_str(&response.text()?)?; let response = ureq::get(VEHICLE_REPO_LIST_URL).call()?.into_reader(); let github_list: HashMap = serde_json::from_reader(response).with_context(||"Cannot parse github vehicle list.")?; let mut vehicle_name_list: Vec = Vec::new(); @@ -2065,4 +2068,10 @@ mod vehicle_utils_tests { .unwrap() .is_empty()); } + + #[test] + fn test_fetch_github_list() { + let list = fetch_github_list().unwrap(); + println!("{:?}", list); + } } From da1f8a404db34f8b6b0d96d920d103ed5cc460a8 Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Fri, 5 Jan 2024 14:56:46 -0700 Subject: [PATCH 14/25] updating test to produce test file for comparison --- rust/fastsim-core/src/traits.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/fastsim-core/src/traits.rs b/rust/fastsim-core/src/traits.rs index 2b0b7674..c573273d 100644 --- a/rust/fastsim-core/src/traits.rs +++ b/rust/fastsim-core/src/traits.rs @@ -303,6 +303,7 @@ mod tests { #[test] fn test_from_url() { let vehicle = crate::vehicle::RustVehicle::from_url("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); + vehicle.to_file("/Users/rsteutev/Documents/GitHub/fastsim/rust/fastsim-core/resources/test_from_url_vehicle.yaml").unwrap(); let comparison_vehicle = crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); // let comparison_vehicle = crate::vehicle::RustVehicle::from_file("/Users/rsteutev/Documents/GitHub/fastsim/rust/fastsim-core/src/test_vehicle.json").unwrap(); println!("{}", vehicle.to_yaml().unwrap()); From 1301482ff3f9707e203d179ac906dc07eb903cbb Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Fri, 5 Jan 2024 21:11:57 -0700 Subject: [PATCH 15/25] troubleshooting fetch_github_list --- rust/fastsim-core/src/traits.rs | 4 - rust/fastsim-core/src/vehicle.rs | 160 ++++++++++++------------- rust/fastsim-core/src/vehicle_utils.rs | 18 ++- 3 files changed, 91 insertions(+), 91 deletions(-) diff --git a/rust/fastsim-core/src/traits.rs b/rust/fastsim-core/src/traits.rs index c573273d..f21ce560 100644 --- a/rust/fastsim-core/src/traits.rs +++ b/rust/fastsim-core/src/traits.rs @@ -1,7 +1,6 @@ use crate::imports::*; use std::collections::HashMap; use ureq; -use isahc::prelude::*; pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { const ACCEPTED_BYTE_FORMATS: &'static [&'static str] = &["yaml", "json", "bin"]; @@ -160,8 +159,6 @@ pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { .and_then(|filename| Path::new(filename).extension()) .and_then(OsStr::to_str) .with_context(|| "Could not parse file format from URL: {url:?}")?; - // let mut response = isahc::get(url.as_ref())?; - // Self::from_str(response.text()?, format) let response = ureq::get(url.as_ref()).call()?.into_reader(); Self::from_reader(response, format) } @@ -303,7 +300,6 @@ mod tests { #[test] fn test_from_url() { let vehicle = crate::vehicle::RustVehicle::from_url("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); - vehicle.to_file("/Users/rsteutev/Documents/GitHub/fastsim/rust/fastsim-core/resources/test_from_url_vehicle.yaml").unwrap(); let comparison_vehicle = crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); // let comparison_vehicle = crate::vehicle::RustVehicle::from_file("/Users/rsteutev/Documents/GitHub/fastsim/rust/fastsim-core/src/test_vehicle.json").unwrap(); println!("{}", vehicle.to_yaml().unwrap()); diff --git a/rust/fastsim-core/src/vehicle.rs b/rust/fastsim-core/src/vehicle.rs index 2e3bc993..27dc121f 100644 --- a/rust/fastsim-core/src/vehicle.rs +++ b/rust/fastsim-core/src/vehicle.rs @@ -1434,50 +1434,49 @@ mod tests { let comparison_vehicle = crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); let data_subdirectory = create_project_subdir("vehicles").unwrap(); let file_path_test = data_subdirectory.join("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"); - // // test when cache = false and no url provided - // let vehicle = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", None, false).unwrap(); - // assert_eq! (vehicle, comparison_vehicle); - // // test when cache = false and url provided - // let vehicle_1 = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"), false).unwrap(); - // assert_eq! (vehicle_1, comparison_vehicle); - // // test when cache = true and no url provided and no file downloaded -- general - // let vehicle_2 = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", None, true).unwrap(); - // assert_eq! (vehicle_2, comparison_vehicle); - // std::fs::remove_file(&file_path_test).unwrap(); - // // test when cache = true and no url provided and no file downloaded -- specific - // let vehicle_file_name = "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"; - // let url: Option = None; - // let dir = create_project_subdir("vehicles").with_context(||"Could not determine FASTSim data directory path.").unwrap(); - // let mut file_already_downloaded = false; - // let mut file_path = PathBuf::new(); - // for entry in fs::read_dir(dir).with_context(||"Could not read data directory.").unwrap() { - // let entry = entry.with_context(||"Could not parse entry: {entry:?}").unwrap(); - // file_path = entry.path(); - // let entry_name = file_path.file_name().with_context(||"Could not parse file name from directory file: {entry:?}").unwrap().to_str().with_context(||"Could not parse file name from directory file: {entry:?}").unwrap(); - // if entry_name == vehicle_file_name { - // file_already_downloaded = true; - // break - // } else { - // continue - // } - // } - // assert_eq!(file_already_downloaded, false); - // if file_already_downloaded { - // let vehicle_3 = RustVehicle::from_resource(file_path).unwrap(); - // println!("Accessed vehicle from cache in no file downloaded scenario -- BAD."); - // assert_eq! (vehicle_3, comparison_vehicle); - // } else { - // let url_internal = match url { - // Some(s) => s.to_owned(), - // None => RustVehicle::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name - // }; - // url_to_cache(&url_internal, "vehicles").with_context(||"Unable to cache vehicle from url").unwrap(); - // let vehicle_3 = RustVehicle::from_url(url_internal).unwrap(); - // println!("Downloaded vehicle from url and saved it to cache in no file downloaded scenario."); - // assert_eq! (vehicle_3, comparison_vehicle); - // } + // test when cache = false and no url provided + let vehicle = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", None, false).unwrap(); + assert_eq! (vehicle, comparison_vehicle); + // test when cache = false and url provided + let vehicle_1 = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"), false).unwrap(); + assert_eq! (vehicle_1, comparison_vehicle); + // test when cache = true and no url provided and no file downloaded -- general + let vehicle_2 = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", None, true).unwrap(); + assert_eq! (vehicle_2, comparison_vehicle); + std::fs::remove_file(&file_path_test).unwrap(); + // test when cache = true and no url provided and no file downloaded -- specific + let vehicle_file_name = "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"; + let url: Option = None; + let dir = create_project_subdir("vehicles").with_context(||"Could not determine FASTSim data directory path.").unwrap(); + let mut file_already_downloaded = false; + let mut file_path = PathBuf::new(); + for entry in fs::read_dir(dir).with_context(||"Could not read data directory.").unwrap() { + let entry = entry.with_context(||"Could not parse entry: {entry:?}").unwrap(); + file_path = entry.path(); + let entry_name = file_path.file_name().with_context(||"Could not parse file name from directory file: {entry:?}").unwrap().to_str().with_context(||"Could not parse file name from directory file: {entry:?}").unwrap(); + if entry_name == vehicle_file_name { + file_already_downloaded = true; + break + } else { + continue + } + } + assert_eq!(file_already_downloaded, false); + if file_already_downloaded { + let vehicle_3 = RustVehicle::from_resource(file_path).unwrap(); + println!("Accessed vehicle from cache in no file downloaded scenario -- BAD."); + assert_eq! (vehicle_3, comparison_vehicle); + } else { + let url_internal = match url { + Some(s) => s.to_owned(), + None => RustVehicle::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name + }; + url_to_cache(&url_internal, "vehicles").with_context(||"Unable to cache vehicle from url").unwrap(); + let vehicle_3 = RustVehicle::from_url(url_internal).unwrap(); + println!("Downloaded vehicle from url and saved it to cache in no file downloaded scenario."); + assert_eq! (vehicle_3, comparison_vehicle); + } // testing when cache = true and no url provided and file downloaded -- general - url_to_cache("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", "vehicles").unwrap(); let vehicle_4 = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", None, true).unwrap(); assert_eq! (vehicle_4, comparison_vehicle); // testing when cache = true and no url provided and file downloaded -- specific @@ -1513,46 +1512,43 @@ mod tests { assert_eq! (vehicle_5, comparison_vehicle); } std::fs::remove_file(&file_path_test).unwrap(); - - - // // test when cache = true and url provided and no file downloaded -- general - // let vehicle_6 = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"), true).unwrap(); - // assert_eq! (vehicle_6, comparison_vehicle); - // std::fs::remove_file(&file_path_test).unwrap(); - // // test when cache = true and url provided and no file downloaded -- specific - // let vehicle_file_name_3 = "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"; - // let url_3: Option = Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml".to_string()); - // let dir_3 = create_project_subdir("vehicles").with_context(||"Could not determine FASTSim data directory path.").unwrap(); - // let mut file_already_downloaded_3 = false; - // let mut file_path_3 = PathBuf::new(); - // for entry_3 in fs::read_dir(dir_3).with_context(||"Could not read data directory.").unwrap() { - // let entry_3 = entry_3.with_context(||"Could not parse entry: {entry_3:?}").unwrap(); - // file_path_3 = entry_3.path(); - // let entry_name_3 = file_path_3.file_name().with_context(||"Could not parse file name from directory file: {entry_3:?}").unwrap().to_str().with_context(||"Could not parse file name from directory file: {entry_3:?}").unwrap(); - // if entry_name_3 == vehicle_file_name_3 { - // file_already_downloaded_3 = true; - // break - // } else { - // continue - // } - // } - // assert_eq!(file_already_downloaded_3, false); - // if file_already_downloaded_3 { - // let vehicle_7 = RustVehicle::from_resource(file_path_3).unwrap(); - // println!("Accessed vehicle from cache in no file downloaded scenario -- BAD."); - // assert_eq! (vehicle_7, comparison_vehicle); - // } else { - // let url_internal_3 = match url_3 { - // Some(s) => s.to_owned(), - // None => RustVehicle::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name_3 - // }; - // url_to_cache(&url_internal_3, "vehicles").with_context(||"Unable to cache vehicle from url").unwrap(); - // let vehicle_7 = RustVehicle::from_url(url_internal_3).unwrap(); - // println!("Downloaded vehicle from url and saved it to cache in no file downloaded scenario."); - // assert_eq! (vehicle_7, comparison_vehicle); - // } + // test when cache = true and url provided and no file downloaded -- general + let vehicle_6 = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"), true).unwrap(); + assert_eq! (vehicle_6, comparison_vehicle); + std::fs::remove_file(&file_path_test).unwrap(); + // test when cache = true and url provided and no file downloaded -- specific + let vehicle_file_name_3 = "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"; + let url_3: Option = Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml".to_string()); + let dir_3 = create_project_subdir("vehicles").with_context(||"Could not determine FASTSim data directory path.").unwrap(); + let mut file_already_downloaded_3 = false; + let mut file_path_3 = PathBuf::new(); + for entry_3 in fs::read_dir(dir_3).with_context(||"Could not read data directory.").unwrap() { + let entry_3 = entry_3.with_context(||"Could not parse entry: {entry_3:?}").unwrap(); + file_path_3 = entry_3.path(); + let entry_name_3 = file_path_3.file_name().with_context(||"Could not parse file name from directory file: {entry_3:?}").unwrap().to_str().with_context(||"Could not parse file name from directory file: {entry_3:?}").unwrap(); + if entry_name_3 == vehicle_file_name_3 { + file_already_downloaded_3 = true; + break + } else { + continue + } + } + assert_eq!(file_already_downloaded_3, false); + if file_already_downloaded_3 { + let vehicle_7 = RustVehicle::from_resource(file_path_3).unwrap(); + println!("Accessed vehicle from cache in no file downloaded scenario -- BAD."); + assert_eq! (vehicle_7, comparison_vehicle); + } else { + let url_internal_3 = match url_3 { + Some(s) => s.to_owned(), + None => RustVehicle::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name_3 + }; + url_to_cache(&url_internal_3, "vehicles").with_context(||"Unable to cache vehicle from url").unwrap(); + let vehicle_7 = RustVehicle::from_url(url_internal_3).unwrap(); + println!("Downloaded vehicle from url and saved it to cache in no file downloaded scenario."); + assert_eq! (vehicle_7, comparison_vehicle); + } // testing when cache = true and url provided and file downloaded -- general - url_to_cache("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", "vehicles").unwrap(); let vehicle_8 = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"), true).unwrap(); assert_eq! (vehicle_8, comparison_vehicle); // testing when cache = true and url provided and file downloaded -- specific diff --git a/rust/fastsim-core/src/vehicle_utils.rs b/rust/fastsim-core/src/vehicle_utils.rs index 58e9d723..3f73df18 100644 --- a/rust/fastsim-core/src/vehicle_utils.rs +++ b/rust/fastsim-core/src/vehicle_utils.rs @@ -16,7 +16,7 @@ use std::num::ParseIntError; use std::option::Option; use std::path::PathBuf; use zip::ZipArchive; -use isahc::prelude::*; +use tempfile::tempdir; use crate::air::*; use crate::cycle::RustCycle; @@ -1799,10 +1799,18 @@ pub struct GitVehicleInfo { const VEHICLE_REPO_LIST_URL: &'static str = &"https://api.github.com/repos/NREL/fastsim-vehicles/contents/public"; pub fn fetch_github_list() -> anyhow::Result> { - // let mut response = isahc::get(VEHICLE_REPO_LIST_URL)?; - // let github_list: HashMap = serde_json::from_str(&response.text()?)?; - let response = ureq::get(VEHICLE_REPO_LIST_URL).call()?.into_reader(); - let github_list: HashMap = serde_json::from_reader(response).with_context(||"Cannot parse github vehicle list.")?; + let temp_dir = tempdir()?; + let file_path = temp_dir.path().join("github_vehicle_list.json"); + download_file_from_url(VEHICLE_REPO_LIST_URL, &file_path).with_context(||"Could not download file from url.")?; + // let response = ureq::get(VEHICLE_REPO_LIST_URL).call()?.into_reader(); + let file = File::open(&file_path).with_context(|| { + if !file_path.exists() { + format!("File not found: {file_path:?}") + } else { + format!("Could not open file: {file_path:?}") + } + })?; + let github_list: HashMap = serde_json::from_reader(file).with_context(||"Could not parse github vehicle list.")?; let mut vehicle_name_list: Vec = Vec::new(); for (_key, value) in github_list.iter() { vehicle_name_list.push(value.name.to_owned()) From d72996691c1b24e95139652707f974980835a620 Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Thu, 18 Jan 2024 15:37:47 -0700 Subject: [PATCH 16/25] updating fetch_github_list --- rust/fastsim-core/src/traits.rs | 30 ++-- rust/fastsim-core/src/vehicle.rs | 220 ++++--------------------- rust/fastsim-core/src/vehicle_utils.rs | 104 ++++++++---- 3 files changed, 124 insertions(+), 230 deletions(-) diff --git a/rust/fastsim-core/src/traits.rs b/rust/fastsim-core/src/traits.rs index 655c5cad..e98a90a9 100644 --- a/rust/fastsim-core/src/traits.rs +++ b/rust/fastsim-core/src/traits.rs @@ -5,7 +5,7 @@ use ureq; pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { const ACCEPTED_BYTE_FORMATS: &'static [&'static str] = &["yaml", "json", "bin"]; const ACCEPTED_STR_FORMATS: &'static [&'static str] = &["yaml", "json"]; - const CACHE_FOLDER: &'static str = &"rust_objects"; + const CACHE_FOLDER: &'static str = &""; /// Specialized code to execute upon initialization fn init(&mut self) -> anyhow::Result<()> { @@ -208,11 +208,8 @@ pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { /// takes an instantiated Rust object and saves it in the FASTSim data directory in /// a rust_objects folder - /// WARNING: if there is a file already in the data subdirectory with the - /// same name, it will be replaced by the new file - /// to save to a folder other than rust_objects for a specific object type, - /// in the object-specific SerdeAPI implementation, redefine the - /// CACHE_FOLDER constant to be your choice of folder name + /// WARNING: If there is a file already in the data subdirectory with the + /// same name, it will be replaced by the new file. /// # Arguments /// - self (rust object) /// - file_path: path to file within subdirectory. If only the file name is @@ -246,6 +243,8 @@ pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { let file_path = data_subdirectory.join(file_name); self.to_file(file_path) } + + //TODO from_cache method } pub trait ApproxEq { @@ -343,23 +342,30 @@ mod tests { #[test] fn test_from_url() { let vehicle = crate::vehicle::RustVehicle::from_url("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); - let comparison_vehicle = crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); + let comparison_vehicle = + crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml") + .unwrap(); // let comparison_vehicle = crate::vehicle::RustVehicle::from_file("/Users/rsteutev/Documents/GitHub/fastsim/rust/fastsim-core/src/test_vehicle.json").unwrap(); println!("{}", vehicle.to_yaml().unwrap()); - assert_eq! (vehicle, comparison_vehicle); + assert_eq!(vehicle, comparison_vehicle); } #[test] fn test_to_cache() { - let vehicle_a = crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); - crate::vehicle::RustVehicle::to_cache(&vehicle_a, "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); + let vehicle_a = + crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml") + .unwrap(); + crate::vehicle::RustVehicle::to_cache( + &vehicle_a, + "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", + ) + .unwrap(); let data_subdirectory = create_project_subdir("vehicles").unwrap(); let file_path = data_subdirectory.join("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"); println!("{}", file_path.to_string_lossy()); println!("{}", crate::vehicle::RustVehicle::CACHE_FOLDER); let vehicle_b = crate::vehicle::RustVehicle::from_file(&file_path).unwrap(); - assert_eq! (vehicle_a, vehicle_b); + assert_eq!(vehicle_a, vehicle_b); std::fs::remove_file(file_path).unwrap(); } } - diff --git a/rust/fastsim-core/src/vehicle.rs b/rust/fastsim-core/src/vehicle.rs index 8b939888..67a5ec77 100644 --- a/rust/fastsim-core/src/vehicle.rs +++ b/rust/fastsim-core/src/vehicle.rs @@ -6,12 +6,10 @@ use crate::params::*; use crate::proc_macros::{add_pyo3_api, doc_field, ApproxEq}; #[cfg(feature = "pyo3")] use crate::pyo3imports::*; -use crate::utils::create_project_subdir; use lazy_static::lazy_static; use regex::Regex; use validator::Validate; -use std::fs; // veh_pt_type options pub const CONV: &str = "Conv"; @@ -547,7 +545,8 @@ pub struct RustVehicle { /// RustVehicle rust methods impl RustVehicle { - const VEHICLE_DIRECTORY_URL: &'static str = &"https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/"; + const VEHICLE_DIRECTORY_URL: &'static str = + &"https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/"; /// Sets the following parameters: /// - `ess_mass_kg` /// - `mc_mass_kg` @@ -1027,54 +1026,30 @@ impl RustVehicle { v } - /// Downloads specified vehicle from vehicle repo or url into - /// VEHICLE_DIRECTORY_URL, if not already downloaded. Returns vehicle. - /// Optionally checks if vehicle is already downloaded and uses downloaded - /// version if so, and downloads and stores for later use if not. + /// Downloads specified vehicle from vehicle repo or url and instantiates it + /// into a RustVehicle. Returns vehicle. /// # Arguments /// - vehicle_file_name: file name for vehicle to be downloaded /// - url: url for vehicle to be downloaded, if None, assumed to be /// downloaded from vehicle FASTSim repo - /// - cache: If True, function will first check for vehicle in FASTSim data - /// directory, and if vehicle is already there, will use local version. If - /// vehicle is not stored locally, will download and store vehicle for later - /// use. If False, will not check for vehicle locally or store vehicle - /// locally for later use /// Note: The URL needs to be a URL pointing directly to a file, for example - /// a raw github URL. - pub fn from_github_or_url>(vehicle_file_name: S, url: Option, cache: bool) -> anyhow::Result { - if cache { - let dir = create_project_subdir("vehicles").with_context(||"Could not determine FASTSim data directory path.")?; - let mut file_already_downloaded = false; - let mut file_path = PathBuf::new(); - for entry in fs::read_dir(dir).with_context(||"Could not read data directory.")? { - let entry = entry.with_context(||"Could not parse entry: {entry:?}")?; - file_path = entry.path(); - let entry_name = file_path.file_name().with_context(||"Could not parse file name from directory file: {entry:?}")?.to_str().with_context(||"Could not parse file name from directory file: {entry:?}")?; - if entry_name == vehicle_file_name.as_ref() { - file_already_downloaded = true; - break - } else { - continue - } - } - if file_already_downloaded { - Self::from_file(file_path) - } else { - let url_internal = match url { - Some(s) => s.as_ref().to_owned(), - None => Self::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name.as_ref() - }; - url_to_cache(&url_internal, "vehicles").with_context(||"Unable to cache vehicle from url")?; - Self::from_url(url_internal) - } - } else { - let url_internal = match url { - Some(s) => s.as_ref().to_owned(), - None => Self::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name.as_ref() - }; - Self::from_url(url_internal) - } + /// a raw github URL. + /// Note: If downloading from the FASTSim Vehicle Repo, the + /// vehicle_file_name should include the path to the file from the root of + /// the Repo, as listed in the output of the + /// vehicle_utils::fetch_github_list() function. + pub fn from_github_or_url>( + vehicle_file_name: S, + url: Option, + ) -> anyhow::Result { + let url_internal = match url { + Some(s) => s.as_ref().to_owned(), + None => Self::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name.as_ref(), + }; + let mut vehicle = + Self::from_url(url_internal).with_context(|| "Could not parse vehicle from url")?; + vehicle.doc = Some("Vehicle from {url_internal:?}".to_string()); + return Ok(vehicle); } } @@ -1104,7 +1079,6 @@ impl Default for RustVehicle { } impl SerdeAPI for RustVehicle { - // is this enough, or do I have to copy paste in the whole to_cache mathod? const CACHE_FOLDER: &'static str = &"vehicles"; fn init(&mut self) -> anyhow::Result<()> { @@ -1421,150 +1395,16 @@ mod tests { #[test] fn test_from_github_or_url() { - let comparison_vehicle = crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); - let data_subdirectory = create_project_subdir("vehicles").unwrap(); - let file_path_test = data_subdirectory.join("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"); + let comparison_vehicle = + crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml") + .unwrap(); // test when cache = false and no url provided - let vehicle = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", None, false).unwrap(); - assert_eq! (vehicle, comparison_vehicle); + let vehicle = + RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", None) + .unwrap(); + assert_eq!(vehicle, comparison_vehicle); // test when cache = false and url provided - let vehicle_1 = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"), false).unwrap(); - assert_eq! (vehicle_1, comparison_vehicle); - // test when cache = true and no url provided and no file downloaded -- general - let vehicle_2 = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", None, true).unwrap(); - assert_eq! (vehicle_2, comparison_vehicle); - std::fs::remove_file(&file_path_test).unwrap(); - // test when cache = true and no url provided and no file downloaded -- specific - let vehicle_file_name = "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"; - let url: Option = None; - let dir = create_project_subdir("vehicles").with_context(||"Could not determine FASTSim data directory path.").unwrap(); - let mut file_already_downloaded = false; - let mut file_path = PathBuf::new(); - for entry in fs::read_dir(dir).with_context(||"Could not read data directory.").unwrap() { - let entry = entry.with_context(||"Could not parse entry: {entry:?}").unwrap(); - file_path = entry.path(); - let entry_name = file_path.file_name().with_context(||"Could not parse file name from directory file: {entry:?}").unwrap().to_str().with_context(||"Could not parse file name from directory file: {entry:?}").unwrap(); - if entry_name == vehicle_file_name { - file_already_downloaded = true; - break - } else { - continue - } - } - assert_eq!(file_already_downloaded, false); - if file_already_downloaded { - panic!("Accessed vehicle from cache in no file downloaded scenario -- BAD."); - } else { - let url_internal = match url { - Some(s) => s.to_owned(), - None => RustVehicle::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name - }; - url_to_cache(&url_internal, "vehicles").with_context(||"Unable to cache vehicle from url").unwrap(); - let vehicle_3 = RustVehicle::from_url(url_internal).unwrap(); - println!("Downloaded vehicle from url and saved it to cache in no file downloaded scenario."); - assert_eq! (vehicle_3, comparison_vehicle); - } - // testing when cache = true and no url provided and file downloaded -- general - let vehicle_4 = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", None, true).unwrap(); - assert_eq! (vehicle_4, comparison_vehicle); - // testing when cache = true and no url provided and file downloaded -- specific - let vehicle_file_name_2 = "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"; - let url_2: Option = None; - let dir_2 = create_project_subdir("vehicles").with_context(||"Could not determine FASTSim data directory path.").unwrap(); - let mut file_already_downloaded_2 = false; - let mut file_path_2 = PathBuf::new(); - for entry_2 in fs::read_dir(dir_2).with_context(||"Could not read data directory.").unwrap() { - let entry_2 = entry_2.with_context(||"Could not parse entry: {entry_2:?}").unwrap(); - file_path_2 = entry_2.path(); - let entry_name_2 = file_path_2.file_name().with_context(||"Could not parse file name from directory file: {entry_2:?}").unwrap().to_str().with_context(||"Could not parse file name from directory file: {entry_2:?}").unwrap(); - if entry_name_2 == vehicle_file_name_2 { - file_already_downloaded_2 = true; - break - } else { - continue - } - } - assert_eq!(file_already_downloaded_2, true); - if file_already_downloaded_2 { - let vehicle_5 = RustVehicle::from_file(file_path_2).unwrap(); - println!("Accessed vehicle from cache in vehicle already downloaded scenario."); - assert_eq! (vehicle_5, comparison_vehicle); - } else { - let url_internal_2 = match url_2 { - Some(s) => s.to_owned(), - None => RustVehicle::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name_2 - }; - url_to_cache(&url_internal_2, "vehicles").with_context(||"Unable to cache vehicle from url").unwrap(); - panic!("Downloaded vehicle from url and saved it to cache in vehicle already downloaded scenario -- BAD."); - } - std::fs::remove_file(&file_path_test).unwrap(); - // test when cache = true and url provided and no file downloaded -- general - let vehicle_6 = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"), true).unwrap(); - assert_eq! (vehicle_6, comparison_vehicle); - std::fs::remove_file(&file_path_test).unwrap(); - // test when cache = true and url provided and no file downloaded -- specific - let vehicle_file_name_3 = "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"; - let url_3: Option = Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml".to_string()); - let dir_3 = create_project_subdir("vehicles").with_context(||"Could not determine FASTSim data directory path.").unwrap(); - let mut file_already_downloaded_3 = false; - let mut file_path_3 = PathBuf::new(); - for entry_3 in fs::read_dir(dir_3).with_context(||"Could not read data directory.").unwrap() { - let entry_3 = entry_3.with_context(||"Could not parse entry: {entry_3:?}").unwrap(); - file_path_3 = entry_3.path(); - let entry_name_3 = file_path_3.file_name().with_context(||"Could not parse file name from directory file: {entry_3:?}").unwrap().to_str().with_context(||"Could not parse file name from directory file: {entry_3:?}").unwrap(); - if entry_name_3 == vehicle_file_name_3 { - file_already_downloaded_3 = true; - break - } else { - continue - } - } - assert_eq!(file_already_downloaded_3, false); - if file_already_downloaded_3 { - panic!("Accessed vehicle from cache in no file downloaded scenario -- BAD."); - } else { - let url_internal_3 = match url_3 { - Some(s) => s.to_owned(), - None => RustVehicle::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name_3 - }; - url_to_cache(&url_internal_3, "vehicles").with_context(||"Unable to cache vehicle from url").unwrap(); - let vehicle_7 = RustVehicle::from_url(url_internal_3).unwrap(); - println!("Downloaded vehicle from url and saved it to cache in no file downloaded scenario."); - assert_eq! (vehicle_7, comparison_vehicle); - } - // testing when cache = true and url provided and file downloaded -- general - let vehicle_8 = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"), true).unwrap(); - assert_eq! (vehicle_8, comparison_vehicle); - // testing when cache = true and url provided and file downloaded -- specific - let vehicle_file_name_4 = "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"; - let url_4: Option = Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml".to_string()); - let dir_4 = create_project_subdir("vehicles").with_context(||"Could not determine FASTSim data directory path.").unwrap(); - let mut file_already_downloaded_4 = false; - let mut file_path_4 = PathBuf::new(); - for entry_4 in fs::read_dir(dir_4).with_context(||"Could not read data directory.").unwrap() { - let entry_4 = entry_4.with_context(||"Could not parse entry: {entry_4:?}").unwrap(); - file_path_4 = entry_4.path(); - let entry_name_4 = file_path_4.file_name().with_context(||"Could not parse file name from directory file: {entry_4:?}").unwrap().to_str().with_context(||"Could not parse file name from directory file: {entry_4:?}").unwrap(); - if entry_name_4 == vehicle_file_name_4 { - file_already_downloaded_4 = true; - break - } else { - continue - } - } - assert_eq!(file_already_downloaded_4, true); - if file_already_downloaded_4 { - let vehicle_9 = RustVehicle::from_file(file_path_4).unwrap(); - println!("Accessed vehicle from cache in vehicle already downloaded scenario."); - assert_eq! (vehicle_9, comparison_vehicle); - } else { - let url_internal_4 = match url_4 { - Some(s) => s.to_owned(), - None => RustVehicle::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name_4 - }; - url_to_cache(&url_internal_4, "vehicles").with_context(||"Unable to cache vehicle from url").unwrap(); - panic!("Downloaded vehicle from url and saved it to cache in vehicle already downloaded scenario -- BAD."); - } - std::fs::remove_file(&file_path_test).unwrap(); + let vehicle_1 = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml")).unwrap(); + assert_eq!(vehicle_1, comparison_vehicle); } } diff --git a/rust/fastsim-core/src/vehicle_utils.rs b/rust/fastsim-core/src/vehicle_utils.rs index 3f73df18..3d788c98 100644 --- a/rust/fastsim-core/src/vehicle_utils.rs +++ b/rust/fastsim-core/src/vehicle_utils.rs @@ -15,8 +15,9 @@ use std::iter::FromIterator; use std::num::ParseIntError; use std::option::Option; use std::path::PathBuf; +use std::{result::Result, thread, time::Duration}; +use ureq::{Error as OtherError, Error::Status, Response}; use zip::ZipArchive; -use tempfile::tempdir; use crate::air::*; use crate::cycle::RustCycle; @@ -1773,47 +1774,91 @@ pub fn import_and_save_all_vehicles( } #[derive(Deserialize)] -pub struct VehicleLinks { +pub struct ObjectLinks { #[serde(rename = "self")] - pub self_url: String, - pub git: String, - pub html: String, + pub self_url: Option, + pub git: Option, + pub html: Option, } #[derive(Deserialize)] -pub struct GitVehicleInfo { +pub struct GitObjectInfo { pub name: String, pub path: String, - pub sha: String, - pub size: i64, + pub sha: Option, + pub size: Option, pub url: String, - pub html_url: String, - pub git_url: String, - pub download_url: String, + pub html_url: Option, + pub git_url: Option, + pub download_url: Option, #[serde(rename = "type")] pub url_type: String, #[serde(rename = "_links")] - pub links: VehicleLinks, + pub links: Option, } -const VEHICLE_REPO_LIST_URL: &'static str = &"https://api.github.com/repos/NREL/fastsim-vehicles/contents/public"; +const VEHICLE_REPO_LIST_URL: &'static str = + &"https://api.github.com/repos/NREL/fastsim-vehicles/contents/public"; + +/// Function that takes a url and calls the url. If a 503 or 429 error is +/// thrown, it tries again after a pause, up to four times. It returns either a +/// result or an error. +/// # Arguments +/// - url: url to be called +/// Source: https://docs.rs/ureq/latest/ureq/enum.Error.html +fn get_response>(url: S) -> Result { + for _ in 1..4 { + match ureq::get(url.as_ref()).call() { + Err(Status(503, r)) | Err(Status(429, r)) => { + let retry: Option = r.header("retry-after").and_then(|h| h.parse().ok()); + let retry = retry.unwrap_or(5); + eprintln!("{} for {}, retry in {}", r.status(), r.get_url(), retry); + thread::sleep(Duration::from_secs(retry)); + } + result => return result, + }; + } + // Ran out of retries; try one last time and return whatever result we get. + ureq::get(url.as_ref()).call() +} -pub fn fetch_github_list() -> anyhow::Result> { - let temp_dir = tempdir()?; - let file_path = temp_dir.path().join("github_vehicle_list.json"); - download_file_from_url(VEHICLE_REPO_LIST_URL, &file_path).with_context(||"Could not download file from url.")?; - // let response = ureq::get(VEHICLE_REPO_LIST_URL).call()?.into_reader(); - let file = File::open(&file_path).with_context(|| { - if !file_path.exists() { - format!("File not found: {file_path:?}") +/// Returns a list of vehicle file names in the Fastsim Vehicle Repo, or, +/// optionally, a different GitHub repo, in which case the url provided needs to +/// be the url for the file tree within GitHub for the root folder the Rust +/// objects, for example +/// "https://api.github.com/repos/NREL/fastsim-vehicles/contents/public" +/// Note: for each file, the output will list the vehicle file name, including +/// the path from the root of the repository +/// # Arguments +/// - repo_url: url to the GitHub repository, Option, if None, defaults to the +/// FASTSim Vehicle Repo +pub fn fetch_github_list(repo_url: Option) -> anyhow::Result> { + let repo_url = repo_url.unwrap_or(VEHICLE_REPO_LIST_URL.to_string()); + let response = get_response(repo_url)?.into_reader(); + let github_list: Vec = + serde_json::from_reader(response).with_context(|| "Cannot parse github vehicle list.")?; + let mut vehicle_name_list: Vec = Vec::new(); + for object in github_list.iter() { + if object.url_type == "dir" { + let url = &object.url; + fetch_github_list(Some(url.to_owned()))?; + } else if object.url_type == "file" { + let url = url::Url::parse(&object.url)?; + let path = &object.path; + let format = url + .path_segments() + .and_then(|segments| segments.last()) + .and_then(|filename| Path::new(filename).extension()) + .and_then(OsStr::to_str) + .with_context(|| "Could not parse file format from URL: {url:?}")?; + match format.trim_start_matches('.').to_lowercase().as_str() { + "yaml" | "yml" => vehicle_name_list.push(path.to_owned()), + "json" => vehicle_name_list.push(path.to_owned()), + _ => continue, + } } else { - format!("Could not open file: {file_path:?}") + continue; } - })?; - let github_list: HashMap = serde_json::from_reader(file).with_context(||"Could not parse github vehicle list.")?; - let mut vehicle_name_list: Vec = Vec::new(); - for (_key, value) in github_list.iter() { - vehicle_name_list.push(value.name.to_owned()) } Ok(vehicle_name_list) } @@ -2079,7 +2124,10 @@ mod vehicle_utils_tests { #[test] fn test_fetch_github_list() { - let list = fetch_github_list().unwrap(); + let list = fetch_github_list(Some( + "https://github.com/NREL/fastsim-vehicles/tree/main".to_owned(), + )) + .unwrap(); println!("{:?}", list); } } From 364395aa7140229636929bd95eb53d34f2b8f720 Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Thu, 18 Jan 2024 16:04:38 -0700 Subject: [PATCH 17/25] fixing looping problem with fetch_github_list --- rust/fastsim-core/src/vehicle_utils.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rust/fastsim-core/src/vehicle_utils.rs b/rust/fastsim-core/src/vehicle_utils.rs index 3d788c98..29823b40 100644 --- a/rust/fastsim-core/src/vehicle_utils.rs +++ b/rust/fastsim-core/src/vehicle_utils.rs @@ -1841,7 +1841,10 @@ pub fn fetch_github_list(repo_url: Option) -> anyhow::Result for object in github_list.iter() { if object.url_type == "dir" { let url = &object.url; - fetch_github_list(Some(url.to_owned()))?; + let vehicle_name_sublist = fetch_github_list(Some(url.to_owned()))?; + for name in vehicle_name_sublist.iter() { + vehicle_name_list.push(name.to_owned()); + } } else if object.url_type == "file" { let url = url::Url::parse(&object.url)?; let path = &object.path; @@ -2128,6 +2131,7 @@ mod vehicle_utils_tests { "https://github.com/NREL/fastsim-vehicles/tree/main".to_owned(), )) .unwrap(); + // let list = fetch_github_list(None).unwrap(); println!("{:?}", list); } } From 49ba05420e40737cd8f13de1383c343e664b699f Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Thu, 18 Jan 2024 16:31:35 -0700 Subject: [PATCH 18/25] fixing fetch_github_list test so it runs --- rust/fastsim-core/src/vehicle_utils.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rust/fastsim-core/src/vehicle_utils.rs b/rust/fastsim-core/src/vehicle_utils.rs index 29823b40..0b3d792b 100644 --- a/rust/fastsim-core/src/vehicle_utils.rs +++ b/rust/fastsim-core/src/vehicle_utils.rs @@ -2128,10 +2128,11 @@ mod vehicle_utils_tests { #[test] fn test_fetch_github_list() { let list = fetch_github_list(Some( - "https://github.com/NREL/fastsim-vehicles/tree/main".to_owned(), + "https://api.github.com/repos/NREL/fastsim-vehicles/contents".to_owned(), )) .unwrap(); - // let list = fetch_github_list(None).unwrap(); + let other_list = fetch_github_list(None).unwrap(); println!("{:?}", list); + println!("{:?}", other_list); } } From 1f9860eddceb5067282cfc2b08fef24ef6df9d4c Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Fri, 19 Jan 2024 15:39:44 -0700 Subject: [PATCH 19/25] adding path_to_cache and from_cache --- rust/fastsim-core/src/traits.rs | 6 +- rust/fastsim-core/src/utils.rs | 26 ++++++- rust/fastsim-core/src/vehicle.rs | 115 ++++++++++++++++++++++++++----- 3 files changed, 125 insertions(+), 22 deletions(-) diff --git a/rust/fastsim-core/src/traits.rs b/rust/fastsim-core/src/traits.rs index e98a90a9..0123ff5a 100644 --- a/rust/fastsim-core/src/traits.rs +++ b/rust/fastsim-core/src/traits.rs @@ -244,7 +244,11 @@ pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { self.to_file(file_path) } - //TODO from_cache method + fn from_cache>(file_path: P) -> anyhow::Result { + let full_file_path = Path::new(Self::CACHE_FOLDER).join(file_path); + let path_including_directory = path_to_cache(Some(full_file_path))?; + Self::from_file(path_including_directory) + } } pub trait ApproxEq { diff --git a/rust/fastsim-core/src/utils.rs b/rust/fastsim-core/src/utils.rs index f2ddbf8c..a3940726 100644 --- a/rust/fastsim-core/src/utils.rs +++ b/rust/fastsim-core/src/utils.rs @@ -526,6 +526,17 @@ pub fn create_project_subdir>(subpath: P) -> anyhow::Result>(subpath: Option

) -> anyhow::Result { + let proj_dirs = ProjectDirs::from("gov", "NREL", "fastsim").ok_or_else(|| { + anyhow!("Could not build path to project directory: \"gov.NREL.fastsim\"") + })?; + let path = match subpath { + Some(subpath) => PathBuf::from(proj_dirs.config_dir()).join(subpath.as_ref()), + None => PathBuf::from(proj_dirs.config_dir()), + }; + Ok(path) +} + /// takes an object from a url and saves it in the FASTSim data directory in a /// rust_objects folder /// WARNING: if there is a file already in the data subdirectory with the same @@ -545,7 +556,8 @@ pub fn url_to_cache, P: AsRef>(url: S, subpath: P) -> anyhow .path_segments() .and_then(|segments| segments.last()) .with_context(|| "Could not parse filename from URL: {url:?}")?; - let data_subdirectory = create_project_subdir(subpath).with_context(||"Could not find or build Fastsim data subdirectory.")?; + let data_subdirectory = create_project_subdir(subpath) + .with_context(|| "Could not find or build Fastsim data subdirectory.")?; let file_path = data_subdirectory.join(file_name); download_file_from_url(url.as_ref(), &file_path)?; Ok(()) @@ -766,8 +778,16 @@ mod tests { let file_path = data_subdirectory.join("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"); println!("{}", file_path.to_string_lossy()); let vehicle = crate::vehicle::RustVehicle::from_file(&file_path).unwrap(); - let comparison_vehicle = crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); - assert_eq! (vehicle, comparison_vehicle); + let comparison_vehicle = + crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml") + .unwrap(); + assert_eq!(vehicle, comparison_vehicle); std::fs::remove_file(file_path).unwrap(); } + + // #[test] + // fn test_path_to_cache() { + // let path = path_to_cache(None).unwrap(); + // println!("{:?}", path) + // } } diff --git a/rust/fastsim-core/src/vehicle.rs b/rust/fastsim-core/src/vehicle.rs index 67a5ec77..ca9697d9 100644 --- a/rust/fastsim-core/src/vehicle.rs +++ b/rust/fastsim-core/src/vehicle.rs @@ -1026,30 +1026,41 @@ impl RustVehicle { v } - /// Downloads specified vehicle from vehicle repo or url and instantiates it - /// into a RustVehicle. Returns vehicle. + /// Downloads specified vehicle from FASTSim vehicle repo or url and + /// instantiates it into a RustVehicle. Notes in vehicle.doc the origin of + /// the vehicle. Returns vehicle. /// # Arguments - /// - vehicle_file_name: file name for vehicle to be downloaded - /// - url: url for vehicle to be downloaded, if None, assumed to be - /// downloaded from vehicle FASTSim repo + /// - vehicle_file_name: file name for vehicle to be downloaded, including + /// path from url directory or FASTSim repository (if applicable) + /// - url: url for vehicle repository where vehicle will be downloaded from, + /// if None, assumed to be downloaded from vehicle FASTSim repo /// Note: The URL needs to be a URL pointing directly to a file, for example - /// a raw github URL. + /// a raw github URL, split up so that the "url" argument is the path to the + /// directory, and the "vehicle_file_name" is the path within the directory + /// to the file. /// Note: If downloading from the FASTSim Vehicle Repo, the /// vehicle_file_name should include the path to the file from the root of /// the Repo, as listed in the output of the - /// vehicle_utils::fetch_github_list() function. + /// vehicle_utils::fetch_github_list() function. + /// Note: the url should not include the file name, only the path to the + /// file or a root directory of the file. pub fn from_github_or_url>( vehicle_file_name: S, url: Option, ) -> anyhow::Result { let url_internal = match url { - Some(s) => s.as_ref().to_owned(), + Some(s) => { + s.as_ref().trim_end_matches('/').to_owned() + + "/" + + &vehicle_file_name.as_ref().trim_start_matches('/') + } None => Self::VEHICLE_DIRECTORY_URL.to_string() + vehicle_file_name.as_ref(), }; let mut vehicle = - Self::from_url(url_internal).with_context(|| "Could not parse vehicle from url")?; - vehicle.doc = Some("Vehicle from {url_internal:?}".to_string()); - return Ok(vehicle); + Self::from_url(&url_internal).with_context(|| "Could not parse vehicle from url")?; + let vehicle_origin = "Vehicle from ".to_owned() + url_internal.as_str(); + vehicle.doc = Some(vehicle_origin); + Ok(vehicle) } } @@ -1084,6 +1095,28 @@ impl SerdeAPI for RustVehicle { fn init(&mut self) -> anyhow::Result<()> { self.set_derived() } + + /// instantiates a vehicle from a url, and notes in vehicle.doc the origin + /// of the vehicle. + /// accepts yaml and json file types + /// # Arguments + /// - url: URL (either as a string or url type) to object + /// Note: The URL needs to be a URL pointing directly to a file, for example + /// a raw github URL. + fn from_url>(url: S) -> anyhow::Result { + let url = url::Url::parse(url.as_ref())?; + let format = url + .path_segments() + .and_then(|segments| segments.last()) + .and_then(|filename| Path::new(filename).extension()) + .and_then(OsStr::to_str) + .with_context(|| "Could not parse file format from URL: {url:?}")?; + let response = ureq::get(url.as_ref()).call()?.into_reader(); + let mut vehicle = Self::from_reader(response, format)?; + let vehicle_origin = "Vehicle from ".to_owned() + url.as_ref(); + vehicle.doc = Some(vehicle_origin); + Ok(vehicle) + } } #[cfg(test)] @@ -1395,16 +1428,62 @@ mod tests { #[test] fn test_from_github_or_url() { - let comparison_vehicle = + let mut comparison_vehicle = crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml") .unwrap(); - // test when cache = false and no url provided - let vehicle = - RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", None) - .unwrap(); + comparison_vehicle.doc = Some("Vehicle from https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml".to_owned()); + // test no url provided + let vehicle = RustVehicle::from_github_or_url( + "public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", + None, + ) + .unwrap(); assert_eq!(vehicle, comparison_vehicle); - // test when cache = false and url provided - let vehicle_1 = RustVehicle::from_github_or_url("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml")).unwrap(); + // test url provided + let vehicle_1 = RustVehicle::from_github_or_url( + "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", + Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/"), + ) + .unwrap(); assert_eq!(vehicle_1, comparison_vehicle); + let vehicle_2 = RustVehicle::from_github_or_url( + "/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", + Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public"), + ) + .unwrap(); + assert_eq!(vehicle_2, comparison_vehicle); + let vehicle_3 = RustVehicle::from_github_or_url( + "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", + Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public"), + ) + .unwrap(); + assert_eq!(vehicle_3, comparison_vehicle); + let vehicle_4 = RustVehicle::from_github_or_url( + "public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", + Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/"), + ) + .unwrap(); + assert_eq!(vehicle_4, comparison_vehicle); + let vehicle_5 = RustVehicle::from_github_or_url( + "/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", + Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main"), + ) + .unwrap(); + assert_eq!(vehicle_5, comparison_vehicle); + let vehicle_6 = RustVehicle::from_github_or_url( + "public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", + Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main"), + ) + .unwrap(); + assert_eq!(vehicle_6, comparison_vehicle); + } + + #[test] + fn test_from_url() { + let vehicle = RustVehicle::from_url("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); + let mut comparison_vehicle = + RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); + comparison_vehicle.doc = Some("Vehicle from https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml".to_owned()); + assert_eq!(vehicle, comparison_vehicle); } } From 6086641e1c68f37de1a8276e693260264ee7d9dd Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Tue, 23 Jan 2024 12:16:31 -0700 Subject: [PATCH 20/25] ading clear cache function --- rust/fastsim-core/src/traits.rs | 39 +++++++++++++++++++++---- rust/fastsim-core/src/utils.rs | 52 +++++++++++++++++++++++++++------ 2 files changed, 77 insertions(+), 14 deletions(-) diff --git a/rust/fastsim-core/src/traits.rs b/rust/fastsim-core/src/traits.rs index 0123ff5a..de7370fb 100644 --- a/rust/fastsim-core/src/traits.rs +++ b/rust/fastsim-core/src/traits.rs @@ -244,9 +244,25 @@ pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { self.to_file(file_path) } + /// Instantiates a Rust object from the subdirectory within the FASTSim data + /// directory corresponding to the Rust Object ("vehices" for a RustVehice, + /// "cycles" for a RustCycle, and the root folder of the data directory for + /// all other objects). + /// # Arguments + /// - file_path: subpath to object, including file name, within subdirectory. + /// If the file sits directly in the subdirectory, this will just be the + /// file name. + /// Note: This function will work for all objects cached using the + /// to_cache() method. If a file has been saved manually to a different + /// subdirectory than the correct one for the object type (for instance a + /// RustVehicle saved within a subdirectory other than "vehicles" using the + /// utils::url_to_cache() function), then from_cache() will not be able to + /// find and instantiate the object. Instead, use the from_file method, and + /// use the utils::path_to_cache() to find the FASTSim data directory + /// location if needed. fn from_cache>(file_path: P) -> anyhow::Result { let full_file_path = Path::new(Self::CACHE_FOLDER).join(file_path); - let path_including_directory = path_to_cache(Some(full_file_path))?; + let path_including_directory = path_to_cache()?.join(full_file_path); Self::from_file(path_including_directory) } } @@ -349,18 +365,17 @@ mod tests { let comparison_vehicle = crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml") .unwrap(); - // let comparison_vehicle = crate::vehicle::RustVehicle::from_file("/Users/rsteutev/Documents/GitHub/fastsim/rust/fastsim-core/src/test_vehicle.json").unwrap(); println!("{}", vehicle.to_yaml().unwrap()); assert_eq!(vehicle, comparison_vehicle); } #[test] fn test_to_cache() { - let vehicle_a = + let comparison_vehicle = crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml") .unwrap(); crate::vehicle::RustVehicle::to_cache( - &vehicle_a, + &comparison_vehicle, "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", ) .unwrap(); @@ -369,7 +384,21 @@ mod tests { println!("{}", file_path.to_string_lossy()); println!("{}", crate::vehicle::RustVehicle::CACHE_FOLDER); let vehicle_b = crate::vehicle::RustVehicle::from_file(&file_path).unwrap(); - assert_eq!(vehicle_a, vehicle_b); + assert_eq!(comparison_vehicle, vehicle_b); std::fs::remove_file(file_path).unwrap(); } + + #[test] + fn test_from_cache() { + let test_path = "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"; + let comparison_vehicle = + crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml") + .unwrap(); + crate::vehicle::RustVehicle::to_cache(&comparison_vehicle, test_path).unwrap(); + let vehicle = crate::vehicle::RustVehicle::from_cache(test_path).unwrap(); + assert_eq!(comparison_vehicle, vehicle); + let full_file_path = Path::new("vehicles").join(test_path); + let path_including_directory = path_to_cache().unwrap().join(full_file_path); + std::fs::remove_file(path_including_directory).unwrap(); + } } diff --git a/rust/fastsim-core/src/utils.rs b/rust/fastsim-core/src/utils.rs index a3940726..5a50032a 100644 --- a/rust/fastsim-core/src/utils.rs +++ b/rust/fastsim-core/src/utils.rs @@ -526,15 +526,33 @@ pub fn create_project_subdir>(subpath: P) -> anyhow::Result>(subpath: Option

) -> anyhow::Result { +/// Returns the path to the OS-specific data directory, if it exists. +pub fn path_to_cache() -> anyhow::Result { let proj_dirs = ProjectDirs::from("gov", "NREL", "fastsim").ok_or_else(|| { anyhow!("Could not build path to project directory: \"gov.NREL.fastsim\"") })?; + Ok(PathBuf::from(proj_dirs.config_dir())) +} + +/// Deletes FASTSim data directory, clearing its contents. If subpath is +/// provided, will only delete the subdirectory pointed to by the subpath, +/// rather than deleting the whole data directory. +/// USE WITH CAUTION, as this function deletes ALL objects stored in the FASTSim +/// data directory or provided subdirectory. +/// # Arguments +/// - subpath (Option): Subpath to a subdirectory within the FASTSim data +/// directory. If None, the function will deleter the whole FASTSim data +/// directory, clearing all its contents. +/// Note: it is not possible to delete single files using this function, only +/// directories. If a single file needs deleting, the path_to_cache() function +/// can be used to find the FASTSim data directory location. The file can then +/// be found and manually deleted. +pub fn clear_cache(subpath: Option) -> anyhow::Result<()> { let path = match subpath { - Some(subpath) => PathBuf::from(proj_dirs.config_dir()).join(subpath.as_ref()), - None => PathBuf::from(proj_dirs.config_dir()), + Some(subpath) => path_to_cache()?.join(subpath), + None => path_to_cache()?, }; - Ok(path) + Ok(std::fs::remove_dir_all(path)?) } /// takes an object from a url and saves it in the FASTSim data directory in a @@ -785,9 +803,25 @@ mod tests { std::fs::remove_file(file_path).unwrap(); } - // #[test] - // fn test_path_to_cache() { - // let path = path_to_cache(None).unwrap(); - // println!("{:?}", path) - // } + #[test] + fn test_path_to_cache() { + let path = path_to_cache().unwrap(); + println!("{:?}", path); + } + + #[test] + fn test_clear_cache() { + // let temp_sub_dir = tempfile::TempDir::new_in(path_to_cache().unwrap()).unwrap(); + // let sub_dir_path = temp_sub_dir.path().to_str().unwrap(); + // url_to_cache("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", "vehicles").unwrap(); + // clear_cache(Some(sub_dir_path.to_string())).unwrap(); + // let still_exists = std::fs::metadata(sub_dir_path).is_ok(); + // assert_eq!(still_exists, false); + // let path_to_vehicle = path_to_cache().unwrap().join("vehicles/https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"); + // let vehicle_still_exists = std::fs::metadata(path_to_vehicle).is_ok(); + // assert_eq!(vehicle_still_exists, true); + // // delete vehicle from subdir (for first attept, use clear_cache to clear entire cache -- but get rid of this once tested) + // std::fs::remove_file(path_to_vehicle).unwrap(); + // // test to see if an empty string will clear entire cache + } } From 4c4f1d9b54bf252c6842a08002fe953f40383c93 Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Tue, 23 Jan 2024 15:20:11 -0700 Subject: [PATCH 21/25] updating tests so they pass --- rust/fastsim-core/src/traits.rs | 26 ++++++----------- rust/fastsim-core/src/utils.rs | 40 +++++++++++++------------- rust/fastsim-core/src/vehicle_utils.rs | 3 ++ 3 files changed, 31 insertions(+), 38 deletions(-) diff --git a/rust/fastsim-core/src/traits.rs b/rust/fastsim-core/src/traits.rs index de7370fb..173b6230 100644 --- a/rust/fastsim-core/src/traits.rs +++ b/rust/fastsim-core/src/traits.rs @@ -360,13 +360,17 @@ mod tests { use super::*; #[test] - fn test_from_url() { - let vehicle = crate::vehicle::RustVehicle::from_url("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); + fn test_from_cache() { + let test_path = "1110_2022_Tesla_Model_Y_RWD_opt45017_from_cache.yaml"; let comparison_vehicle = crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml") .unwrap(); - println!("{}", vehicle.to_yaml().unwrap()); - assert_eq!(vehicle, comparison_vehicle); + crate::vehicle::RustVehicle::to_cache(&comparison_vehicle, test_path).unwrap(); + let vehicle = crate::vehicle::RustVehicle::from_cache(test_path).unwrap(); + assert_eq!(comparison_vehicle, vehicle); + let full_file_path = Path::new("vehicles").join(test_path); + let path_including_directory = path_to_cache().unwrap().join(full_file_path); + std::fs::remove_file(path_including_directory).unwrap(); } #[test] @@ -387,18 +391,4 @@ mod tests { assert_eq!(comparison_vehicle, vehicle_b); std::fs::remove_file(file_path).unwrap(); } - - #[test] - fn test_from_cache() { - let test_path = "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"; - let comparison_vehicle = - crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml") - .unwrap(); - crate::vehicle::RustVehicle::to_cache(&comparison_vehicle, test_path).unwrap(); - let vehicle = crate::vehicle::RustVehicle::from_cache(test_path).unwrap(); - assert_eq!(comparison_vehicle, vehicle); - let full_file_path = Path::new("vehicles").join(test_path); - let path_including_directory = path_to_cache().unwrap().join(full_file_path); - std::fs::remove_file(path_including_directory).unwrap(); - } } diff --git a/rust/fastsim-core/src/utils.rs b/rust/fastsim-core/src/utils.rs index 5a50032a..707399b3 100644 --- a/rust/fastsim-core/src/utils.rs +++ b/rust/fastsim-core/src/utils.rs @@ -536,22 +536,20 @@ pub fn path_to_cache() -> anyhow::Result { /// Deletes FASTSim data directory, clearing its contents. If subpath is /// provided, will only delete the subdirectory pointed to by the subpath, -/// rather than deleting the whole data directory. +/// rather than deleting the whole data directory. If the subpath is an empty +/// string, deletes the entire FASTSim directory. /// USE WITH CAUTION, as this function deletes ALL objects stored in the FASTSim /// data directory or provided subdirectory. /// # Arguments -/// - subpath (Option): Subpath to a subdirectory within the FASTSim data -/// directory. If None, the function will deleter the whole FASTSim data +/// - subpath: Subpath to a subdirectory within the FASTSim data directory. If +/// an empty string, the function will delete the whole FASTSim data /// directory, clearing all its contents. /// Note: it is not possible to delete single files using this function, only /// directories. If a single file needs deleting, the path_to_cache() function /// can be used to find the FASTSim data directory location. The file can then /// be found and manually deleted. -pub fn clear_cache(subpath: Option) -> anyhow::Result<()> { - let path = match subpath { - Some(subpath) => path_to_cache()?.join(subpath), - None => path_to_cache()?, - }; +pub fn clear_cache>(subpath: P) -> anyhow::Result<()> { + let path = path_to_cache()?.join(subpath); Ok(std::fs::remove_dir_all(path)?) } @@ -811,17 +809,19 @@ mod tests { #[test] fn test_clear_cache() { - // let temp_sub_dir = tempfile::TempDir::new_in(path_to_cache().unwrap()).unwrap(); - // let sub_dir_path = temp_sub_dir.path().to_str().unwrap(); - // url_to_cache("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", "vehicles").unwrap(); - // clear_cache(Some(sub_dir_path.to_string())).unwrap(); - // let still_exists = std::fs::metadata(sub_dir_path).is_ok(); - // assert_eq!(still_exists, false); - // let path_to_vehicle = path_to_cache().unwrap().join("vehicles/https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"); - // let vehicle_still_exists = std::fs::metadata(path_to_vehicle).is_ok(); - // assert_eq!(vehicle_still_exists, true); - // // delete vehicle from subdir (for first attept, use clear_cache to clear entire cache -- but get rid of this once tested) - // std::fs::remove_file(path_to_vehicle).unwrap(); - // // test to see if an empty string will clear entire cache + let temp_sub_dir = tempfile::TempDir::new_in(create_project_subdir("").unwrap()).unwrap(); + let sub_dir_path = temp_sub_dir.path().to_str().unwrap(); + let still_exists_before = std::fs::metadata(sub_dir_path).is_ok(); + assert_eq!(still_exists_before, true); + url_to_cache("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", "").unwrap(); + clear_cache(sub_dir_path).unwrap(); + let still_exists = std::fs::metadata(sub_dir_path).is_ok(); + assert_eq!(still_exists, false); + let path_to_vehicle = path_to_cache() + .unwrap() + .join("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"); + let vehicle_still_exists = std::fs::metadata(&path_to_vehicle).is_ok(); + assert_eq!(vehicle_still_exists, true); + std::fs::remove_file(path_to_vehicle).unwrap(); } } diff --git a/rust/fastsim-core/src/vehicle_utils.rs b/rust/fastsim-core/src/vehicle_utils.rs index 0b3d792b..9b9e471d 100644 --- a/rust/fastsim-core/src/vehicle_utils.rs +++ b/rust/fastsim-core/src/vehicle_utils.rs @@ -2125,6 +2125,9 @@ mod vehicle_utils_tests { .is_empty()); } + // NOTE: this test does not seem to reliably pass. Sometimes the function + // will give a 403 error and sometimes it will succeed -- I don't think + // there's any way to ensure the function succeeds 100% of the time. #[test] fn test_fetch_github_list() { let list = fetch_github_list(Some( From 512284abf0ae33939a2efd5d9b83a3b3b7442a39 Mon Sep 17 00:00:00 2001 From: Robin Steuteville Date: Tue, 23 Jan 2024 16:01:03 -0700 Subject: [PATCH 22/25] adding 403 error to retry errors for fetch_github_list --- rust/fastsim-core/src/vehicle_utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/fastsim-core/src/vehicle_utils.rs b/rust/fastsim-core/src/vehicle_utils.rs index 9b9e471d..3c0bf498 100644 --- a/rust/fastsim-core/src/vehicle_utils.rs +++ b/rust/fastsim-core/src/vehicle_utils.rs @@ -1809,7 +1809,7 @@ const VEHICLE_REPO_LIST_URL: &'static str = fn get_response>(url: S) -> Result { for _ in 1..4 { match ureq::get(url.as_ref()).call() { - Err(Status(503, r)) | Err(Status(429, r)) => { + Err(Status(503, r)) | Err(Status(429, r)) | Err(Status(403, r)) => { let retry: Option = r.header("retry-after").and_then(|h| h.parse().ok()); let retry = retry.unwrap_or(5); eprintln!("{} for {}, retry in {}", r.status(), r.get_url(), retry); From 7960663c3eea34f2493d8ca785d2af3db4e4e593 Mon Sep 17 00:00:00 2001 From: Chad Baker Date: Tue, 6 Feb 2024 12:40:45 -0700 Subject: [PATCH 23/25] deleted deprecated comment --- rust/fastsim-core/src/cycle.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/rust/fastsim-core/src/cycle.rs b/rust/fastsim-core/src/cycle.rs index 0c9ed303..95c4e2ed 100644 --- a/rust/fastsim-core/src/cycle.rs +++ b/rust/fastsim-core/src/cycle.rs @@ -631,7 +631,6 @@ pub struct RustCycle { impl SerdeAPI for RustCycle { const ACCEPTED_BYTE_FORMATS: &'static [&'static str] = &["yaml", "json", "bin", "csv"]; const ACCEPTED_STR_FORMATS: &'static [&'static str] = &["yaml", "json", "csv"]; - // is this enough, or do I have to copy paste in the whole to_cache mathod? const CACHE_FOLDER: &'static str = &"cycles"; fn init(&mut self) -> anyhow::Result<()> { From 4fb10f3a7f72f56f643a17851d1c0e7fccf0956c Mon Sep 17 00:00:00 2001 From: Chad Baker Date: Tue, 6 Feb 2024 12:43:26 -0700 Subject: [PATCH 24/25] minor tweaks --- rust/fastsim-core/src/traits.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/rust/fastsim-core/src/traits.rs b/rust/fastsim-core/src/traits.rs index 173b6230..7510e813 100644 --- a/rust/fastsim-core/src/traits.rs +++ b/rust/fastsim-core/src/traits.rs @@ -188,8 +188,7 @@ pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { Ok(bincode_de) } - /// instantiates an object from a url - /// accepts yaml and json file types + /// Instantiates an object from a url. Accepts yaml and json file types /// # Arguments /// - url: URL (either as a string or url type) to object /// Note: The URL needs to be a URL pointing directly to a file, for example @@ -206,8 +205,8 @@ pub trait SerdeAPI: Serialize + for<'a> Deserialize<'a> { Self::from_reader(response, format) } - /// takes an instantiated Rust object and saves it in the FASTSim data directory in - /// a rust_objects folder + /// Takes an instantiated Rust object and saves it in the FASTSim data directory in + /// a rust_objects folder. /// WARNING: If there is a file already in the data subdirectory with the /// same name, it will be replaced by the new file. /// # Arguments From 867f72f20faed1f37a6602d6592f15be912c7d75 Mon Sep 17 00:00:00 2001 From: Kyle Carow Date: Tue, 6 Feb 2024 13:06:42 -0700 Subject: [PATCH 25/25] reorganize reference vehicle for github caching --- rust/fastsim-core/src/traits.rs | 38 --------- rust/fastsim-core/src/utils.rs | 14 ---- rust/fastsim-core/src/vehicle.rs | 61 -------------- .../1110_2022_Tesla_Model_Y_RWD_opt45017.yaml | 0 rust/fastsim-core/tests/integration-tests.rs | 81 +++++++++++++++++++ 5 files changed, 81 insertions(+), 113 deletions(-) rename rust/fastsim-core/{resources => tests/assets}/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml (100%) create mode 100644 rust/fastsim-core/tests/integration-tests.rs diff --git a/rust/fastsim-core/src/traits.rs b/rust/fastsim-core/src/traits.rs index 7510e813..5924a8de 100644 --- a/rust/fastsim-core/src/traits.rs +++ b/rust/fastsim-core/src/traits.rs @@ -353,41 +353,3 @@ where .all(|(key, value)| other.get(key).map_or(false, |v| value.approx_eq(v, tol))); } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_from_cache() { - let test_path = "1110_2022_Tesla_Model_Y_RWD_opt45017_from_cache.yaml"; - let comparison_vehicle = - crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml") - .unwrap(); - crate::vehicle::RustVehicle::to_cache(&comparison_vehicle, test_path).unwrap(); - let vehicle = crate::vehicle::RustVehicle::from_cache(test_path).unwrap(); - assert_eq!(comparison_vehicle, vehicle); - let full_file_path = Path::new("vehicles").join(test_path); - let path_including_directory = path_to_cache().unwrap().join(full_file_path); - std::fs::remove_file(path_including_directory).unwrap(); - } - - #[test] - fn test_to_cache() { - let comparison_vehicle = - crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml") - .unwrap(); - crate::vehicle::RustVehicle::to_cache( - &comparison_vehicle, - "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", - ) - .unwrap(); - let data_subdirectory = create_project_subdir("vehicles").unwrap(); - let file_path = data_subdirectory.join("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"); - println!("{}", file_path.to_string_lossy()); - println!("{}", crate::vehicle::RustVehicle::CACHE_FOLDER); - let vehicle_b = crate::vehicle::RustVehicle::from_file(&file_path).unwrap(); - assert_eq!(comparison_vehicle, vehicle_b); - std::fs::remove_file(file_path).unwrap(); - } -} diff --git a/rust/fastsim-core/src/utils.rs b/rust/fastsim-core/src/utils.rs index 707399b3..0b35d63b 100644 --- a/rust/fastsim-core/src/utils.rs +++ b/rust/fastsim-core/src/utils.rs @@ -787,20 +787,6 @@ mod tests { assert_eq!(expected_y_lookup, y_lookup); } - #[test] - fn test_url_to_cache() { - url_to_cache("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", "vehicles").unwrap(); - let data_subdirectory = create_project_subdir("vehicles").unwrap(); - let file_path = data_subdirectory.join("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"); - println!("{}", file_path.to_string_lossy()); - let vehicle = crate::vehicle::RustVehicle::from_file(&file_path).unwrap(); - let comparison_vehicle = - crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml") - .unwrap(); - assert_eq!(vehicle, comparison_vehicle); - std::fs::remove_file(file_path).unwrap(); - } - #[test] fn test_path_to_cache() { let path = path_to_cache().unwrap(); diff --git a/rust/fastsim-core/src/vehicle.rs b/rust/fastsim-core/src/vehicle.rs index ca9697d9..fd65e436 100644 --- a/rust/fastsim-core/src/vehicle.rs +++ b/rust/fastsim-core/src/vehicle.rs @@ -1425,65 +1425,4 @@ mod tests { .all(|key| bad_fields.contains(key))); assert!(validation_errs_hashmap.len() == bad_fields.len()); } - - #[test] - fn test_from_github_or_url() { - let mut comparison_vehicle = - crate::vehicle::RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml") - .unwrap(); - comparison_vehicle.doc = Some("Vehicle from https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml".to_owned()); - // test no url provided - let vehicle = RustVehicle::from_github_or_url( - "public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", - None, - ) - .unwrap(); - assert_eq!(vehicle, comparison_vehicle); - // test url provided - let vehicle_1 = RustVehicle::from_github_or_url( - "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", - Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/"), - ) - .unwrap(); - assert_eq!(vehicle_1, comparison_vehicle); - let vehicle_2 = RustVehicle::from_github_or_url( - "/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", - Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public"), - ) - .unwrap(); - assert_eq!(vehicle_2, comparison_vehicle); - let vehicle_3 = RustVehicle::from_github_or_url( - "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", - Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public"), - ) - .unwrap(); - assert_eq!(vehicle_3, comparison_vehicle); - let vehicle_4 = RustVehicle::from_github_or_url( - "public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", - Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/"), - ) - .unwrap(); - assert_eq!(vehicle_4, comparison_vehicle); - let vehicle_5 = RustVehicle::from_github_or_url( - "/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", - Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main"), - ) - .unwrap(); - assert_eq!(vehicle_5, comparison_vehicle); - let vehicle_6 = RustVehicle::from_github_or_url( - "public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", - Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main"), - ) - .unwrap(); - assert_eq!(vehicle_6, comparison_vehicle); - } - - #[test] - fn test_from_url() { - let vehicle = RustVehicle::from_url("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); - let mut comparison_vehicle = - RustVehicle::from_resource("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); - comparison_vehicle.doc = Some("Vehicle from https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml".to_owned()); - assert_eq!(vehicle, comparison_vehicle); - } } diff --git a/rust/fastsim-core/resources/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml b/rust/fastsim-core/tests/assets/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml similarity index 100% rename from rust/fastsim-core/resources/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml rename to rust/fastsim-core/tests/assets/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml diff --git a/rust/fastsim-core/tests/integration-tests.rs b/rust/fastsim-core/tests/integration-tests.rs new file mode 100644 index 00000000..9834230f --- /dev/null +++ b/rust/fastsim-core/tests/integration-tests.rs @@ -0,0 +1,81 @@ +use std::path::Path; + +use fastsim_core::traits::*; +use fastsim_core::*; + +const REFERENCE_VEHICLE: &str = include_str!("assets/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"); + +#[test] +fn test_from_cache() { + let test_path = "1110_2022_Tesla_Model_Y_RWD_opt45017_from_cache.yaml"; + let comparison_vehicle = crate::vehicle::RustVehicle::from_yaml(REFERENCE_VEHICLE).unwrap(); + crate::vehicle::RustVehicle::to_cache(&comparison_vehicle, test_path).unwrap(); + let vehicle = crate::vehicle::RustVehicle::from_cache(test_path).unwrap(); + assert_eq!(comparison_vehicle, vehicle); + let full_file_path = Path::new("vehicles").join(test_path); + let path_including_directory = utils::path_to_cache().unwrap().join(full_file_path); + std::fs::remove_file(path_including_directory).unwrap(); +} + +#[test] +fn test_to_cache() { + let comparison_vehicle = crate::vehicle::RustVehicle::from_yaml(REFERENCE_VEHICLE).unwrap(); + crate::vehicle::RustVehicle::to_cache( + &comparison_vehicle, + "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", + ) + .unwrap(); + let data_subdirectory = utils::create_project_subdir("vehicles").unwrap(); + let file_path = data_subdirectory.join("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"); + println!("{}", file_path.to_string_lossy()); + println!("{}", crate::vehicle::RustVehicle::CACHE_FOLDER); + let vehicle_b = crate::vehicle::RustVehicle::from_file(&file_path).unwrap(); + assert_eq!(comparison_vehicle, vehicle_b); + std::fs::remove_file(file_path).unwrap(); +} + +#[test] +fn test_url_to_cache() { + utils::url_to_cache("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", "vehicles").unwrap(); + let data_subdirectory = utils::create_project_subdir("vehicles").unwrap(); + let file_path = data_subdirectory.join("1110_2022_Tesla_Model_Y_RWD_opt45017.yaml"); + println!("{}", file_path.to_string_lossy()); + let vehicle = crate::vehicle::RustVehicle::from_file(&file_path).unwrap(); + let comparison_vehicle = crate::vehicle::RustVehicle::from_yaml(REFERENCE_VEHICLE).unwrap(); + assert_eq!(vehicle, comparison_vehicle); + std::fs::remove_file(file_path).unwrap(); +} + +#[test] +fn test_from_github_or_url() { + let mut comparison_vehicle = vehicle::RustVehicle::from_yaml(REFERENCE_VEHICLE).unwrap(); + comparison_vehicle.doc = Some("Vehicle from https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml".to_owned()); + // test no url provided + let vehicle = vehicle::RustVehicle::from_github_or_url( + "public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", + None, + ) + .unwrap(); + assert_eq!(vehicle, comparison_vehicle); + // test url provided + let vehicle_1 = vehicle::RustVehicle::from_github_or_url( + "1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", + Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/"), + ) + .unwrap(); + assert_eq!(vehicle_1, comparison_vehicle); + let vehicle_2 = vehicle::RustVehicle::from_github_or_url( + "public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml", + Some("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/"), + ) + .unwrap(); + assert_eq!(vehicle_2, comparison_vehicle); +} + +#[test] +fn test_from_url() { + let vehicle = vehicle::RustVehicle::from_url("https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml").unwrap(); + let mut comparison_vehicle = vehicle::RustVehicle::from_yaml(REFERENCE_VEHICLE).unwrap(); + comparison_vehicle.doc = Some("Vehicle from https://raw.githubusercontent.com/NREL/fastsim-vehicles/main/public/1110_2022_Tesla_Model_Y_RWD_opt45017.yaml".to_owned()); + assert_eq!(vehicle, comparison_vehicle); +}