diff --git a/example/auth.rs b/example/auth.rs new file mode 100644 index 0000000..ade3eea --- /dev/null +++ b/example/auth.rs @@ -0,0 +1,16 @@ +use tokio; + +#[tokio::main] +async fn main() { + let _build = libhfd::api::tokio::ApiBuilder::new(); + + let api = _build.with_endpoint("https://hf-mirror.com") + .with_token("xxxxxxxxxxx") + .build() + .unwrap(); + + let _filename = api + .dataset("OpenVideo/pexels-raw".to_string()) + .get(format!("data/{:06}.parquet", 0).as_ref()) + .await; +} \ No newline at end of file diff --git a/example/dataset.rs b/example/dataset.rs new file mode 100644 index 0000000..a916844 --- /dev/null +++ b/example/dataset.rs @@ -0,0 +1,31 @@ +use tokio; +use tokio_stream::StreamExt; + +#[tokio::main] +async fn main() { + let nums: Vec> = (0..204) + .collect::>() + .chunks(10) + .map(|chunk| chunk.to_vec()) + .collect(); + + for chunk in nums{ + let mut tasks: Vec<_> = chunk.into_iter() + .map(|i| tokio::spawn(async move { + let _build = libhfd::api::tokio::ApiBuilder::new(); + let api = _build.with_endpoint("https://hf-mirror.com") + .with_token("hf_xxxxxxxx") + .build() + .unwrap(); + let _filename = api + .dataset("OpenVideo/pexels-raw".to_string()) + .get(format!("data/{:06}.parquet", i).as_ref()) + .await; + })) + .collect(); + + for task in tasks { + task.await.unwrap(); + } + } +} \ No newline at end of file diff --git a/example/mirror.rs b/example/mirror.rs new file mode 100644 index 0000000..83fb2e4 --- /dev/null +++ b/example/mirror.rs @@ -0,0 +1,15 @@ +use tokio; + +#[tokio::main] +async fn main() { + let _build = libhfd::api::tokio::ApiBuilder::new(); + let api = _build.with_endpoint("https://hf-mirror.com") + .build() + .unwrap(); + + let _filename = api + .model("ByteDance/Hyper-SD".to_string()) + .get("Hyper-SDXL-8steps-lora.safetensors") + .await + .unwrap(); +} \ No newline at end of file diff --git a/hfd-cli/Cargo.toml b/hfd-cli/Cargo.toml index d34b732..5fa8c69 100644 --- a/hfd-cli/Cargo.toml +++ b/hfd-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hfd-cli" -version = "0.1.5" +version = "0.1.6" edition = "2021" [dependencies] diff --git a/hfd-cli/src/cli.rs b/hfd-cli/src/cli.rs index e854245..e60b6b2 100644 --- a/hfd-cli/src/cli.rs +++ b/hfd-cli/src/cli.rs @@ -1,12 +1,12 @@ use tokio; -use tokio_stream::StreamExt; +// use tokio_stream::StreamExt; #[tokio::main] async fn main() { let api = libhfd::api::tokio::Api::new().unwrap(); let _filename = api - .dataset("ByteDance/Hyper-SD".to_string()) + .model("ByteDance/Hyper-SD".to_string()) .get("Hyper-SDXL-8steps-lora.safetensors") .await .unwrap(); diff --git a/hfd/src/api/tokio.rs b/hfd/src/api/tokio.rs index 714641c..8b7e21d 100644 --- a/hfd/src/api/tokio.rs +++ b/hfd/src/api/tokio.rs @@ -86,7 +86,7 @@ impl ApiBuilder { let progress = true; Self { - endpoint: "https://hf-mirror.com".to_string(), + endpoint: "https://huggingface.co".to_string(), url_template: "{endpoint}/{repo_id}/resolve/{revision}/{filename}".to_string(), cache, token, @@ -98,6 +98,11 @@ impl ApiBuilder { } } + pub fn with_endpoint(mut self, endpoint: &str) -> Self { + self.endpoint = endpoint.to_string(); + self + } + pub fn with_progress(mut self, progress: bool) -> Self { self.progress = progress; self @@ -108,8 +113,8 @@ impl ApiBuilder { self } - pub fn with_token(mut self, token: Option) -> Self { - self.token = token; + pub fn with_token(mut self, token: &str) -> Self { + self.token = Some(token.to_string()); self }