Skip to content

Commit

Permalink
Merge pull request #76 from dandi/gh-13
Browse files Browse the repository at this point in the history 
Add links to version & asset metadata to web view
  • Loading branch information
@jwodder
jwodder authored Feb 15, 2024
2 parents 4998bd8 + 79f8578 commit bfc7ffe
Show file tree
Hide file tree
Showing 8 changed files with 207 additions and 92 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ In Development
- Log errors that cause 404 and 500 responses
- Added breadcrumbs to HTML views of collections
- `FAST_NOT_EXIST` components are now checked for case-insensitively
- Add links to version & asset metadata to the web view

v0.2.0 (2024-02-07)
-------------------
Expand Down
88 changes: 57 additions & 31 deletions src/dandi/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,12 +111,22 @@ impl DandiClient {
pub(crate) fn get_all_dandisets(
&self,
) -> impl Stream<Item = Result<Dandiset, DandiError>> + '_ {
self.paginate(self.get_url(["dandisets"]))
self.paginate::<RawDandiset>(self.get_url(["dandisets"]))
.map_ok(|ds| ds.with_metadata_urls(self))
}

pub(crate) fn dandiset(&self, dandiset_id: DandisetId) -> DandisetEndpoint<'_> {
DandisetEndpoint::new(self, dandiset_id)
}

fn version_metadata_url(&self, dandiset_id: &DandisetId, version_id: &VersionId) -> Url {
self.get_url([
"dandisets",
dandiset_id.as_ref(),
"versions",
version_id.as_ref(),
])
}
}

#[derive(Clone, Debug)]
Expand All @@ -139,21 +149,29 @@ impl<'a> DandisetEndpoint<'a> {

pub(crate) async fn get(&self) -> Result<Dandiset, DandiError> {
self.client
.get(
.get::<RawDandiset>(
self.client
.get_url(["dandisets", self.dandiset_id.as_ref()]),
)
.await
.map(|ds| ds.with_metadata_urls(self.client))
}

pub(crate) fn get_all_versions(
&self,
) -> impl Stream<Item = Result<DandisetVersion, DandiError>> + '_ {
self.client.paginate(self.client.get_url([
"dandisets",
self.dandiset_id.as_ref(),
"versions",
]))
self.client
.paginate::<RawDandisetVersion>(self.client.get_url([
"dandisets",
self.dandiset_id.as_ref(),
"versions",
]))
.map_ok(|v| {
let url = self
.client
.version_metadata_url(&self.dandiset_id, &v.version);
v.with_metadata_url(url)
})
}
}

Expand All @@ -175,32 +193,45 @@ impl<'a> VersionEndpoint<'a> {

pub(crate) async fn get(&self) -> Result<DandisetVersion, DandiError> {
self.client
.get(self.client.get_url([
.get::<RawDandisetVersion>(self.client.get_url([
"dandisets",
self.dandiset_id.as_ref(),
"versions",
self.version_id.as_ref(),
"info",
]))
.await
.map(|v| v.with_metadata_url(self.metadata_url()))
}

fn metadata_url(&self) -> Url {
self.client
.version_metadata_url(&self.dandiset_id, &self.version_id)
}

fn asset_metadata_url(&self, asset_id: &str) -> Url {
self.client.get_url([
"dandisets",
self.dandiset_id.as_ref(),
"versions",
self.version_id.as_ref(),
"assets",
asset_id,
])
}

pub(crate) async fn get_metadata(&self) -> Result<VersionMetadata, DandiError> {
let data = self
.client
.get::<serde_json::Value>(self.client.get_url([
"dandisets",
self.dandiset_id.as_ref(),
"versions",
self.version_id.as_ref(),
]))
.get::<serde_json::Value>(self.metadata_url())
.await?;
Ok(VersionMetadata(dump_json_as_yaml(data).into_bytes()))
}

async fn get_asset_by_id(&self, id: &str) -> Result<Asset, DandiError> {
self.client
.get(self.client.get_url([
let raw_asset = self
.client
.get::<RawAsset>(self.client.get_url([
"dandisets",
self.dandiset_id.as_ref(),
"versions",
Expand All @@ -209,7 +240,8 @@ impl<'a> VersionEndpoint<'a> {
id,
"info",
]))
.await
.await?;
raw_asset.try_into_asset(self).map_err(Into::into)
}

pub(crate) fn get_root_children(
Expand Down Expand Up @@ -272,14 +304,14 @@ impl<'a> VersionEndpoint<'a> {
.append_pair("metadata", "1")
.append_pair("order", "path");
let dirpath = path.to_dir_path();
let stream = self.client.paginate::<Asset>(url.clone());
let stream = self.client.paginate::<RawAsset>(url.clone());
tokio::pin!(stream);
while let Some(asset) = stream.try_next().await? {
if asset.path() == path {
return Ok(AtAssetPath::Asset(asset));
} else if asset.path().is_strictly_under(&dirpath) {
if &asset.path == path {
return Ok(AtAssetPath::Asset(asset.try_into_asset(self)?));
} else if asset.path.is_strictly_under(&dirpath) {
return Ok(AtAssetPath::Folder(AssetFolder { path: dirpath }));
} else if asset.path().as_ref() > dirpath.as_ref() {
} else if asset.path.as_ref() > dirpath.as_ref() {
break;
}
}
Expand Down Expand Up @@ -308,18 +340,10 @@ impl<'a> VersionEndpoint<'a> {
match self.get_path(&zarr_path).await? {
AtAssetPath::Folder(_) => continue,
AtAssetPath::Asset(Asset::Blob(_)) => {
let mut url = self.client.get_url([
"dandisets",
self.dandiset_id.as_ref(),
"versions",
self.version_id.as_ref(),
"assets",
]);
url.query_pairs_mut().append_pair("path", path.as_ref());
return Err(DandiError::PathUnderBlob {
path: path.clone(),
blob_path: zarr_path,
});
})
}
AtAssetPath::Asset(Asset::Zarr(zarr)) => {
let s3 = self.client.get_s3client_for_zarr(&zarr).await?;
Expand Down Expand Up @@ -414,6 +438,8 @@ pub(crate) enum DandiError {
source: ZarrToS3Error,
},
#[error(transparent)]
AssetType(#[from] AssetTypeError),
#[error(transparent)]
S3(#[from] S3Error),
}

Expand Down
126 changes: 86 additions & 40 deletions src/dandi/types.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,29 +13,78 @@ pub(super) struct Page<T> {
}

#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
pub(crate) struct Dandiset {
pub(crate) identifier: DandisetId,
pub(crate) struct RawDandiset {
identifier: DandisetId,
#[serde(with = "time::serde::rfc3339")]
pub(crate) created: OffsetDateTime,
created: OffsetDateTime,
#[serde(with = "time::serde::rfc3339")]
pub(crate) modified: OffsetDateTime,
modified: OffsetDateTime,
//contact_person: String,
//embargo_status: ...,
draft_version: RawDandisetVersion,
most_recent_published_version: Option<RawDandisetVersion>,
}

impl RawDandiset {
pub(super) fn with_metadata_urls(self, client: &super::DandiClient) -> Dandiset {
let draft_version = self
.draft_version
.with_metadata_url(client.version_metadata_url(&self.identifier, &VersionId::Draft));
let most_recent_published_version = self.most_recent_published_version.map(|v| {
let url = client.version_metadata_url(&self.identifier, &v.version);
v.with_metadata_url(url)
});
Dandiset {
identifier: self.identifier,
created: self.created,
modified: self.modified,
draft_version,
most_recent_published_version,
}
}
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) struct Dandiset {
pub(crate) identifier: DandisetId,
pub(crate) created: OffsetDateTime,
pub(crate) modified: OffsetDateTime,
pub(crate) draft_version: DandisetVersion,
pub(crate) most_recent_published_version: Option<DandisetVersion>,
}

#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
pub(crate) struct DandisetVersion {
pub(crate) version: VersionId,
pub(crate) struct RawDandisetVersion {
pub(super) version: VersionId,
//name: String,
//asset_count: u64,
pub(crate) size: i64,
size: i64,
//status: ...,
#[serde(with = "time::serde::rfc3339")]
pub(crate) created: OffsetDateTime,
created: OffsetDateTime,
#[serde(with = "time::serde::rfc3339")]
modified: OffsetDateTime,
}

impl RawDandisetVersion {
pub(super) fn with_metadata_url(self, metadata_url: Url) -> DandisetVersion {
DandisetVersion {
version: self.version,
size: self.size,
created: self.created,
modified: self.modified,
metadata_url,
}
}
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) struct DandisetVersion {
pub(crate) version: VersionId,
pub(crate) size: i64,
pub(crate) created: OffsetDateTime,
pub(crate) modified: OffsetDateTime,
pub(crate) metadata_url: Url,
}

#[derive(Clone, Debug, Eq, PartialEq)]
Expand Down Expand Up @@ -95,28 +144,19 @@ struct RawFolderEntryAsset {
asset_id: String,
}

#[allow(clippy::large_enum_variant)]
#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) enum AtAssetPath {
Folder(AssetFolder),
Asset(Asset),
}

#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
#[serde(try_from = "RawAsset")]
#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) enum Asset {
Blob(BlobAsset),
Zarr(ZarrAsset),
}

impl Asset {
pub(crate) fn path(&self) -> &PurePath {
match self {
Asset::Blob(a) => &a.path,
Asset::Zarr(a) => &a.path,
}
}
}

#[derive(Clone, Debug, Eq, PartialEq)]
pub(crate) struct BlobAsset {
pub(crate) asset_id: String,
Expand All @@ -126,6 +166,7 @@ pub(crate) struct BlobAsset {
pub(crate) created: OffsetDateTime,
pub(crate) modified: OffsetDateTime,
pub(crate) metadata: AssetMetadata,
pub(crate) metadata_url: Url,
}

impl BlobAsset {
Expand Down Expand Up @@ -160,6 +201,7 @@ pub(crate) struct ZarrAsset {
pub(crate) created: OffsetDateTime,
pub(crate) modified: OffsetDateTime,
pub(crate) metadata: AssetMetadata,
pub(crate) metadata_url: Url,
}

impl ZarrAsset {
Expand Down Expand Up @@ -211,11 +253,11 @@ pub(crate) struct AssetDigests {
}

#[derive(Clone, Debug, Deserialize, Eq, PartialEq)]
struct RawAsset {
pub(super) struct RawAsset {
asset_id: String,
blob: Option<String>,
zarr: Option<String>,
path: PurePath,
pub(super) path: PurePath,
size: i64,
#[serde(with = "time::serde::rfc3339")]
created: OffsetDateTime,
Expand All @@ -224,34 +266,38 @@ struct RawAsset {
metadata: AssetMetadata,
}

impl TryFrom<RawAsset> for Asset {
type Error = AssetTypeError;

fn try_from(value: RawAsset) -> Result<Asset, AssetTypeError> {
match (value.blob, value.zarr) {
impl RawAsset {
pub(super) fn try_into_asset(
self,
endpoint: &super::VersionEndpoint<'_>,
) -> Result<Asset, AssetTypeError> {
let metadata_url = endpoint.asset_metadata_url(&self.asset_id);
match (self.blob, self.zarr) {
(Some(blob_id), None) => Ok(Asset::Blob(BlobAsset {
asset_id: value.asset_id,
asset_id: self.asset_id,
blob_id,
path: value.path,
size: value.size,
created: value.created,
modified: value.modified,
metadata: value.metadata,
path: self.path,
size: self.size,
created: self.created,
modified: self.modified,
metadata: self.metadata,
metadata_url,
})),
(None, Some(zarr_id)) => Ok(Asset::Zarr(ZarrAsset {
asset_id: value.asset_id,
asset_id: self.asset_id,
zarr_id,
path: value.path,
size: value.size,
created: value.created,
modified: value.modified,
metadata: value.metadata,
path: self.path,
size: self.size,
created: self.created,
modified: self.modified,
metadata: self.metadata,
metadata_url,
})),
(None, None) => Err(AssetTypeError::Neither {
asset_id: value.asset_id,
asset_id: self.asset_id,
}),
(Some(_), Some(_)) => Err(AssetTypeError::Both {
asset_id: value.asset_id,
asset_id: self.asset_id,
}),
}
}
Expand Down
Loading

0 comments on commit bfc7ffe

Please sign in to comment.