-
Notifications
You must be signed in to change notification settings - Fork 17
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
15 changed files
with
6,615 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,108 @@ | ||
import os | ||
from typing import List, Dict, Any, Optional | ||
from jinja2 import Environment, FileSystemLoader | ||
from dandi.dandiapi import DandiAPIClient | ||
|
||
|
||
def get_dandiset_metadata(dandiset_id: str) -> Optional[Dict[str, Any]]: | ||
""" | ||
Fetch metadata for a given dandiset ID. | ||
Parameters | ||
---------- | ||
dandiset_id : str | ||
The ID of the dandiset to fetch metadata for. | ||
Returns | ||
------- | ||
Optional[Dict[str, Any]] | ||
A dictionary containing the dandiset metadata if successful, None otherwise. | ||
""" | ||
with DandiAPIClient() as client: | ||
try: | ||
dandiset = client.get_dandiset(dandiset_id) | ||
metadata = dandiset.get_raw_metadata() | ||
return metadata | ||
except Exception as e: | ||
print(f"Error fetching metadata for dandiset {dandiset_id}: {str(e)}") | ||
return None | ||
|
||
|
||
def find_notebooks(folder: str) -> List[str]: | ||
""" | ||
Recursively find all Jupyter notebooks in a given folder. | ||
Parameters | ||
---------- | ||
folder : str | ||
The path to the folder to search in. | ||
Returns | ||
------- | ||
List[str] | ||
A list of relative paths to the found notebooks. | ||
""" | ||
notebooks = [] | ||
for root, _, files in os.walk(folder): | ||
for file in files: | ||
if file.endswith('.ipynb'): | ||
rel_path = os.path.relpath(os.path.join(root, file), folder) | ||
notebooks.append(rel_path) | ||
return notebooks | ||
|
||
|
||
def collect_metadata() -> List[Dict[str, Any]]: | ||
""" | ||
Collect metadata and notebook information for all dandisets in the current directory. | ||
Returns | ||
------- | ||
List[Dict[str, Any]] | ||
A list of dictionaries, each containing information about a dandiset, | ||
sorted by dandiset ID. | ||
""" | ||
dandisets = [] | ||
for folder in os.listdir('.'): | ||
if os.path.isdir(folder) and folder.isdigit(): | ||
metadata = get_dandiset_metadata(folder) | ||
if metadata: | ||
notebooks = find_notebooks(folder) | ||
dandisets.append({ | ||
'id': folder, | ||
'metadata': metadata, | ||
'notebooks': notebooks | ||
}) | ||
|
||
dandisets.sort(key=lambda x: x['id']) | ||
return dandisets | ||
|
||
|
||
def render_webpage(dandisets: List[Dict[str, Any]]) -> None: | ||
""" | ||
Render the webpage using the collected dandiset information. | ||
Parameters | ||
---------- | ||
dandisets : List[Dict[str, Any]] | ||
A list of dictionaries containing information about each dandiset. | ||
Returns | ||
------- | ||
None | ||
""" | ||
current_dir = os.path.dirname(os.path.abspath(__file__)) | ||
template_dir = os.path.join(current_dir, '..', 'templates') | ||
|
||
env = Environment(loader=FileSystemLoader(template_dir)) | ||
template = env.get_template('index.html') | ||
|
||
output = template.render(dandisets=dandisets) | ||
|
||
output_dir = os.path.join(current_dir, '..', '..', 'output') | ||
os.makedirs(output_dir, exist_ok=True) | ||
with open(os.path.join(output_dir, 'index.html'), 'w') as f: | ||
f.write(output) | ||
|
||
if __name__ == "__main__": | ||
dandisets = collect_metadata() | ||
render_webpage(dandisets) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,103 @@ | ||
<!DOCTYPE html> | ||
<html lang="en"> | ||
<head> | ||
<meta charset="UTF-8"> | ||
<meta name="viewport" content="width=device-width, initial-scale=1.0"> | ||
<title>DANDI Datasets</title> | ||
<style> | ||
body { | ||
font-family: 'Arial', sans-serif; | ||
line-height: 1.6; | ||
margin: 0; | ||
padding: 20px; | ||
} | ||
.container { | ||
max-width: 1200px; | ||
margin: 0 auto; | ||
} | ||
h1 { | ||
text-align: center; | ||
margin-bottom: 30px; | ||
} | ||
.dandiset { | ||
border-radius: 8px; | ||
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); | ||
margin-bottom: 30px; | ||
padding: 20px; | ||
transition: box-shadow 0.3s ease; | ||
} | ||
.dandiset:hover { | ||
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.15); | ||
} | ||
.dandiset h2 { | ||
margin-top: 0; | ||
margin-bottom: 15px; | ||
} | ||
.dandiset p { | ||
margin-bottom: 10px; | ||
} | ||
.dandiset a { | ||
text-decoration: none; | ||
font-weight: bold; | ||
} | ||
.dandiset a:hover { | ||
text-decoration: underline; | ||
} | ||
.notebooks { | ||
margin-top: 15px; | ||
} | ||
.notebooks h3 { | ||
font-size: 1.1em; | ||
margin-bottom: 10px; | ||
} | ||
.notebooks ul { | ||
list-style-type: disc; | ||
padding-left: 20px; | ||
} | ||
.notebooks li { | ||
margin-bottom: 5px; | ||
} | ||
@media (max-width: 768px) { | ||
body { | ||
padding: 10px; | ||
} | ||
.dandiset { | ||
padding: 15px; | ||
} | ||
} | ||
</style> | ||
</head> | ||
<body> | ||
<div class="container"> | ||
<h1>DANDI Datasets</h1> | ||
|
||
<div class="toc"> | ||
<h2>Table of Contents</h2> | ||
<ul> | ||
{% for dandiset in dandisets %} | ||
<li><a href="#dandiset-{{ dandiset.id }}">{{ dandiset.id }} - {{ dandiset.metadata.name }}</a></li> | ||
{% endfor %} | ||
</ul> | ||
</div> | ||
|
||
{% for dandiset in dandisets %} | ||
<div id="dandiset-{{ dandiset.id }}" class="dandiset"> | ||
<h2>{{ dandiset.metadata.name }}</h2> | ||
<p><strong>ID:</strong> {{ dandiset.id }}</p> | ||
<p><strong>Description:</strong> {{ dandiset.metadata.description }}</p> | ||
<p><a href="https://dandiarchive.org/dandiset/{{ dandiset.id }}" target="_blank">View on DANDI Archive</a></p> | ||
{% if dandiset.notebooks %} | ||
<div class="notebooks"> | ||
<h3>Notebooks:</h3> | ||
<ul> | ||
{% for notebook in dandiset.notebooks %} | ||
<li><a href="https://github.com/dandi/example-notebooks/blob/master/{{ dandiset.id }}/{{ notebook }}" target="_blank">{{ notebook }}</a></li> | ||
{% endfor %} | ||
</ul> | ||
</div> | ||
{% endif %} | ||
</div> | ||
{% endfor %} | ||
</div> | ||
</body> | ||
</html> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
name: DANDI Metadata Collector | ||
|
||
on: | ||
push: | ||
branches: [ master ] | ||
|
||
jobs: | ||
collect-and-render: | ||
runs-on: ubuntu-latest | ||
steps: | ||
- uses: actions/checkout@v2 | ||
|
||
- name: Set up Python | ||
uses: actions/setup-python@v2 | ||
with: | ||
python-version: '3.9' | ||
|
||
- name: Install dependencies | ||
run: | | ||
python -m pip install --upgrade pip | ||
pip install requests jinja2 dandi | ||
- name: Run metadata collector and renderer | ||
run: python .github/scripts/collect_and_render.py | ||
|
||
- name: Deploy to GitHub Pages | ||
uses: peaceiris/actions-gh-pages@v3 | ||
with: | ||
github_token: ${{ secrets.GITHUB_TOKEN }} | ||
publish_dir: ./output |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,6 @@ | ||
.ipynb_checkpoints | ||
*.DS_Store | ||
*__pycache__/ | ||
*.ipynb_checkpoints/ | ||
output/ | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
Oops, something went wrong.