diff --git a/.github/workflows/pr-close.yml b/.github/workflows/pr-close.yml new file mode 100644 index 00000000..a0868bf2 --- /dev/null +++ b/.github/workflows/pr-close.yml @@ -0,0 +1,29 @@ +name: delete preview on PR close +on: + pull_request: + types: [closed] + +jobs: + delete_preview: + runs-on: ubuntu-latest + env: + PR_PATH: pull/${{github.event.number}} + steps: + - name: make empty dir + run: mkdir ${{ env.PR_PATH }} + + - name: delete folder + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ${{ env.PR_PATH }} + destination_dir: ${{ env.PR_PATH }} + + - name: Comment on PR + uses: hasura/comment-progress@v2.2.0 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + repository: ${{ github.repository }} + number: ${{ github.event.number }} + id: deploy-preview + message: "🪓 PR closed, deleted preview at https://github.com/${{ github.repository }}/tree/gh-pages/${{ env.PR_PATH }}/" \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..c9426afd --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,54 @@ +name: github pages + +on: + push: + branches: + - master + pull_request: + branches: + - master + schedule: + - cron: '0 0 * * 1' + +permissions: + contents: write + +jobs: + deploy: + runs-on: ubuntu-latest + env: + PR_PATH: pull/${{github.event.number}} + steps: + - name: Comment on PR + uses: hasura/comment-progress@v2.2.0 + if: github.ref != 'refs/heads/master' + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + repository: ${{ github.repository }} + number: ${{ github.event.number }} + id: deploy-preview + message: "Starting deployment of preview ⏳..." + + - name: Checkout website repo + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + # - name: Install dependencies + # run: | + # python -m pip install --upgrade pip + # pip install -r requirements.txt + + - name: Build website + uses: ammaraskar/sphinx-action@master + with: + docs-folder: "." + + - name: Upload HTML + uses: actions/upload-artifact@v1 + with: + name: DocumentationHTML + path: _build/html/ diff --git a/Makefile b/Makefile index 74065bfa..d2f308c4 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,6 @@ help: clean: -rm -rf _build/* - -rm *-stamp gitwash-update: python ../tools/gitwash_dumper.py devel dipy --repo-name=dipy --github-user=dipy \ @@ -71,9 +70,7 @@ qthelp: @echo "To view the help file:" @echo "# assistant -collectionFile _build/qthelp/dipy.qhc" -latex: rstexamples latex-after-examples - -latex-after-examples: +latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) _build/latex @echo @echo "Build finished; the LaTeX files are in _build/latex." @@ -95,22 +92,3 @@ doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) _build/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in _build/doctest/output.txt." - -rstexamples: rstexamples-stamp -rstexamples-stamp: - cd examples_built && ../../tools/make_examples.py - touch $@ - -pdf: pdf-stamp -pdf-stamp: latex - cd _build/latex && make all-pdf - touch $@ - -upload: html - ./upload-gh-pages.sh _build/html/ dipy dipy - -xvfb: - export TEST_WITH_XVFB=true && make html - -memory_profile: - export TEST_WITH_MEMPROF=true && make html diff --git a/README.md b/README.md index 3e7b11f4..268d12c9 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,27 @@ -# Documentation Generation +# DIPY.ORG +## Background + +This site makes use of [Sphinx](https://www.sphinx-doc.org/en/stable/) and was built upon [Bootstrap](https://getbootstrap.com) via the [GRG Sphinx theme](https://github.com/GRG-Projects/grg-sphinx-theme) and [PYData Sphinx Theme](https://pydata-sphinx-theme.readthedocs.io/en/latest/). +We use Github Action to deploy the website and Github Page to host the [website](https://dipy.org.github.io). ## Index -- `_static`: Contains images, css, js for Sphinx to look at -- `_templates`: Contains html layout for custom Sphinx design -- `build`: Contains the generated documentation -- `devel`: Contains `*.rst` files for the Developer's Guide -- `examples`: DIPY application showcases. Add any tutorial here -- `examples_built`: Keep it empty. Only for example generation -- `releases_notes`: Contains all API changes / PRs, issues resolved for a specific release -- `sphinxext`: Sphinx custom plugins -- `theory`: Diffusion theory + FAQ files -- `tools`: Scripts to generate some parts of the documentation, like the API +- `_static`: Contains all assets (images, CSS, JS) for Sphinx to look at. +- `_templates`: Contains html layout for custom Sphinx design. +- `_build`: Contains the generated documentation. +- `sphinxext`: Sphinx custom plugins. +- `theory`: Diffusion theory + FAQ files. +- `tools`: Contains all scripts to generate some parts of the documentation, like the API. +- `posts`: Contains all blog posts. -## Doc generation steps: +## Testing Locally: Doc generation steps: ### Installing requirements +To set up your computer to run this site locally, you need to install the various Python packages in the [requirements.txt](requirements.txt) at the top level of this repository. + ```bash -$ pip install -U -r doc-requirements.txt +$ pip install -U -r requirements.txt ``` ### Generate all the Documentation @@ -35,3 +38,123 @@ $ make -C . clean && make -C . html $ ./make.bat clean $ ./make.bat html ``` + +This will build a collection of html pages under `_build/html` and you can open the `index.html` using your browser of choice. + +## Creating a PR + +When you are happy with any changes you have made to the website. +We recommend building the website and making sure that everything is building fine. +You should see no warnings for the build. + +Once you are sure everything is in order, you can send a PR to this repository. +If you are unfamiliar with this, please see this guide from [GitHub.](https://help.github.com/articles/about-pull-requests/) + +## PR Review + +When a PR is opened, Github Action will create a preview of any content or style changes. + +This must pass before the PR will be merged, furthermore, one review is required before a PR can be merged as well. + +## Creating a Blog Post + +Blog posts can be added by creating a new text file in the `posts/` directory. +The filename must use the following naming convention `YEAR-MONTH-DAY-title.{ext}` and be written in one of the following formats: + +- [RST](https://www.sphinx-doc.org/en/stable/rest.html) formatted text, `ext=rst`, +- [Jupyter notebook](https://jupyter.org/), `ext=ipynb`; (notebooks are converted to RST using the [nbsphinx](https://nbsphinx.readthedocs.io) extension) +- [MD](https://www.markdownguide.org/cheat-sheet/) formatted text, `ext=md`, + +Please also see the [ABlog documentation](https://ablog.readthedocs.io/) for more information. + +### RST + +If you write your post in RST formatted text, each file must also contain the following header for Sphinx via [Ablog](https://github.com/sunpy/ablog) to parse the post properly: + +```rst +.. post:: + :author: + :tags: + :category: + + +========= + +``` + +or + +```rst +:blogpost: true +:date: <Date> +:author: <Name> +:category: <One of the below> + +<Title> +========= + +``` + +### Jupyter Notebook + +When writing posts as Jupyter notebooks, the first cell should be a Markdown cell with the title as a top-level heading (i.e. using a single `#`) and the second cell should be a raw cell containing the following + +```rst +.. post:: <Date> + :author: <Name> + :tags: <Tag list with commas> + :category: <One of the below> + :exclude: + + <Short description of post> +``` + +The short description will appear as a preview of your post on the blog page. +See the [nbsphinx docs](https://nbsphinx.readthedocs.io/raw-cells.html) for information on making raw notebook cells compatible with Sphinx and RST. + +You might have to open the notebook in a text editor and change the "metadata" for the post cell to include the following + +``` + "metadata": { + "raw_mimetype": "text/restructuredtext" + }, +``` + +In theory the alternative rst style and the below markdown style should also work in this cell. + +Additionally, Sphinx will automatically add a link to the interactive version of your notebook, hosted on [Binder](https://mybinder.org/), to the top of your post. +If your notebook requires any other dependencies besides SunPy (or its dependencies), they will need to be added to `binder/requirements.txt`. + +### Markdown + +If you write your post in markdown formatted text, each file must contain the following header for Sphinx via [Ablog](https://github.com/sunpy/ablog) to parse the post properly: + +``` +--- +blogpost: true +date: <Date> +author: <Name> +category: <One of the below> +--- + +# <Title> + +``` + +### Metadata + +Please note that the date for the post is different from the way it is written for the blog filename. +Since this date is reader-facing, we want month day year **e.g.,** May 14 2056. + +The current range of categories we have "officially" are: + +- release +- update +- gsoc +- news +- tutorial +- event + +Please select the more appropriate one, for many `update` or `news` would be enough. + +For tags, you can choose what you prefer for your post but please don't use any that are in the categories list. diff --git a/_static/DM-MNIST-112epoch.png b/_static/images/DM-MNIST-112epoch.png similarity index 100% rename from _static/DM-MNIST-112epoch.png rename to _static/images/DM-MNIST-112epoch.png diff --git a/_static/DM-MNIST-DDIM300-108epoch.png b/_static/images/DM-MNIST-DDIM300-108epoch.png similarity index 100% rename from _static/DM-MNIST-DDIM300-108epoch.png rename to _static/images/DM-MNIST-DDIM300-108epoch.png diff --git a/_static/colorfa.png b/_static/images/colorfa.png similarity index 100% rename from _static/colorfa.png rename to _static/images/colorfa.png diff --git a/_static/dipy-banner.png b/_static/images/dipy-banner.png similarity index 100% rename from _static/dipy-banner.png rename to _static/images/dipy-banner.png diff --git a/_static/dipy-logo.png b/_static/images/dipy-logo.png similarity index 100% rename from _static/dipy-logo.png rename to _static/images/dipy-logo.png diff --git a/_static/dipy-ws-header.png b/_static/images/dipy-ws-header.png similarity index 100% rename from _static/dipy-ws-header.png rename to _static/images/dipy-ws-header.png diff --git a/_static/dipy_paper_logo.jpg b/_static/images/dipy_paper_logo.jpg similarity index 100% rename from _static/dipy_paper_logo.jpg rename to _static/images/dipy_paper_logo.jpg diff --git a/_static/dm3d-monai-B8-DM500.png b/_static/images/dm3d-monai-B8-DM500.png similarity index 100% rename from _static/dm3d-monai-B8-DM500.png rename to _static/images/dm3d-monai-B8-DM500.png diff --git a/_static/dm3d-monai-training-curves.png b/_static/images/dm3d-monai-training-curves.png similarity index 100% rename from _static/dm3d-monai-training-curves.png rename to _static/images/dm3d-monai-training-curves.png diff --git a/_static/dm3d-reconst-D200-D300.png b/_static/images/dm3d-reconst-D200-D300.png similarity index 100% rename from _static/dm3d-reconst-D200-D300.png rename to _static/images/dm3d-reconst-D200-D300.png diff --git a/_static/dm3d-training-curves.png b/_static/images/dm3d-training-curves.png similarity index 100% rename from _static/dm3d-training-curves.png rename to _static/images/dm3d-training-curves.png diff --git a/_static/formula_.png b/_static/images/formula_.png similarity index 100% rename from _static/formula_.png rename to _static/images/formula_.png diff --git a/_static/hbm2015_exhibitors.jpg b/_static/images/hbm2015_exhibitors.jpg similarity index 100% rename from _static/hbm2015_exhibitors.jpg rename to _static/images/hbm2015_exhibitors.jpg diff --git a/_static/pretty_tracks.png b/_static/images/pretty_tracks.png similarity index 100% rename from _static/pretty_tracks.png rename to _static/images/pretty_tracks.png diff --git a/_static/simplified_tractography.png b/_static/images/simplified_tractography.png similarity index 100% rename from _static/simplified_tractography.png rename to _static/images/simplified_tractography.png diff --git a/_static/three_brains_golden_new_small.png b/_static/images/three_brains_golden_new_small.png similarity index 100% rename from _static/three_brains_golden_new_small.png rename to _static/images/three_brains_golden_new_small.png diff --git a/_static/vq-vae-results.png b/_static/images/vq-vae-results.png similarity index 100% rename from _static/vq-vae-results.png rename to _static/images/vq-vae-results.png diff --git a/_static/vqvae-f3-higher-epochs.png b/_static/images/vqvae-f3-higher-epochs.png similarity index 100% rename from _static/vqvae-f3-higher-epochs.png rename to _static/images/vqvae-f3-higher-epochs.png diff --git a/_static/vqvae-monai-B12-CC.png b/_static/images/vqvae-monai-B12-CC.png similarity index 100% rename from _static/vqvae-monai-B12-CC.png rename to _static/images/vqvae-monai-B12-CC.png diff --git a/_static/vqvae-monai-B12-both.png b/_static/images/vqvae-monai-B12-both.png similarity index 100% rename from _static/vqvae-monai-B12-both.png rename to _static/images/vqvae-monai-B12-both.png diff --git a/_static/vqvae-reconstructions-comparison.png b/_static/images/vqvae-reconstructions-comparison.png similarity index 100% rename from _static/vqvae-reconstructions-comparison.png rename to _static/images/vqvae-reconstructions-comparison.png diff --git a/_static/vqvae3d-monai-B10-ICNR.png b/_static/images/vqvae3d-monai-B10-ICNR.png similarity index 100% rename from _static/vqvae3d-monai-B10-ICNR.png rename to _static/images/vqvae3d-monai-B10-ICNR.png diff --git a/_static/vqvae3d-monai-B10.png b/_static/images/vqvae3d-monai-B10.png similarity index 100% rename from _static/vqvae3d-monai-B10.png rename to _static/images/vqvae3d-monai-B10.png diff --git a/_static/vqvae3d-monai-B5.png b/_static/images/vqvae3d-monai-B5.png similarity index 100% rename from _static/vqvae3d-monai-B5.png rename to _static/images/vqvae3d-monai-B5.png diff --git a/_static/vqvae3d-monai-training-plots.png b/_static/images/vqvae3d-monai-training-plots.png similarity index 100% rename from _static/vqvae3d-monai-training-plots.png rename to _static/images/vqvae3d-monai-training-plots.png diff --git a/_static/vqvae3d-monai-training.png b/_static/images/vqvae3d-monai-training.png similarity index 100% rename from _static/vqvae3d-monai-training.png rename to _static/images/vqvae3d-monai-training.png diff --git a/_static/vqvae3d-reconst-f2.png b/_static/images/vqvae3d-reconst-f2.png similarity index 100% rename from _static/vqvae3d-reconst-f2.png rename to _static/images/vqvae3d-reconst-f2.png diff --git a/_static/vqvae3d-reconst-f3.png b/_static/images/vqvae3d-reconst-f3.png similarity index 100% rename from _static/vqvae3d-reconst-f3.png rename to _static/images/vqvae3d-reconst-f3.png diff --git a/_static/vqvae3d-training-curves.png b/_static/images/vqvae3d-training-curves.png similarity index 100% rename from _static/vqvae3d-training-curves.png rename to _static/images/vqvae3d-training-curves.png diff --git a/_templates/documentation.html b/_templates/documentation.html deleted file mode 100644 index 8f489c4f..00000000 --- a/_templates/documentation.html +++ /dev/null @@ -1,14 +0,0 @@ - -{# documentation.html #} -{% for topic in topics %} -<div class="section"> - <h3 class="section-title">{{ topic.name }}</h3> - <div class="section-sub-topic-list"> - {% for sub_topic in topic["sub-topics"] %} - <a class="section-sub-topic" href="{{ sub_topic.link }}"> - <span> {{ sub_topic.name }} </span> - </a> - {% endfor %} - </div> -</div> -{% endfor %} \ No newline at end of file diff --git a/blog.rst b/blog.rst new file mode 100644 index 00000000..0f5f52cb --- /dev/null +++ b/blog.rst @@ -0,0 +1,3 @@ +==== +Blog +==== \ No newline at end of file diff --git a/conf.py b/conf.py index b666e0a3..cbb8e485 100644 --- a/conf.py +++ b/conf.py @@ -37,7 +37,7 @@ 'github', 'ablog', 'jinja' -] + ] # ghissue config github_project_url = "https://github.com/dipy/dipy.org" diff --git a/doc-requirements.txt b/doc-requirements.txt deleted file mode 100644 index 7507a132..00000000 --- a/doc-requirements.txt +++ /dev/null @@ -1,19 +0,0 @@ -cython -numpy>=1.22.4 -scipy>=1.8.1 -nibabel>=4.0.0 -h5py -cvxpy -pandas -tables -matplotlib -fury>=0.9.0 -scikit-learn -scikit-image -statsmodels -numpydoc -sphinx!=4.4 -sphinx-gallery>=0.10.0 -tomli>=2.0.1 -pydata-sphinx-theme==0.13.3 -Jinja2 \ No newline at end of file diff --git a/documentation.rst b/documentation.rst index 8cb41bf6..f9965d57 100644 --- a/documentation.rst +++ b/documentation.rst @@ -10,26 +10,9 @@ Documentation introduction mission installation - data - examples_built/index - interfaces/index faq developers - cite - devel/index - theory/index - reference/index - reference_cmd/index - api_changes .. Main content will be displayed using the jinja template -.. - Indices and tables - ================== - - * :ref:`genindex` - * :ref:`modindex` - * :ref:`search` - diff --git a/glossary.rst b/glossary.rst index 898cccfa..bae8f98b 100644 --- a/glossary.rst +++ b/glossary.rst @@ -6,7 +6,7 @@ Affine matrix A matrix implementing an :term:`affine transformation` in - :term:`homogenous coordinates`. For a 3 dimensional transform, the + :term:`Homogeneous coordinates`. For a 3 dimensional transform, the matrix is shape 4 by 4. Affine transformation diff --git a/index.rst b/index.rst index dd52ba22..26a989e8 100644 --- a/index.rst +++ b/index.rst @@ -79,7 +79,7 @@ As an exercise, you can try to calculate `color FA` with your datasets. You will to replace the filepaths `fdwi`, `fbval` and `fbvec`. Here is what a slice should look like. -.. image:: _static/colorfa.png +.. image:: _static/images/colorfa.png :align: center ********** diff --git a/introduction.rst b/introduction.rst index ba6ab66c..1bbc5bea 100644 --- a/introduction.rst +++ b/introduction.rst @@ -13,7 +13,7 @@ guidelines and try the :ref:`examples`. Didn't find what you are looking for? Then try :ref:`faq` and then if this doesn't help send an e-mail to our e-mail list neuroimaging@python.org with subject starting with ``[DIPY]``. -.. figure:: _static/pretty_tracks.png +.. figure:: _static/images/pretty_tracks.png :align: center This is a depiction of tractography created with DIPY. diff --git a/posts/2023/2023_05_29_Shilpi_Week_0_1.rst b/posts/2023/2023_05_29_Shilpi_Week_0_1.rst index e3536e08..3f6e9647 100644 --- a/posts/2023/2023_05_29_Shilpi_Week_0_1.rst +++ b/posts/2023/2023_05_29_Shilpi_Week_0_1.rst @@ -11,20 +11,20 @@ About Myself ~~~~~~~~~~~~ Hey there! I'm Shilpi, a Computer Science and Engineering undergrad at Dayananda Sagar College of Engineering, Bangalore. I'm on track to grab my degree in 2024. -My relationship with Python started just before I started college - got my hands dirty with this awesome Python Specialization course on Coursera. +My relationship with Python started just before I started college - got my hands dirty with this awesome Python Specialization course on Coursera. When it comes to what makes me tick, it's all things tech. I mean, new technology always excites me. Ubuntu, with its fancy terminal and all, used to intimidate me at first, but now, I get a thrill out of using it to do even the simplest things. -Up until 2nd year I used to do competitive programming and a bit of ML. But from 3rd year I've been into ML very seriously, doing several courses on ML as well solving ML problems on kaggle. ML is very fun and I've done a few project on ML as well. -Coding? Absolutely love it. It's like, this is what I was meant to do, y'know? I got introduced to git and GitHub in my first year - was super curious about how the whole version control thing worked. And then, I stumbled upon the world of open source in my second year and made my first contribution to Tardis: (`https://github.com/tardis-sn/tardis/pull/1825`_) +Up until 2nd year I used to do competitive programming and a bit of ML. But from 3rd year I've been into ML very seriously, doing several courses on ML as well solving ML problems on kaggle. ML is very fun and I've done a few project on ML as well. +Coding? Absolutely love it. It's like, this is what I was meant to do, y'know? I got introduced to git and GitHub in my first year - was super curious about how the whole version control thing worked. And then, I stumbled upon the world of open source in my second year and made my first contribution to Tardis: (`<https://github.com/tardis-sn/tardis/pull/1825>`_) Initially, I intended on doing GSoC during my second year but ended up stepping back for reasons. This time, though, I was fired up to send in a proposal to at least one organization in GSoC. And, well, here we are! Intro to Open-Source and GSoC ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ So, I started off finding out about GSoC - how many hours do selected folks put in, the kind of projects people usually tackle, and all that stuff. To get a handle on what they want in a proposal, I turned to some successful ones from previous years. Really gave me an idea of the kind of stuff they expect you to bring to the table. -Trying to find the organization that'd go with my skill set, I stumbled upon Python Software Foundation, and I was like, "This is it!". And under PSF, there was DIPY. +Trying to find the organization that'd go with my skill set, I stumbled upon Python Software Foundation, and I was like, "This is it!". And under PSF, there was DIPY. Diving into DIPY's docs was a breeze as they've got it so well put together that I managed to get my head around a completely new topic, "Diffusion MRI", just by going through their introductory docs and a bit of gpt. -While exploring DIPY, I noticed this issue that needed a new feature. It took a good bit of reading to really understand what they were looking for and how to actually build that feature. And then, I submitted my first PR (`check it out here: https://github.com/dipy/dipy/pull/2749`_)! Getting it merged wasn't exactly easy - there was a lot of room for improvement up in my code, but honestly, I feel like it's all part of the learning curve. -I was a bit of a latecomer to GSoC, so I didn't have much time to make up a ton of PRs. Plus, by the time I'd submitted my first PR, the proposal submission period had already begun. So, I focused all my energy on increasing my knowledge on the topic and polishing my proposal. Plus, I'd wanted to get my proposal reviewed at least once before I submitted it. +While exploring DIPY, I noticed this issue that needed a new feature. It took a good bit of reading to really understand what they were looking for and how to actually build that feature. And then, I submitted my first PR (`check it out here <https://github.com/dipy/dipy/pull/2749>`__)! Getting it merged wasn't exactly easy - there was a lot of room for improvement up in my code, but honestly, I feel like it's all part of the learning curve. +I was a bit of a latecomer to GSoC, so I didn't have much time to make up a ton of PRs. Plus, by the time I'd submitted my first PR, the proposal submission period had already begun. So, I focused all my energy on increasing my knowledge on the topic and polishing my proposal. Plus, I'd wanted to get my proposal reviewed at least once before I submitted it. Code contributions: @@ -38,16 +38,16 @@ I tried logging into Google to check, but couldn't. Too many people doing the sa Fast forward to 1:30am - I figured by now, the log-in rush should have calmed down. I gave it another shot and... I got in! I clicked over to the dashboard, and there it was. My project. Right there, listed under the Projects section. I had heard that if you get selected, your proposal shows up there. To confirm that it was actually happening, I picked my phone to check if I'd gotten any official email yet. And yes!! I'd gotten it at 12:49 am. I just hadn't checked. I whooped, woke up my roomies, rushed to call my parents. -Honestly, words can't even begin to capture how I felt at that moment. -Pure, undiluted joy, that's what it was. My parents, surprisingly actually picked up my call. But the minute I told them I'd made the cut, they congratulated me. It was heck of a day, (^^). +Honestly, words can't even begin to capture how I felt at that moment. +Pure, undiluted joy, that's what it was. My parents, surprisingly actually picked up my call. But the minute I told them I'd made the cut, they congratulated me. It was heck of a day, (^^). What I did this week ~~~~~~~~~~~~~~~~~~~~ As this was my first week I majorly spent a lot of time knowing about the codebase of the organization. I also went through a couple of research papers of projects which have already been implemented to get information related to my branch. -I'm currently in the middle of reading through the research paper, which is directly related to my project: `here <https://www.sciencedirect.com/science/article/pii/S1053811920300926>`_ -I also went through some of the videos related to information on cti, a couple of them are: `this <https://www.youtube.com/watch?v=bTFLGdbSi9M>`_ and also, `this <https://www.youtube.com/watch?v=2WtGl3YQou8&list=PLRZ9VSqV-6srrTAcDh4JYwrlef2Zpjucw&index=16>`_ -I also submitted `this <https://github.com/dipy/dipy/pull/2813>`_ PR. In this PR members of my organization are supposed to submit all the +I'm currently in the middle of reading through the research paper, which is directly related to my project: `here <https://www.sciencedirect.com/science/article/pii/S1053811920300926>`__ +I also went through some of the videos related to information on cti, a couple of them are: `this <https://www.youtube.com/watch?v=bTFLGdbSi9M>`__ and also, `this <https://www.youtube.com/watch?v=2WtGl3YQou8&list=PLRZ9VSqV-6srrTAcDh4JYwrlef2Zpjucw&index=16>`__ +I also submitted `this <https://github.com/dipy/dipy/pull/2813>`__ PR. In this PR members of my organization are supposed to submit all the blogs. But mostly I spent a lot of time in implementing the already existing MultiTensor Simulation on my local system , and also completing the assignment which my mentor gave me. In this assignment, I was given a specific number of directions, 'n' and some steps on how to produce bvals and bvecs. I had to create ``gtab1`` and ``gtab2``. and then taking ``gtab1`` & ``gtab2`` as input, I was supposed to create a function which would give output btensor i.e btens. @@ -60,17 +60,23 @@ These simulations were basically the first task of the proposal. So after the btensor I intend on producing the synthetic signals using the qti model (hint on how it is done in qti tests). make a figure similar to figure 1 of the 2021 CTI paper: -`here <https://onlinelibrary.wiley.com/doi/full/10.1002/mrm.28938>`_ +`here <https://onlinelibrary.wiley.com/doi/full/10.1002/mrm.28938>`__ Did I get stuck anywhere ~~~~~~~~~~~~~~~~~~~~~~~~ -I got stuck while creating `this <https://github.com/dipy/dipy/pull/2813>`_ PR. I had to rebase a bunch of commits, and this was a new concept to me, so it took me a while to figure it out. Due to rebasing, I ended up creating a bunch of other commits, which made the commit history of this PR a mess. So, I had to learn about the concept of squashing the commits. +I got stuck while creating `this <https://github.com/dipy/dipy/pull/2813>`__ PR. I had to rebase a bunch of commits, and this was a new concept to me, so it took me a while to figure it out. Due to rebasing, I ended up creating a bunch of other commits, which made the commit history of this PR a mess. So, I had to learn about the concept of squashing the commits. + +I also got stuck a lot while trying to find out the perpendicular directions to the vectors used in ``gtab1``. I was supposed to implement the following formula: + +.. image:: https://github.com/dipy/dipy/blob/09a8c4f8436f995e55231fb3d11fbfe6749610a9/_static/images/formula_.png?raw=true + :width: 400 + :alt: formula cti gtab -I also got stuck a lot while trying to find out the perpendicular directions to the vectors used in ``gtab1``. I was supposed to implement the formula in .. image:: https://github.com/dipy/dipy/blob/09a8c4f8436f995e55231fb3d11fbfe6749610a9/doc/_static/formula_.png?raw=true I had to spend a lot of time figuring out how to combine 3 vectors of shape (81, 3) to get V. And also working on the function which would give me the perpendicular vector to the vector in ``gtab1``. I got a bunch of ``ValueErrors`` saying: could not broadcast input array from shape (3,3,1) into shape (3,3) and some ``IndexError`` saying: shape mismatch: indexing arrays could not be broadcast together with shapes (81,) (3,1) (3,). I also had to experiment on how to concatenate different vectors to get the vector of the right shape, since there are a bunch of possible options while stacking, such as vstack, hstack, stack, etc. + diff --git a/posts/2023/2023_06_12_vara_week3.rst b/posts/2023/2023_06_12_vara_week3.rst index ec936a46..c0803223 100644 --- a/posts/2023/2023_06_12_vara_week3.rst +++ b/posts/2023/2023_06_12_vara_week3.rst @@ -17,7 +17,7 @@ The official code in Keras uses a ``tfp.layers.DistributionLambda(tfp.distributi Therefore, the autoregressive nature of PixelCNN combined with a sampling layer for every pixel delivers an effective generative model. The outputs for all my experiments are shown in the image below - -.. image:: /doc/_static/vq-vae-results.png +.. image:: /_static/images/vq-vae-results.png :width: 800 Based on qualitative analysis, PixelCNN outputs may require some extra work. This leads me to the next step in my research - to explore Diffusion models. The first breakthrough paper on Diffusion models is by DDPM - Denoising Diffusion Probabilistic models. Inspired by previous work on nonequilibrium thermodynamics, they show that training diffusion models while maximizing the posterior likelihood in an image generation task is mathematically equivalent to denoising score matching. diff --git a/posts/2023/2023_06_13_Shipi_week3.rst b/posts/2023/2023_06_13_Shipi_week3.rst index b596a387..9dfeae63 100644 --- a/posts/2023/2023_06_13_Shipi_week3.rst +++ b/posts/2023/2023_06_13_Shipi_week3.rst @@ -6,20 +6,20 @@ CTI Simulation and QTI tutorial : Week 3 :tags: google :category: gsoc -What I did this week: +What I did this week: _____________________ -This week I worked on finishing the simulations with the appropriate documentation. I also worked on creating a general tutorial for CTI/ QTI as one doesn't already exist for QTI. +This week I worked on finishing the simulations with the appropriate documentation. I also worked on creating a general tutorial for CTI/ QTI as one doesn't already exist for QTI. The idea behind this general tutorial was that there isn't any tutorial for advanced diffusion encoding. The closest documentation QTI has is :ref:`here <sphx_glr_examples_built_reconstruction_reconst_qti.py>`. However, there are several youtube videos. So, in this tutorial we started with simulating qti, and then we make things a little more complex by adding information on CTI as QTI can only handle a single Gradient Table whereas CTI can handle multiple Gradient Tables. -This week I also started by initializing ``cti_tests.py`` file by adding relevant simulations to it. +This week I also started by initializing ``cti_tests.py`` file by adding relevant simulations to it. -What Is coming up next week: +What Is coming up next week: ____________________________ I intend on finishing the simulations with appropriate documentation and theory lines. If time permits, I'll resume working on the ``cti.py`` file and it's tests section. The work on creating simulations is not entirely complete as it requires conversion of all .py files into rst so that it could be generated by sphinx. I also intend on making the tutorial more complete by maybe adding some more detail to it. -The major thing I intend on working on is the construction of the design matrix for the CTI model. +The major thing I intend on working on is the construction of the design matrix for the CTI model. In the context of diffusion MRI, a design matrix refers to a matrix that encodes the relationship between the data (diffusion-weighted signals) and the model parameters we want to estimate. It is essentially a way to map the model parameters to the observed data. The design matrix in case of CTI captures more complex tissue microstructure information than either DTI or DKI or QTI. diff --git a/posts/2023/2023_06_19_Shilpi_week4.rst b/posts/2023/2023_06_19_Shilpi_week4.rst index d07462eb..17677d03 100644 --- a/posts/2023/2023_06_19_Shilpi_week4.rst +++ b/posts/2023/2023_06_19_Shilpi_week4.rst @@ -11,10 +11,10 @@ Re-Engineering Simulation Codes with the QTI Model and Design Matrix What I did this week ~~~~~~~~~~~~~~~~~~~~ -I had to change the ``cti_test.py`` file as the signals generated were not exactly correct. I was advised to follow the multiple gaussian signal generation method. While doing this I had to look closely at several already implemented methods and go in depth to understand how those functions were achieving the desired output. +I had to change the ``cti_test.py`` file as the signals generated were not exactly correct. I was advised to follow the multiple gaussian signal generation method. While doing this I had to look closely at several already implemented methods and go in depth to understand how those functions were achieving the desired output. The multiple gaussian signal generation method is preferred because the CTI signal generation closely resembles the multiple gaussian signals. We're using the multiple gaussian signals so that we can have a priori of what to expect from the outcome, if we fit our model to this signal. I also managed to implement the design matrix for the CTI tensor and managed to save it up in the ``utils.py`` file. The design matrix is a crucial component of the CTI tensor as it represents the relationships between the different variables in our model. By accurately modeling these relationships, we can generate more realistic simulations and gain a deeper understanding of the CTI tensor. -The link of my work: :ref:`Here<https://github.com/dipy/dipy/pull/2816>` +The link of my work: `Here <https://github.com/dipy/dipy/pull/2816>__` @@ -26,4 +26,4 @@ This week I'll work on fitting CTI on multiple Gaussian simulations and see if i Did I get stuck anywhere ~~~~~~~~~~~~~~~~~~~~~~~~ -No, I didn't get stuck anywhere. +No, I didn't get stuck anywhere. diff --git a/posts/2023/2023_06_26_vara_week5.rst b/posts/2023/2023_06_26_vara_week5.rst index 6e2c44c4..497e6a03 100644 --- a/posts/2023/2023_06_26_vara_week5.rst +++ b/posts/2023/2023_06_26_vara_week5.rst @@ -23,14 +23,14 @@ Continuing on my MNIST experiments, I ran into Multi Distribution issues while t I followed the architecture described in my previous blog for the DM. I trained this on VQ-VAE latents of MNIST dataset for 200 diffusion steps, 2 Nvidia V100 GPUs, Adam Optimizer with 2e-4 learning rate, 200 batch size per GPU for 100+ epochs. For the generative process, I denoised random samples for 50, 100 and 200 steps on the best performing model(112 epochs). Here are the results I achieved - -.. image:: /doc/_static/DM-MNIST-112epoch.png +.. image:: /_static/images/DM-MNIST-112epoch.png :width: 800 We see some resemblance of digit shapes in the generated outputs. On further training for 300 diffusion timesteps for the best performing model( 108 epochs) with least training loss, the visuals have improved drastically - -.. image:: /doc/_static/DM-MNIST-DDIM300-108epoch.png +.. image:: /_static/images/DM-MNIST-DDIM300-108epoch.png :width: 800 diff --git a/posts/2023/2023_07_10_vara_week6_and_week7.rst b/posts/2023/2023_07_10_vara_week6_and_week7.rst index 8e2cf499..a4886526 100644 --- a/posts/2023/2023_07_10_vara_week6_and_week7.rst +++ b/posts/2023/2023_07_10_vara_week6_and_week7.rst @@ -1,5 +1,5 @@ Diffusion Model results on pre-trained VQVAE latents of NFBS MRI Dataset: Week 6 & Week 7 -======================================================================================== +========================================================================================= .. post:: July 10, 2023 @@ -17,36 +17,36 @@ What I did this week My current code for VQVAE & DM is well tested on MNIST dataset as shown in the previous blog posts. I extended the current codebase for MRI dataset by using 3D convolutions instead of 2D ones, which resulted in 600k parameters for VQVAE for a downsampling factor f=3. I used a preprocess function to transform MRI volumes to the desired shape (128,128,128,1) through DIPY's reslice and scipy's affine_transform functions, followed by MinMax normalization. I trained the VQVAE architecture for batch_size=10, Adam optimizer's lr=2e-4, 100 epochs. I followed suit for downsampling factor f=2 as well and got the following training curves- -.. image:: /doc/_static/vqvae3d-training-curves.png +.. image:: /_static/images/vqvae3d-training-curves.png :width: 800 The reconstructed brain volumes on the test dataset on the best performing model are as shown below. As seen in the first image, there are black artifacts in the captured blurry brain structure. Whereas the second image(f=2) does a better job in producing less blurrier brain structure. Nonetheless we only see the outline of the brain being captured with no micro-structural information inside them. -.. image:: /doc/_static/vqvae3d-reconst-f3.png +.. image:: /_static/images/vqvae3d-reconst-f3.png :width: 800 -.. image:: /doc/_static/vqvae3d-reconst-f2.png +.. image:: /_static/images/vqvae3d-reconst-f2.png :width: 800 Later, the 3D Diffusion Model was trained for approximately 200 epochs for 200 & 300 diffusion time steps in two different experiments respectively. The training curves and obtained generations are shown respectively. Both the generations are noisy and don't really have a convincing outlook. -.. image:: /doc/_static/dm3d-training-curves.png +.. image:: /_static/images/dm3d-training-curves.png :width: 800 -.. image:: /doc/_static/dm3d-reconst-D200-D300.png +.. image:: /_static/images/dm3d-reconst-D200-D300.png :width: 800 Given the achieved noisy generations, I decided to train VQVAE for a higher number of epochs. This may also indicate that the performance of DM is hitched on good latent representations i.e., a trained encoder capable of perfect reconstructions. So I trained f=3 VQVAE for a higher number of epochs as shown below. -.. image:: /doc/_static/vqvae-f3-higher-epochs.png +.. image:: /_static/images/vqvae-f3-higher-epochs.png :width: 800 diff --git a/posts/2023/2023_07_24_vara_week_8_9.rst b/posts/2023/2023_07_24_vara_week_8_9.rst index 4a96488a..a4643c17 100644 --- a/posts/2023/2023_07_24_vara_week_8_9.rst +++ b/posts/2023/2023_07_24_vara_week_8_9.rst @@ -30,7 +30,7 @@ Both methods have shown to perform better qualitatively in dealing with the chec I ran multiple experiments with batch_size=5,10,10(with ICNR). The training loss curves obtained are as follows, all of them trained on 1 GPU for 24hrs. We see that all of them converge except the last one(B=10 with ICNR). -.. image:: /doc/_static/vqvae3d-monai-training.png +.. image:: /_static/images/vqvae3d-monai-training.png :width: 800 @@ -40,20 +40,20 @@ The best training checkpoint has been used to reconstruct test images. Following The first one is for B=10, the best training checkpoint had training loss=0.0037. Compared to our previous VQVAE model, we see a better performance in capturing the brain outer structure. Moreover, we don't see white blobs or artifacts as inner matter, rather some curvatures contributing to the inner microstructure of a human brain. -.. image:: /doc/_static/vqvae3d-monai-B10.png +.. image:: /_static/images/vqvae3d-monai-B10.png :width: 800 The second one is for B=10 with ICNR kernel initialization, the best training checkpoint had training loss=0.0067. Although, the test results do not look complete. I implemented ICNR through DIPY's resize function to achieve NN resize equivalent output on the kernel filters. This initialization didn't work as it was intended to, further proving that the training is yet to be converged. -.. image:: /doc/_static/vqvae3d-monai-B10-ICNR.png +.. image:: /_static/images/vqvae3d-monai-B10-ICNR.png :width: 800 The next & last image is for B=5, the best training checkpoint had training loss = 0.0031. By far the best one quantitatively as well as visually. The test loss for the below reconstructions is 0.0013. The superior performance of this batch size can be owed to the Batch Normalization(BN) layers in the architecture that calculate mean & average of the batch to perform normalization over all batch elements using these statistics. Having lesser batch size may contribute to least variation in the output of the layer & helps in achieving converging outputs faster. This explanation stems from the concept of Contrastive Learning, where BN layers are used as the source of implicit negative loss learners. Higher the batch size, more implicit negative samples to move away from. Whereas our objective is to minimize the reconstruction loss, having lesser batch size consequently may help in lesser variation. -.. image:: /doc/_static/vqvae3d-monai-B5.png +.. image:: /_static/images/vqvae3d-monai-B5.png :width: 800 diff --git a/posts/2023/2023_08_07_vara_week_10_11.rst b/posts/2023/2023_08_07_vara_week_10_11.rst index f433a567..5bf4822a 100644 --- a/posts/2023/2023_08_07_vara_week_10_11.rst +++ b/posts/2023/2023_08_07_vara_week_10_11.rst @@ -1,5 +1,5 @@ Carbonate issues, GPU availability, Tensorflow errors: Week 10 & Week 11 -======================================================================= +======================================================================== .. post:: August 7, 2023 :author: Vara Lakshmi Bayanagari @@ -10,11 +10,11 @@ Carbonate issues, GPU availability, Tensorflow errors: Week 10 & Week 11 What I did this week ~~~~~~~~~~~~~~~~~~~~ -Recently, I've been an assigned RP(Research Project) account on University of Bloomington's HPC cluster - Carbonate. This account lets me access multiple GPUs for my experiments in a dedicated account. +Recently, I've been an assigned RP(Research Project) account on University of Bloomington's HPC cluster - Carbonate. This account lets me access multiple GPUs for my experiments in a dedicated account. -Once I started configuring my sbatch file accordingly, I started running into issues like GPU access. My debug print statements revealed that I'm accessing 1 CPU despite configuring the sbatch job for more than 1 GPUs. I double checked my dataloader definition, DistributionStrategy, train function. I read through IU's blogs as well as other resources online to see if I'm missing something. +Once I started configuring my sbatch file accordingly, I started running into issues like GPU access. My debug print statements revealed that I'm accessing 1 CPU despite configuring the sbatch job for more than 1 GPUs. I double checked my dataloader definition, DistributionStrategy, train function. I read through IU's blogs as well as other resources online to see if I'm missing something. -Nothing worked, my mentor eventually asked me to raise a IT request on Carbonate, the IT personnel couldn't help either. This could only mean that Tensorflow is picking upon assigned GPUs. So, on my mentor's suggestion, I loaded an older version of the deeplearning module 2.9.1(used 2.11.1 earlier). This worked! +Nothing worked, my mentor eventually asked me to raise a IT request on Carbonate, the IT personnel couldn't help either. This could only mean that Tensorflow is picking upon assigned GPUs. So, on my mentor's suggestion, I loaded an older version of the deeplearning module 2.9.1(used 2.11.1 earlier). This worked! This also meant using a downgraded version of tensorflow(2.9). This meant I ran into errors again, time taking yet resolvable. I made some architectural changes - replaced GroupNorm with BatchNorm layers, tensor_slices based DataLoader to DataGenerator - to accommodate for the older tensorflow version. Additionally, I also had to change the model structure from a list of layers to ``tensorflow.keras.Sequential`` set of layers with input_shape information defined in the first layer. Without this last change, I ran into ``None`` object errors. diff --git a/posts/2023/2023_08_21_vara_week_12_13.rst b/posts/2023/2023_08_21_vara_week_12_13.rst index 3174e577..ee8dee4a 100644 --- a/posts/2023/2023_08_21_vara_week_12_13.rst +++ b/posts/2023/2023_08_21_vara_week_12_13.rst @@ -17,37 +17,37 @@ Monai's VQVAE results on T1-weighted NFBS dataset, 125 samples, for batch size o Using existing training parameters, carried out two experiments, one on CC359 alone & another on both datasets combined. Additionally, I made a slight modification in the loss definition by attributing different weights of 0.5 & 1 to background & foreground pixels compared to equal weights from previous experiments. This resulted in faster convergence as shown in the red, blue & purple lines in the combined plot shown below. (Naming convention for each each training curve is ``B<batch_size>-<dataset_used>``, where CC=CC359, NFBS=NFBS, both=[NFBS,CC359]) -.. image:: /doc/_static/vqvae3d-monai-training-plots.png +.. image:: /_static/images/vqvae3d-monai-training-plots.png :alt: Combined trainings plots for all experiments :width: 800 Inference results on the best performing model, B12-both, is shown below, where every two rows correspond to reconstructions & original volumes respectively, with equally spaced slices in each row. These slices visualised are anterior-posterior topdown & ventral-dorsal LR. -.. image:: /doc/_static/vqvae-monai-B12-both.png +.. image:: /_static/images/vqvae-monai-B12-both.png :alt: VQVAE-Monai-B12-both reconstructions & originals showing equally spaced 5 slices for 2 different test samples :width: 800 Here's a similar visualization of the inference on the next best performing model, B12-CC. -.. image:: /doc/_static/vqvae-monai-B12-CC.png +.. image:: /_static/images/vqvae-monai-B12-CC.png :alt: VQVAE-Monai-B12-CC reconstructions & originals showing equally spaced 5 slices for 2 different test samples :width: 800 This shows that our training not only converged quickly but also improved visually. Here's a comparison of our current best performing model i.e., VQVAE-Monai-B12-both & the previous one on NFBS i.e., VQVAE-Monai-B5-NFBS. The test reconstruction loss is 0.0013 & 0.0015 respectively. -.. image:: /doc/_static/vqvae-reconstructions-comparison.png +.. image:: /_static/images/vqvae-reconstructions-comparison.png :alt: VQVAE reconstruction comparison for B12-both & B5-NFBS :width: 800 I also carried out Diffusion Model training on the best performing VQVAE-Monai-B12-both model for 300 & 500 diffusion steps and the training curves obtained are as follows- -.. image:: /doc/_static/dm3d-monai-training-curves.png +.. image:: /_static/images/dm3d-monai-training-curves.png :alt: Diffusion Model training plots for 300 & 500 diffusion steps :width: 800 These curves seemed to converge pretty quickly but the sampling outputs in the generation pipeline are still pure noise as shown below- -.. image:: /doc/_static/dm3d-monai-B8-DM500.png +.. image:: /_static/images/dm3d-monai-B8-DM500.png :alt: Diffusion Model training plots for 300 & 500 diffusion steps :width: 800 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..e768fbe2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +numpydoc +sphinx!=4.4 +tomli>=2.0.1 +grg-sphinx-theme>=0.1.0 +Jinja2 +ablog==0.11.5 \ No newline at end of file diff --git a/stateoftheart.rst b/stateoftheart.rst index a9d7cf79..fa1baaf7 100644 --- a/stateoftheart.rst +++ b/stateoftheart.rst @@ -4,7 +4,7 @@ A quick overview of features ============================ -Here are just a few of the state-of-the-art :ref:`technologies <examples>` and algorithms which are provided in DIPY_: +Here are just a few of the state-of-the-art `technologies <https//doc.dipy.org/examples>` and algorithms which are provided in DIPY_: - Reconstruction algorithms: CSD, DSI, GQI, DTI, DKI, QBI, SHORE and MAPMRI. - Fiber tracking algorithms: deterministic and probabilistic. @@ -21,33 +21,8 @@ With the help of some external tools you can also: - Read many different file formats e.g. Trackvis or Nifti (with nibabel). - Examine your datasets interactively (with ipython). -For more information on specific algorithms we recommend starting by looking at DIPY's :ref:`gallery <examples>` of examples. - -For a full list of the features implemented in the most recent release cycle, check out the release notes. - -.. toctree:: - :maxdepth: 1 - - release_notes/release1.7 - release_notes/release1.6 - release_notes/release1.5 - release_notes/release1.4.1 - release_notes/release1.4 - release_notes/release1.3 - release_notes/release1.2 - release_notes/release1.1 - release_notes/release1.0 - release_notes/release0.16 - release_notes/release0.15 - release_notes/release0.14 - release_notes/release0.13 - release_notes/release0.12 - release_notes/release0.11 - release_notes/release0.10 - release_notes/release0.9 - release_notes/release0.8 - release_notes/release0.7 - release_notes/release0.6 +For more information on specific algorithms we recommend starting by looking at DIPY's `gallery <https//doc.dipy.org/examples>` of examples. + ================= Systems supported diff --git a/tools/LICENSE.txt b/tools/LICENSE.txt deleted file mode 100644 index 9e1d415a..00000000 --- a/tools/LICENSE.txt +++ /dev/null @@ -1,7 +0,0 @@ -These files were obtained from - -https://www.mail-archive.com/sphinx-dev@googlegroups.com/msg02472.html - -and were released under a BSD/MIT license by Fernando Perez, Matthew Brett and -the PyMVPA folks. Further cleanups by the scikit-image crew. - diff --git a/tools/apigen.py b/tools/apigen.py deleted file mode 100644 index ed8d9733..00000000 --- a/tools/apigen.py +++ /dev/null @@ -1,560 +0,0 @@ -"""Attempt to generate templates for module reference with Sphinx. - -To include extension modules, first identify them as valid in the -``_uri2path`` method, then handle them in the ``_parse_module_with_import`` -script. - -Notes ------ -This parsing is based on the import and introspection of modules. -Previously functions and classes were found by parsing the text of .py files. - -Extension modules should be discovered and included as well. - -This is a modified version of a script originally shipped with the PyMVPA -project, then adapted for use first in NIPY and then in skimage. PyMVPA -is an MIT-licensed project. -""" - -import ast -import os -import re -from importlib import import_module - -from types import BuiltinFunctionType, FunctionType -from inspect import ismethod, getmodule - -# suppress print statements (warnings for empty files) -DEBUG = True - - -class ApiDocWriter(object): - """Automatic detection and parsing of API docs to Sphinx-parsable reST - format.""" - - # only separating first two levels - rst_section_levels = ['*', '=', '-', '~', '^'] - - def __init__(self, - package_name, - rst_extension='.txt', - package_skip_patterns=None, - module_skip_patterns=None, - object_skip_patterns=None, - other_defines=True - ): - r"""Initialize package for parsing. - - Parameters - ---------- - package_name : string - Name of the top-level package. *package_name* must be the - name of an importable package - rst_extension : string, optional - Extension for reST files, default '.rst' - package_skip_patterns : None or sequence of {strings, regexps} - Sequence of strings giving URIs of packages to be excluded - Operates on the package path, starting at (including) the - first dot in the package path, after *package_name* - so, - if *package_name* is ``sphinx``, then ``sphinx.util`` will - result in ``.util`` being passed for searching by these - regexps. If is None, gives default. Default is: - ['\.tests$'] - module_skip_patterns : None or sequence - Sequence of strings giving URIs of modules to be excluded - Operates on the module name including preceding URI path, - back to the first dot after *package_name*. For example - ``sphinx.util.console`` results in the string to search of - ``.util.console`` - If is None, gives default. Default is: - ['\.setup$', '\._'] - object_skip_patterns: None or sequence of {strings, regexps} - skip some specific class or function - other_defines : {True, False}, optional - Whether to include classes and functions that are imported in a - particular module but not defined there. - """ - if package_skip_patterns is None: - package_skip_patterns = ['\\.tests$'] - if module_skip_patterns is None: - module_skip_patterns = ['\\.setup$', '\\._'] - if object_skip_patterns is None: - object_skip_patterns = [] - - self.package_name = package_name - self.rst_extension = rst_extension - self.package_skip_patterns = package_skip_patterns - self.module_skip_patterns = module_skip_patterns - self.object_skip_patterns = object_skip_patterns - self.other_defines = other_defines - - def get_package_name(self): - return self._package_name - - def set_package_name(self, package_name): - """Set package_name. - - >>> docwriter = ApiDocWriter('sphinx') - >>> import sphinx - >>> docwriter.root_path == sphinx.__path__[0] - True - >>> docwriter.package_name = 'docutils' - >>> import docutils - >>> docwriter.root_path == docutils.__path__[0] - True - """ - # It's also possible to imagine caching the module parsing here - self._package_name = package_name - root_module = self._import(package_name) - self.root_path = root_module.__path__[-1] - self.written_modules = None - - package_name = property(get_package_name, set_package_name, None, - 'get/set package_name') - - def _import(self, name): - """Import namespace package.""" - mod = __import__(name) - components = name.split('.') - for comp in components[1:]: - mod = getattr(mod, comp) - return mod - - def _get_object_name(self, line): - """Get second token in line. - - Examples - -------- - >>> docwriter = ApiDocWriter('sphinx') - >>> docwriter._get_object_name(" def func(): ") - 'func' - >>> docwriter._get_object_name(" class Klass(object): ") - 'Klass' - >>> docwriter._get_object_name(" class Klass: ") - 'Klass' - """ - name = line.split()[1].split('(')[0].strip() - # in case we have classes which are not derived from object - # ie. old style classes - return name.rstrip(':') - - def _uri2path(self, uri): - """Convert uri to absolute filepath. - - Parameters - ---------- - uri : string - URI of python module to return path for - - Returns - ------- - path : None or string - Returns None if there is no valid path for this URI - Otherwise returns absolute file system path for URI - - Examples - -------- - >>> docwriter = ApiDocWriter('sphinx') - >>> import sphinx - >>> modpath = sphinx.__path__[0] - >>> res = docwriter._uri2path('sphinx.builder') - >>> res == os.path.join(modpath, 'builder.py') - True - >>> res = docwriter._uri2path('sphinx') - >>> res == os.path.join(modpath, '__init__.py') - True - >>> docwriter._uri2path('sphinx.does_not_exist') - - """ - if uri == self.package_name: - return os.path.join(self.root_path, '__init__.py') - path = uri.replace(self.package_name + '.', '') - path = path.replace('.', os.path.sep) - path = os.path.join(self.root_path, path) - # XXX maybe check for extensions as well? - if os.path.exists(path + '.py'): # file - path += '.py' - elif os.path.exists(path + '.pyx'): # file - path += '.pyx' - elif os.path.exists(os.path.join(path, '__init__.py')): - path = os.path.join(path, '__init__.py') - else: - return None - return path - - def _path2uri(self, dirpath): - """Convert directory path to uri.""" - package_dir = self.package_name.replace('.', os.path.sep) - relpath = dirpath.replace(self.root_path, package_dir) - if relpath.startswith(os.path.sep): - relpath = relpath[1:] - return relpath.replace(os.path.sep, '.') - - def _parse_module(self, uri): - """Parse module defined in *uri*.""" - filename = self._uri2path(uri) - if filename is None: - print(filename, 'erk') - # nothing that we could handle here. - return [], [] - - f = open(filename, 'rt') - functions, classes = self._parse_lines(f) - f.close() - return functions, classes - - def _parse_module_with_import(self, uri): - """Look for functions and classes in an importable module. - - Parameters - ---------- - uri : str - The name of the module to be parsed. This module needs to be - importable. - - Returns - ------- - functions : list of str - A list of (public) function names in the module. - classes : list of str - A list of (public) class names in the module. - """ - mod = import_module(uri) - patterns = '(?:{0})'.format('|'.join(self.object_skip_patterns)) - pat = re.compile(patterns) - - if mod.__file__.endswith('.py'): - with open(mod.__file__) as fi: - node = ast.parse(fi.read()) - - functions = [] - classes = [] - for n in node.body: - - if not hasattr(n, 'name'): - if not isinstance(n, ast.Assign): - continue - - if isinstance(n, ast.ClassDef): - if n.name.startswith('_') or pat.search(n.name): - continue - classes.append(n.name) - elif isinstance(n, ast.FunctionDef): - if n.name.startswith('_') or pat.search(n.name): - continue - functions.append(n.name) - # Specific condition for vtk and fury - elif isinstance(n, ast.Assign): - try: - if isinstance(n.value, ast.Call): - if isinstance(n.targets[0], ast.Tuple): - continue - functions.append(n.targets[0].id) - elif hasattr(n.value, 'attr') and n.value.attr.startswith('vtk'): - classes.append(n.targets[0].id) - except Exception: - print(mod.__file__) - print(n.lineno) - print(n.targets[0]) - - return functions, classes - else: - # find all public objects in the module. - obj_strs = [obj for obj in dir(mod) if not obj.startswith('_')] - functions = [] - classes = [] - for obj_str in obj_strs: - # find the actual object from its string representation - if obj_str not in mod.__dict__: - continue - obj = mod.__dict__[obj_str] - # Check if function / class defined in module - if not self.other_defines and not getmodule(obj) == mod: - continue - # figure out if obj is a function or class - if (hasattr(obj, 'func_name') or - isinstance(obj, BuiltinFunctionType) or - ismethod(obj) or isinstance(obj, FunctionType)): - functions.append(obj_str) - else: - try: - issubclass(obj, object) - classes.append(obj_str) - except TypeError: - # not a function or class - pass - return functions, classes - - def _parse_lines(self, linesource): - """Parse lines of text for functions and classes.""" - functions = [] - classes = [] - for line in linesource: - if line.startswith('def ') and line.count('('): - # exclude private stuff - name = self._get_object_name(line) - if not name.startswith('_'): - functions.append(name) - elif line.startswith('class '): - # exclude private stuff - name = self._get_object_name(line) - if not name.startswith('_'): - classes.append(name) - else: - pass - functions.sort() - classes.sort() - return functions, classes - - def generate_api_doc(self, uri): - """Make autodoc documentation template string for a module. - - Parameters - ---------- - uri : string - python location of module - e.g 'sphinx.builder' - - Returns - ------- - head : string - Module name, table of contents. - body : string - Function and class docstrings. - """ - # get the names of all classes and functions - functions, classes = self._parse_module_with_import(uri) - if not len(functions) and not len(classes) and DEBUG: - print('WARNING: Empty -', uri) # dbg - - # Make a shorter version of the uri that omits the package name for - # titles - uri_short = re.sub(r'^%s\.' % self.package_name, '', uri) - - head = '.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n' - body = '' - - # Set the chapter title to read 'module' for all modules except for the - # main packages - if '.' in uri_short: - title = 'Module: :mod:`' + uri_short + '`' - head += title + '\n' + self.rst_section_levels[2] * len(title) - else: - title = ':mod:`' + uri_short + '`' - head += title + '\n' + self.rst_section_levels[1] * len(title) - - head += '\n.. automodule:: ' + uri + '\n' - head += '\n.. currentmodule:: ' + uri + '\n' - body += '\n.. currentmodule:: ' + uri + '\n\n' - for c in classes: - body += '\n:class:`' + c + '`\n' \ - + self.rst_section_levels[3] * \ - (len(c) + 9) + '\n\n' - body += '\n.. autoclass:: ' + c + '\n' - # must NOT exclude from index to keep cross-refs working - body += ' :members:\n' \ - ' :undoc-members:\n' \ - ' :show-inheritance:\n' \ - '\n' \ - ' .. automethod:: __init__\n\n' - head += '.. autosummary::\n\n' - for f in classes + functions: - head += ' ' + f + '\n' - head += '\n' - - for f in functions: - # must NOT exclude from index to keep cross-refs working - body += f + '\n' - body += self.rst_section_levels[3] * len(f) + '\n' - body += '\n.. autofunction:: ' + f + '\n\n' - - return head, body - - def _survives_exclude(self, matchstr, match_type): - r"""Return True if *matchstr* does not match patterns. - - ``self.package_name`` removed from front of string if present - - Examples - -------- - >>> dw = ApiDocWriter('sphinx') - >>> dw._survives_exclude('sphinx.okpkg', 'package') - True - >>> dw.package_skip_patterns.append('^\\.badpkg$') - >>> dw._survives_exclude('sphinx.badpkg', 'package') - False - >>> dw._survives_exclude('sphinx.badpkg', 'module') - True - >>> dw._survives_exclude('sphinx.badmod', 'module') - True - >>> dw.module_skip_patterns.append('^\\.badmod$') - >>> dw._survives_exclude('sphinx.badmod', 'module') - False - """ - if match_type == 'module': - patterns = self.module_skip_patterns - elif match_type == 'package': - patterns = self.package_skip_patterns - else: - raise ValueError('Cannot interpret match type "%s"' - % match_type) - # Match to URI without package name - L = len(self.package_name) - if matchstr[:L] == self.package_name: - matchstr = matchstr[L:] - for pat in patterns: - try: - pat.search - except AttributeError: - pat = re.compile(pat) - if pat.search(matchstr): - return False - - return True - - def discover_modules(self): - r"""Return module sequence discovered from ``self.package_name``. - - Parameters - ---------- - None - - Returns - ------- - mods : sequence - Sequence of module names within ``self.package_name`` - - Examples - -------- - >>> dw = ApiDocWriter('sphinx') - >>> mods = dw.discover_modules() - >>> 'sphinx.util' in mods - True - >>> dw.package_skip_patterns.append('\.util$') - >>> 'sphinx.util' in dw.discover_modules() - False - >>> - """ - modules = [self.package_name] - # raw directory parsing - for dirpath, dirnames, filenames in os.walk(self.root_path): - # Check directory names for packages - root_uri = self._path2uri(os.path.join(self.root_path, - dirpath)) - - # Normally, we'd only iterate over dirnames, but since - # dipy does not import a whole bunch of modules we'll - # include those here as well (the *.py filenames). - filenames = [os.path.splitext(f)[0] for f in filenames - if (f.endswith('.py') and - not f.startswith('__init__')) - or f.endswith('.pyx')] - - for subpkg_name in dirnames + filenames: - package_uri = '.'.join((root_uri, subpkg_name)) - package_path = self._uri2path(package_uri) - if (package_path and - self._survives_exclude(package_uri, 'package')): - modules.append(package_uri) - - return sorted(modules) - - def write_modules_api(self, modules, outdir): - # upper-level modules - ulms = ['.'.join(m.split('.')[:2]) if m.count('.') >= 1 - else m.split('.')[0] for m in modules] - - from collections import OrderedDict - module_by_ulm = OrderedDict() - - for v, k in zip(modules, ulms): - if k in module_by_ulm: - module_by_ulm[k].append(v) - else: - module_by_ulm[k] = [v] - - written_modules = [] - - for ulm, mods in module_by_ulm.items(): - print("Generating docs for %s:" % ulm) - document_head = [] - document_body = [] - - for m in mods: - print(" -> " + m) - head, body = self.generate_api_doc(m) - - document_head.append(head) - document_body.append(body) - - out_module = ulm + self.rst_extension - outfile = os.path.join(outdir, out_module) - fileobj = open(outfile, 'wt') - - fileobj.writelines(document_head + document_body) - fileobj.close() - written_modules.append(out_module) - - self.written_modules = written_modules - - def write_api_docs(self, outdir): - """Generate API reST files. - - Parameters - ---------- - outdir : string - Directory name in which to store files - We create automatic filenames for each module - - Returns - ------- - None - - Notes - ----- - Sets self.written_modules to list of written modules - """ - if not os.path.exists(outdir): - os.mkdir(outdir) - # compose list of modules - modules = self.discover_modules() - self.write_modules_api(modules, outdir) - - def write_index(self, outdir, froot='gen', relative_to=None): - """Make a reST API index file from written files. - - Parameters - ---------- - path : string - Filename to write an index to - outdir : string - Directory to which to write generated index file - froot : string, optional - root (filename without extension) of filename to write to - Defaults to 'gen'. We add ``self.rst_extension``. - relative_to : string - path to which written filenames are relative. This - component of the written file path will be removed from - outdir, in the generated index. Default is None, meaning, - leave the path as it is. - """ - if self.written_modules is None: - raise ValueError('No modules written') - # Get full filename path - path = os.path.join(outdir, froot + self.rst_extension) - # Path written into index is relative to rootpath - if relative_to is not None: - relpath = (outdir + os.path.sep).replace( - relative_to + os.path.sep, '') - else: - relpath = outdir - idx = open(path, 'wt') - w = idx.write - w('.. AUTO-GENERATED FILE -- DO NOT EDIT!\n\n') - - title = "API Reference" - w(title + "\n") - w("=" * len(title) + "\n\n") - w('.. toctree::\n\n') - for f in self.written_modules: - w(' %s\n' % os.path.join(relpath, f)) - idx.close() diff --git a/tools/build_modref_templates.py b/tools/build_modref_templates.py deleted file mode 100644 index 1b520317..00000000 --- a/tools/build_modref_templates.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env python -"""Script to auto-generate our API docs. -""" - -# stdlib imports -import sys -import re -from os.path import join as pjoin - -# local imports -from apigen import ApiDocWriter - -# version comparison -from packaging.version import Version - -# ***************************************************************************** - - -def abort(error): - print('*WARNING* API documentation not generated: %s' % error) - exit() - - -if __name__ == '__main__': - package = sys.argv[1] - outdir = sys.argv[2] - try: - other_defines = sys.argv[3] - except IndexError: - other_defines = True - else: - other_defines = other_defines in ('True', 'true', '1') - - # Check that the package is available. If not, the API documentation is not - # (re)generated and existing API documentation sources will be used. - - try: - __import__(package) - except ImportError as e: - abort("Can not import " + package) - - module = sys.modules[package] - - # Check that the source version is equal to the installed - # version. If the versions mismatch the API documentation sources - # are not (re)generated. This avoids automatic generation of documentation - # for older or newer versions if such versions are installed on the system. - - installed_version = Version(module.__version__) - - info_file = pjoin('..', package, 'info.py') - info_lines = open(info_file).readlines() - source_version = '.'.join([v.split('=')[1].strip(" '\n.") - for v in info_lines if re.match( - '^_version_(major|minor|micro|extra)', v - )]).strip('.') - source_version = Version(source_version) - print('***', source_version) - - if source_version != installed_version: - print('***', installed_version) - abort("Installed version does not match source version") - - docwriter = ApiDocWriter(package, rst_extension='.rst', - other_defines=other_defines) - docwriter.package_skip_patterns += [r'\.tracking\.interfaces.*$', - r'\.tracking\.gui_tools.*$', - r'.*test.*$', - r'^\.utils.*', - r'\.boots\.resampling.*$', - r'\.info.*$', - r'\.pkg_info.*$', - ] - docwriter.object_skip_patterns += [ - r'.*FetcherError.*$', - r'.*urlopen.*', - r'.*add_callback.*', - ] - docwriter.write_api_docs(outdir) - docwriter.write_index(outdir, 'index', relative_to=outdir) - print('%d files written' % len(docwriter.written_modules)) diff --git a/tools/docgen_cmd.py b/tools/docgen_cmd.py deleted file mode 100644 index 9d643b9c..00000000 --- a/tools/docgen_cmd.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python -""" -Script to generate documentation for command line utilities -""" -import os -from os.path import join as pjoin -import re -from subprocess import Popen, PIPE, CalledProcessError -import sys -import importlib -import inspect - -# version comparison -from packaging.version import Version - -# List of workflows to ignore -SKIP_WORKFLOWS_LIST = ('Workflow', 'CombinedWorkflow') - - -def sh3(cmd): - """ - Execute command in a subshell, return stdout, stderr - If anything appears in stderr, print it out to sys.stderr - - https://github.com/scikit-image/scikit-image/blob/master/doc/gh-pages.py - - Copyright (C) 2011, the scikit-image team All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - Neither the name of skimage nor the names of its contributors may be used - to endorse or promote products derived from this software without specific - prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF - USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - """ - p = Popen(cmd, stdout=PIPE, stderr=PIPE, shell=True) - out, err = p.communicate() - retcode = p.returncode - if retcode: - raise CalledProcessError(retcode, cmd) - else: - return out.rstrip(), err.rstrip() - - -def abort(error): - print('*WARNING* Command line API documentation not generated: %s' % error) - exit() - - -def get_help_string(class_obj): - # return inspect.getdoc(class_obj.run) - try: - ia_module = importlib.import_module("dipy.workflows.base") - parser = ia_module.IntrospectiveArgumentParser() - parser.add_workflow(class_obj()) - except Exception as e: - abort("Error on {0}: {1}".format(class_obj.__name__, e)) - - return parser.format_help() - - -def format_title(text): - text = text.title() - line = '-' * len(text) - return '{0}\n{1}\n\n'.format(text, line) - - -if __name__ == '__main__': - # package name: Eg: dipy - package = sys.argv[1] - # directory in which the generated rst files will be saved - outdir = sys.argv[2] - - try: - __import__(package) - except ImportError: - abort("Cannot import " + package) - - module = sys.modules[package] - - # Check that the source version is equal to the installed - # version. If the versions mismatch the API documentation sources - # are not (re)generated. This avoids automatic generation of documentation - # for older or newer versions if such versions are installed on the system. - - installed_version = Version(module.__version__) - - info_file = pjoin('..', package, 'info.py') - info_lines = open(info_file).readlines() - source_version = '.'.join( - [v.split('=')[1].strip(" '\n.") - for v in info_lines - if re.match('^_version_(major|minor|micro|extra)', v)]).strip('.') - source_version = Version(source_version) - print('***', source_version) - - if source_version != installed_version: - print('***', installed_version) - abort("Installed version does not match source version") - - - # generate docs - command_list = [] - - workflows_folder = pjoin('..', 'bin') - workflow_module = importlib.import_module("dipy.workflows.workflow") - - workflow_flist = [os.path.abspath(pjoin(workflows_folder, f)) - for f in os.listdir(workflows_folder) - if os.path.isfile(pjoin(workflows_folder, f)) and - f.lower().startswith("dipy_")] - - workflow_flist = sorted(workflow_flist) - workflow_desc = {} - # We get all workflows class obj in a dictionary - for path_file in os.listdir(pjoin('..', 'dipy', 'workflows')): - module_name = inspect.getmodulename(path_file) - if module_name is None: - continue - - module = importlib.import_module("dipy.workflows." + module_name) - members = inspect.getmembers(module) - d_wkflw = {name: {"module": obj, "help": get_help_string(obj)} - for name, obj in members - if inspect.isclass(obj) and - issubclass(obj, workflow_module.Workflow) and - name not in SKIP_WORKFLOWS_LIST - } - - workflow_desc.update(d_wkflw) - - cmd_list = [] - for fpath in workflow_flist: - fname = os.path.basename(fpath) - with open(fpath) as file_object: - flow_name = set(re.findall(r"[A-Z]\w+Flow", file_object.read(), - re.X | re.M)) - - if not flow_name or len(flow_name) != 1: - continue - - flow_name = list(flow_name)[-1] - print("Generating docs for: {0} ({1})".format(fname, flow_name)) - out_fname = fname + ".rst" - with open(pjoin(outdir, out_fname), "w") as fp: - dashes = "=" * len(fname) - fp.write("\n{0}\n{1}\n{0}\n\n".format(dashes, fname)) - # Trick to avoid docgen_cmd.py as cmd line - help_txt = workflow_desc[flow_name]["help"] - help_txt = help_txt.replace("docgen_cmd.py", fname) - help_txt = help_txt.replace("usage:", format_title('usage')) - help_txt = help_txt.replace("positional arguments:", - format_title("positional arguments")) - help_txt = help_txt.replace("optional arguments:", - format_title("optional arguments")) - help_txt = help_txt.replace( - "output arguments(optional):", - format_title("output arguments(optional)")) - help_txt = help_txt.replace("References:", - format_title("References")) - help_txt = help_txt.rstrip() - fp.write(help_txt) - - cmd_list.append(out_fname) - print("Done") - - # generate index.rst - print("Generating index.rst") - with open(pjoin(outdir, "index.rst"), "w") as index: - index.write(".. _workflows_reference:\n\n") - index.write("Command Line Utilities Reference\n") - index.write("================================\n\n") - index.write(".. toctree::\n\n") - for cmd in cmd_list: - index.write(" " + cmd) - index.write("\n") - print("Done")