Merge branch 'dev' into (feat)customized_loss_and_val_func

# Conflicts:
#	pypots/classification/base.py
#	pypots/classification/grud/core.py
#	pypots/classification/raindrop/core.py
#	pypots/clustering/base.py
#	pypots/clustering/crli/model.py
#	pypots/clustering/vader/core.py
#	pypots/clustering/vader/model.py
#	pypots/forecasting/base.py
#	pypots/forecasting/csdi/core.py
#	pypots/forecasting/csdi/model.py
#	pypots/forecasting/template/model.py
#	pypots/imputation/base.py
#	pypots/imputation/csdi/core.py
#	pypots/imputation/csdi/model.py
#	pypots/imputation/gpvae/model.py
#	pypots/imputation/nonstationary_transformer/core.py
#	pypots/imputation/saits/core.py
#	pypots/imputation/saits/model.py
#	pypots/imputation/template/model.py
#	pypots/imputation/usgan/core.py
#	pypots/nn/modules/csdi/backbone.py

.github/ISSUE_TEMPLATE/config.yml

@@ -1,6 +1,10 @@
 blank_issues_enabled: true
 version: 2.1
 contact_links:
-  - name: PyPOTS Community on Slack
+  - name: 🤗 PyPOTS Community on Slack
     url: https://join.slack.com/t/pypots-org/shared_invite/zt-1gq6ufwsi-p0OZdW~e9UW_IA4_f1OfxA
     about: General usage questions, community discussions, and the development team are here.
+
+  - name: 🇨🇳 PyPOTS微信社区 (Community on WeChat)
+    url: https://mp.weixin.qq.com/s/CGkirx7ODkngk6tGGANksw
+    about: 中文社区讨论群，欢迎加入！

.github/ISSUE_TEMPLATE/model-addition.yml

@@ -1,6 +1,6 @@
 name: "🤙 New Model Addition"
 description: Submit a request to implement a new model.
-labels: ["new model", "enhancement"]
+labels: ["new algo", "enhancement"]
 
 body:
   - type: textarea

.github/workflows/greetings.yml

@@ -18,11 +18,14 @@ jobs:
     steps:
     - uses: actions/first-interaction@v1
       with:
-        repo-token: ${{ secrets.ACCESS_TOKEN }}
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
         issue-message: |
           Hi there 👋,
 
-          Thank you so much for your attention to PyPOTS! You can follow me on GitHub to receive the latest news of PyPOTS. If you find PyPOTS helpful to your work, please star⭐️ this repository. Your star is your recognition, which can help more people notice PyPOTS and grow PyPOTS community. It matters and is definitely a kind of contribution to the community.
+          Thank you so much for your attention to PyPOTS! You can [follow me](https://github.com/WenjieDu) on GitHub 
+          to receive the latest news of PyPOTS. If you find PyPOTS helpful to your work, please star⭐️ this repository.
+          Your star is your recognition, which can help more people notice PyPOTS and grow PyPOTS community.
+          It matters and is definitely a kind of contribution to the community.
 
           I have received your message and will respond ASAP. Thank you for your patience! 😃
 
@@ -31,7 +34,7 @@ jobs:
         pr-message: |
           Hi there 👋,
 
-          We really really appreciate that you have taken the time to make this PR on PyPOTS!
+          We really appreciate that you have taken the time to make this PR on PyPOTS!
 
           If you are trying to fix a bug, please reference the issue number in the description or give your details about the bug.
           If you are implementing a feature request, please check with the maintainers that the feature will be accepted first.

.github/workflows/linting.yml

@@ -23,8 +23,8 @@ jobs:
 
             - name: Install Flake8
               run: |
-                pip install flake8
+                pip install flake8 flake8-pyproject
 
             - name: Run linting
               run: |
-                flake8 .
+                flake8 .

.github/workflows/publish_to_PyPI.yml

@@ -31,15 +31,11 @@ jobs:
       with:
         python-version: '3.10'
         check-latest: true
-        cache: pip
-        cache-dependency-path: |
-          setup.cfg
 
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install build wheel
-        pip install -r requirements.txt
+        pip install build
 
     - name: Fetch the test environment details
       run: |
@@ -48,7 +44,7 @@ jobs:
 
     - name: Build package
       run: |
-        python -m build --no-isolation
+        python -m build
 
     - name: Publish the new package to PyPI
       uses: pypa/[email protected]

.github/workflows/testing_ci.yml

@@ -19,53 +19,49 @@ jobs:
         strategy:
             fail-fast: false
             matrix:
-                os: [ubuntu-latest, windows-latest, macOS-13]
-                python-version: ["3.7", "3.11"]
+                os: [ubuntu-latest, windows-latest, macOS-latest]
+                python-version: ["3.8", "3.11"]
+                pytorch-version: ["2.3.0"]
 
         steps:
             - name: Check out the repo code
               uses: actions/checkout@v3
 
-            - name: Determine the PyTorch version
-              uses: haya14busa/action-cond@v1
-              id: determine_pytorch_ver
-              with:
-                cond: ${{ matrix.python-version == 3.7 }}
-                if_true: "1.13.1"
-                if_false: "2.1.0"
-
             - name: Set up Python
               uses: actions/setup-python@v4
               with:
                 python-version: ${{ matrix.python-version }}
                 check-latest: true
                 cache: pip
                 cache-dependency-path: |
-                  setup.cfg
+                  requirements/requirements.txt
+                  requirements/requirements_dev.txt
 
-            - name: Install PyTorch ${{ steps.determine_pytorch_ver.outputs.value }}+cpu
+            - name: Install PyTorch ${{ matrix.pytorch-version }}+cpu
               # we have to install torch in advance because torch_sparse needs it for compilation,
               # refer to https://github.com/rusty1s/pytorch_sparse/issues/156#issuecomment-1304869772 for details
               run: |
                   which python
                   which pip
-                  pip install torch==${{ steps.determine_pytorch_ver.outputs.value }} -f https://download.pytorch.org/whl/cpu
+                  pip install torch==${{ matrix.pytorch-version }} -f https://download.pytorch.org/whl/cpu
                   python -c "import torch; print('PyTorch:', torch.__version__)"
 
             - name: Install other dependencies
               run: |
-                  pip install -r requirements.txt
+                  pip install -r requirements/requirements.txt
 
             - name: Test building package
               # we need to know if the package can be built successfully without optional dependencies
               run: |
                   pip install build wheel
-                  python -m build --no-isolation
+                  python -m build
 
             - name: Continue to install torch-geometric dependencies
               run: |
-                  pip install torch-geometric torch-scatter torch-sparse -f "https://data.pyg.org/whl/torch-${{ steps.determine_pytorch_ver.outputs.value }}+cpu.html"
+                  pip install torch-geometric torch-scatter torch-sparse -f "https://data.pyg.org/whl/torch-${{ matrix.pytorch-version }}+cpu.html"
                   pip install pypots[dev]
+                  pip install numpy==1.24  # many libs not compatible with numpy 2.0. Note 3.12 requests for numpy>=2.0
+                  pip install pandas==1.5  # fix pandas version to avoid installing pandas 2.0, the same reason with numpy
                   python_site_path=`python -c "import site; print(site.getsitepackages()[0])"`
                   echo "python site-packages path: $python_site_path"
                   rm -rf $python_site_path/pypots
@@ -88,6 +84,7 @@ jobs:
 
             - name: Submit the report
               uses: coverallsapp/github-action@master
+              continue-on-error: true
               with:
                   github-token: ${{ secrets.GITHUB_TOKEN }}
                   path-to-lcov: "coverage.lcov"

.github/workflows/testing_daily.yml

@@ -4,6 +4,9 @@ on:
     schedule:
         # https://crontab.guru. Run everyday at 0:00AM UTC, i.e. 08:00AM Beijing, i.e. 08:00PM Montreal (summer time)
         - cron: "0 0 * * *"
+    push:
+        branches:
+            - temp_test_branch  # if in need, create such a temporary branch to test some functions
 
 jobs:
     Daily-testing:
@@ -14,56 +17,52 @@ jobs:
         strategy:
             fail-fast: false
             matrix:
-                os: [ubuntu-latest, windows-latest, macOS-13]
-                python-version: ["3.7", "3.11"]
+                os: [ ubuntu-latest, windows-latest, macOS-latest ]
+                python-version: [ "3.8" ]
+                pytorch-version: ["2.3.0"]
 
         steps:
             - name: Check out the repo code
               uses: actions/checkout@v3
 
-            - name: Determine the PyTorch version
-              uses: haya14busa/action-cond@v1
-              id: determine_pytorch_ver
-              with:
-                cond: ${{ matrix.python-version == 3.7 }}
-                if_true: "1.13.1"
-                if_false: "2.1.0"
 
             - name: Set up Conda
               uses: conda-incubator/setup-miniconda@v2
               with:
-                  activate-environment: pypots-test
+                  miniconda-version: "latest"
+                  activate-environment: pypots
                   python-version: ${{ matrix.python-version }}
-                  environment-file: tests/environment_for_conda_test.yml
+                  environment-file: requirements/conda_dev_env.yml
                   auto-activate-base: false
 
             - name: Fetch the test environment details
               run: |
+                  # many libs not compatible with numpy 2.0. Note 3.12 requests for numpy>=2.0. fix pandas version to avoid installing pandas 2.0, the same reason with numpy
+                  conda install -c conda-forge numpy=1.24 pandas=1.5
                   which python
                   conda info
                   conda list
 
+            - name: Replace with the latest PyPOTS code for testing
+              run: |
+                  python_site_path=`python -c "import site; print(site.getsitepackages()[0])"`
+                  echo "python site-packages path: $python_site_path"
+                  rm -rf $python_site_path/pypots
+                  python -c "import shutil;import site;shutil.copytree('pypots',site.getsitepackages()[0]+'/pypots')"
+
             - name: Test with pytest
               run: |
-                  # run tests separately here due to Segmentation Fault in test_clustering when run all in
-                  # one command with `pytest` on MacOS. Bugs not caught, so this is a trade-off to avoid SF.
-                  python tests/global_test_config.py
                   rm -rf testing_results && rm -rf tests/__pycache__ && rm -rf tests/*/__pycache__
-                  python -m pytest -rA tests/classification/* -s -n auto --cov=pypots --dist=loadgroup --cov-config=.coveragerc
-                  python -m pytest -rA tests/imputation/* -s -n auto --cov=pypots --cov-append --dist=loadgroup --cov-config=.coveragerc
-                  python -m pytest -rA tests/clustering/* -s -n auto --cov=pypots --cov-append --dist=loadgroup --cov-config=.coveragerc
-                  python -m pytest -rA tests/forecasting/* -s -n auto --cov=pypots --cov-append --dist=loadgroup --cov-config=.coveragerc
-                  python -m pytest -rA tests/optim/* -s -n auto --cov=pypots --cov-append --dist=loadgroup --cov-config=.coveragerc
-                  python -m pytest -rA tests/data/* -s -n auto --cov=pypots --cov-append --dist=loadgroup --cov-config=.coveragerc
-                  python -m pytest -rA tests/utils/* -s -n auto --cov=pypots --cov-append --dist=loadgroup --cov-config=.coveragerc
-                  python -m pytest -rA tests/cli/* -s -n auto --cov=pypots --cov-append --dist=loadgroup --cov-config=.coveragerc
+                  python tests/global_test_config.py
+                  python -m pytest -rA tests/*/* -s -n auto --cov=pypots --dist=loadgroup --cov-config=.coveragerc
 
             - name: Generate the LCOV report
               run: |
                   python -m coverage lcov
 
             - name: Submit the report
               uses: coverallsapp/github-action@master
+              continue-on-error: true
               with:
                   github-token: ${{ secrets.GITHUB_TOKEN }}
                   path-to-lcov: 'coverage.lcov'

.gitignore

@@ -19,3 +19,8 @@ docs/_build
 # ignore specific kinds of files like all PDFs
 *.pdf
 *.ipynb
+*TODO
+
+# ignore executable and config files
+*.sh
+*.yml

.pre-commit-config.yaml

@@ -1,23 +1,26 @@
 repos:
     # hooks for checking files
     -   repo: https://github.com/pre-commit/pre-commit-hooks
-        rev: v4.4.0
+        rev: v4.6.0
         hooks:
             -   id: trailing-whitespace
             -   id: end-of-file-fixer
             -   id: check-yaml
 
     # hooks for linting code
     -   repo: https://github.com/psf/black
-        rev: 22.10.0
+        rev: 24.8.0
         hooks:
             -   id: black
+                args: [
+                    --line-length=120, # refer to pyproject.toml
+                ]
 
     -   repo: https://github.com/PyCQA/flake8
-        rev: 6.0.0
+        rev: 7.1.1
         hooks:
             -   id: flake8
                 args: [
-                    --max-line-length=120, # refer to setup.cfg
-                    --extend-ignore=E203, # why ignore E203? Refer to https://github.com/PyCQA/pycodestyle/issues/373
+                    --max-line-length=120, # refer to pyproject.toml
+                    --extend-ignore=E203,E231
                 ]

.readthedocs.yaml

@@ -23,17 +23,18 @@ build:
             - python -m pip install --upgrade pip
             - pip install torch==1.13.1 -f https://download.pytorch.org/whl/cpu
             - pip install torch-geometric==2.3.1 torch-scatter==2.1.1 torch-sparse==0.6.17 -f "https://data.pyg.org/whl/torch-1.13.1+cpu.html"
-            - pip install -r requirements.txt
+            - pip install -r requirements/requirements.txt
             - pip install pypots
-            - pip install sphinx==6.2.1 docutils==0.19 sphinxcontrib-bibtex==2.1.4 sphinxcontrib-gtagjs sphinx-autodoc-typehints furo==2023.07.26
-            - git clone https://github.com/WenjieDu/TSDB TSDB_repo --depth 1
-            - git clone https://github.com/WenjieDu/PyGrinder PyGrinder_repo --depth 1
-            - pip install ./TSDB_repo && pip install ./PyGrinder_repo && pip install .
+            - pip install sphinx==7.4.7 docutils==0.21.2 sphinxcontrib-bibtex==2.6.2 sphinxcontrib-gtagjs sphinx-autodoc-typehints furo==2024.7.18
+            - git clone https://github.com/WenjieDu/TSDB TSDB_repo --depth 1 && pip install ./TSDB_repo
+            - git clone https://github.com/WenjieDu/PyGrinder PyGrinder_repo --depth 1 && pip install ./PyGrinder_repo
+            - git clone https://github.com/WenjieDu/BenchPOTS BenchPOTS_repo --depth 1 && pip install ./BenchPOTS_repo
+            - pip install .
 
         post_install:
             # To fix the exception: This documentation is not using `furo.css` as the stylesheet.
             #   If you have set `html_style` in your conf.py file, remove it.
-            - pip install sphinx==7.2.6
+            - pip install sphinx==7.4.7
             # this docutils version fixes issue#102, put it in post_install to avoid being
             #   overwritten by other versions (like 0.19) while installing other packages
-            - pip install docutils==0.20
+            - pip install docutils==0.21.2