Merge pull request #40 from saraedum/ccpcache

Rework caching to use (possibly local) .json files

.github/workflows/lint.yml

@@ -14,7 +14,7 @@ jobs:
       - uses: actions/checkout@v2
         with: { submodules: recursive }
       - uses: conda-incubator/setup-miniconda@v2
-        with: { mamba-version: "*", channels: "conda-forge", python-version: "3.9", channel-priority: true }
+        with: { miniforge-variant: "Mambaforge", miniforge-version: "latest" }
       - name: install dependencies
         shell: bash -l {0}
         run: |

.github/workflows/test.yml

@@ -19,15 +19,42 @@ jobs:
       - uses: actions/checkout@v2
         with: { submodules: recursive }
       - uses: conda-incubator/setup-miniconda@v2
-        with: { mamba-version: "*", channels: "flatsurf,conda-forge", python-version: "${{ matrix.python }}", channel-priority: true }
+        with: { miniforge-variant: "Mambaforge", miniforge-version: "latest", python-version: "${{ matrix.python }}" }
       - name: install dependencies
-        shell: bash -l {0}
+        shell: bash -el {0}
         run: |
           mamba env update --quiet -n test -f environment.yml
           conda list
+      - name: install flatsurvey
+        shell: bash -el {0}
+        run: |
+          pip install -e .
       - name: doctest
-        shell: bash -l {0}
+        shell: bash -el {0}
         run: |
           # Make sure that initialization message does not show up in doctest output.
           python -c 'import cppyy'
           pytest -n auto -x --doctest-modules flatsurvey
+      - name: run survey
+        shell: bash -el {0}
+        run: |
+          flatsurvey orbit-closure ngons --vertices 3 --limit 7 json
+      - name: build cache database from previous run
+        shell: bash -el {0}
+        run: |
+          # Shrink cache files by externalizing the referenced surfaces.
+          mkdir -p pickles
+          flatsurvey-maintenance externalize-pickles --inplace --pickles pickles/ *.json
+
+          # Creata single cache file from the runs.
+          flatsurvey-maintenance join *.json
+
+          cat orbit-closure.json
+      - name: run survey from cache
+        shell: bash -el {0}
+        run: |
+          # The survey succeeds with the cache enabled.
+          flatsurvey orbit-closure ngons --vertices 3 --limit 7 local-cache --json orbit-closure.json log
+
+          # The survey succeeds without running any computations.
+          flatsurvey orbit-closure --cache-only ngons --vertices 3 --limit 7 local-cache --json orbit-closure.json log

README.md

@@ -1,6 +1,6 @@
 Automation scripts for the [flatsurf](https://github.com/flatsurf) stack to survey large sets of objects.
 
-To perform a full survey, use [`flatsurvey`](./flatsurvey/__main__.py). To investigate a single object, run [`flatsurvey-worker`](./flatsurvey/worker/__main__.py).
+To perform a full survey, use [`flatsurvey`](./flatsurvey/survey.py). To investigate a single object, run [`flatsurvey-worker`](./flatsurvey/worker/worker.py).
 
 For example, here is a typical survey that collects data about triangles, quadrilaterals, and pentagons:
 
@@ -11,3 +11,4 @@ nice flatsurvey ngons --vertices 3 ngons --vertices 4 ngons --vertices 5 orbit-c
 # Troubleshooting
 
 * For large surveys, RAM might become the limiting factor. It appears that we are not actually leaking memory but are hit by memory fragmentation during the Boshernitzan criterion. The issue can be fully mitigated by replacing malloc with [Mesh](https://github.com/plasma-umass/Mesh), i.e., setting `LD_PRELOAD=/path/to/libmesh.so`.
+* SageMath (or one of its dependencies) might decide that it's beneficial to parallelize things further. However, this can easily overload a system. Typically, some linear algebra might spawn a process for each CPU on the system which could then easily lead to CPU² many processes if you run a lengthy survey. To disable this behaviour, set `MKL_NUM_THREADS=1 OMP_NUM_THREADS=1 SAGE_NUM_THREADS=1`.

Survey.ipynb

@@ -153,7 +153,7 @@
        "  'result': <function Results._resolve.<locals>.restore at 0x7fee27a7a440>,\n",
        "  'command': ['undetermined-iet'],\n",
        "  'intervals': 8,\n",
-       "  'invocation': ['/dev/shm/ruth/flatsurvey/flatsurvey/worker/__main__.py',\n",
+       "  'invocation': ['/dev/shm/ruth/flatsurvey/flatsurvey/worker/worker.py',\n",
        "   'log',\n",
        "   '--output=ngon-1-3-3-3.log',\n",
        "   'graphql',\n",

environment.yml

@@ -3,26 +3,22 @@
 # conda env create -f environment.yml
 name: flatsurvey
 channels:
-  - flatsurf
   - conda-forge
-  - defaults
 dependencies:
   - black >=22,<23
-  - boto3
   - click
+  - humanfriendly
   - isort
-  - moto
   - pip
-  - plumbum
+  - psutil
   - pyflatsurf
   - pyintervalxt
   - pylint
   - pytest
   - pytest-xdist
   - python-dateutil
+  - rich
   - ruamel.yaml
-  - sage-flatsurf >=0.4.4
+  - sage-flatsurf >=0.5.2,<0.6
   - pip:
-    - aioresponses
-    - gql==3.0.0a6
     - pinject