finishing up (#1)

* finishing up

Signed-off-by: vsoch <[email protected]>

.devcontainer/Dockerfile

@@ -0,0 +1,26 @@
+FROM ghcr.io/converged-computing/metric-ior:latest
+
+LABEL maintainer="Vanessasaurus <@vsoch>"
+
+# Match the default user id for a single system so we aren't root
+ARG USERNAME=vscode
+ARG USER_UID=1000
+ARG USER_GID=1000
+ENV USERNAME=${USERNAME}
+ENV USER_UID=${USER_UID}
+ENV USER_GID=${USER_GID}
+USER root
+
+# extra interactive utilities
+RUN apt-get update \
+ && apt-get -qq install -y --no-install-recommends \
+    vim less sudo python3-pip
+
+# Ensure regular python is visible
+RUN ln -s /usr/bin/python3 /usr/bin/python
+
+# Add the group and user that match our ids
+RUN groupadd -g ${USER_GID} ${USERNAME} && \
+    adduser --disabled-password --uid ${USER_UID} --gid ${USER_GID} --gecos "" ${USERNAME} && \
+    echo "${USERNAME} ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers
+USER $USERNAME

.devcontainer/devcontainer.json

@@ -0,0 +1,17 @@
+{
+  "name": "Compspec IOR Development Environment",
+  "dockerFile": "Dockerfile",
+  "context": "../",
+
+  "customizations": {
+    "vscode": {
+      "settings": {
+        "terminal.integrated.defaultProfile.linux": "bash"
+  },
+      "extensions": [
+         "ms-vscode.cmake-tools"
+      ]
+    }
+  },
+  "postStartCommand": "git config --global --add safe.directory /workspaces/compspec-ior"
+}

.github/workflows/main.yaml

@@ -0,0 +1,54 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request: []
+
+jobs:
+  test:
+    name: Test IOR
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+      - name: Install compspec
+        run: pip install compspec
+      - name: Install compspec-ior
+        run: pip install .
+      - name: Test with loading data
+        run: compspec extract ior --ior-load ./examples/test/ior-data.json
+      - name: Test Python
+        run: python ./examples/singleton-run.py
+
+  validate-schema:
+    name: Validate schema
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+      - name: Validate Schema
+        uses: compspec/actions/validate-schema@main
+        with:
+          schema: ./compspec_ior/schema.json
+
+  formatting:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Setup black linter
+      run: conda create --quiet --name black pyflakes
+
+    - name: Check Spelling
+      uses: crate-ci/typos@7ad296c72fa8265059cc03d1eda562fbdfcd6df2 # v1.9.0
+      with:
+        files: ./README.md
+
+    - name: Lint and format Python code
+      run: |
+        export PATH="/usr/share/miniconda/bin:$PATH"
+        source activate black
+        pip install -r .github/dev-requirements.txt
+        pre-commit run --all-files

README.md

@@ -4,6 +4,8 @@
   <img height="300" src="https://raw.githubusercontent.com/compspec/spec/main/img/compspec-circle.png">
 </p>
 
+[![PyPI version](https://badge.fury.io/py/compspec-ior.svg)](https://badge.fury.io/py/compspec-ior)
+
 A compspec (Composition spec) is a specification and model for comparing things. Compspec IOR is
 a plugin for extraction of [IOR](https://github.com/hpc/ior) metadata from applications, and packaging in compatibility specification
 artifacts. This means that we also maintain the compatibility schema here. To learn more:
@@ -23,21 +25,205 @@ pip install compspec-ior
 ```
 
 Then run an extraction with IOR. You can use defaults, or add any parameters to IOR after the plugin name "ior"
+Here is how to print to the terminal:
+
+```bash
+compspec extract ior
+```
+
+<details>
+
+<summary>IOR output</summary>
+
+```console
+{
+    "version": "0.0.0",
+    "kind": "CompatibilitySpec",
+    "metadata": {
+        "name": "compat-experiment",
+        "schemas": {
+            "io.compspec.ior": "https://raw.githubusercontent.com/compspec/compspec-ior/main/compspec_ior/schema.json"
+        }
+    },
+    "compatibilities": [
+        {
+            "name": "io.compspec.ior",
+            "version": "0.0.0",
+            "attributes": {
+                "version": "4.0.0rc1",
+                "began": "Thu Feb 22 00:36:12 2024",
+                "machine": "Linux 2b3ee0c4c948",
+                "finished": "Thu Feb 22 00:36:12 2024",
+                "command_line": "ior -O summaryFormat=JSON",
+                "summary.write.operation": "write",
+                "summary.write.API": "POSIX",
+                "summary.write.TestID": 0,
+                "summary.write.ReferenceNumber": 0,
+                "summary.write.segmentCount": 1,
+                "summary.write.blockSize": 1048576,
+                "summary.write.transferSize": 262144,
+                "summary.write.numTasks": 1,
+                "summary.write.tasksPerNode": 1,
+                "summary.write.repetitions": 1,
+                "summary.write.filePerProc": 0,
+                "summary.write.reorderTasks": 0,
+                "summary.write.taskPerNodeOffset": 1,
+                "summary.write.reorderTasksRandom": 0,
+                "summary.write.reorderTasksRandomSeed": 0,
+                "summary.write.bwMaxMIB": 904.92,
+                "summary.write.bwMinMIB": 904.92,
+                "summary.write.bwMeanMIB": 904.92,
+                "summary.write.bwStdMIB": 0.0,
+                "summary.write.OPsMax": 3619.6798,
+                "summary.write.OPsMin": 3619.6798,
+                "summary.write.OPsMean": 3619.6798,
+                "summary.write.OPsSD": 0.0,
+                "summary.write.MeanTime": 0.0011,
+                "summary.write.xsizeMiB": 1.0,
+                "summary.read.operation": "read",
+                "summary.read.API": "POSIX",
+                "summary.read.TestID": 0,
+                "summary.read.ReferenceNumber": 0,
+                "summary.read.segmentCount": 1,
+                "summary.read.blockSize": 1048576,
+                "summary.read.transferSize": 262144,
+                "summary.read.numTasks": 1,
+                "summary.read.tasksPerNode": 1,
+                "summary.read.repetitions": 1,
+                "summary.read.filePerProc": 0,
+                "summary.read.reorderTasks": 0,
+                "summary.read.taskPerNodeOffset": 1,
+                "summary.read.reorderTasksRandom": 0,
+                "summary.read.reorderTasksRandomSeed": 0,
+                "summary.read.bwMaxMIB": 6615.6215,
+                "summary.read.bwMinMIB": 6615.6215,
+                "summary.read.bwMeanMIB": 6615.6215,
+                "summary.read.bwStdMIB": 0.0,
+                "summary.read.OPsMax": 26462.4858,
+                "summary.read.OPsMin": 26462.4858,
+                "summary.read.OPsMean": 26462.4858,
+                "summary.read.OPsSD": 0.0,
+                "summary.read.MeanTime": 0.0002,
+                "summary.read.xsizeMiB": 1.0,
+                "test.0.starttime": "Thu Feb 22 00:36:12 2024",
+                "test.0.capacity": "1.8 TiB",
+                "test.0.used_capacity": "20.2%",
+                "test.0.inodes": "116.4 Mi",
+                "test.0.used_inodes": "5.3%",
+                "test.0.parameters.testID": 0,
+                "test.0.parameters.refnum": 0,
+                "test.0.parameters.api": "POSIX",
+                "test.0.parameters.platform": "2b3ee0c4c(Linux)",
+                "test.0.parameters.testFileName": "testFile",
+                "test.0.parameters.deadlineForStonewall": 0,
+                "test.0.parameters.stoneWallingWearOut": 0,
+                "test.0.parameters.maxTimeDuration": 0,
+                "test.0.parameters.outlierThreshold": 0,
+                "test.0.parameters.options": "(null)",
+                "test.0.parameters.dryRun": 0,
+                "test.0.parameters.nodes": 1,
+                "test.0.parameters.memoryPerTask": 0,
+                "test.0.parameters.memoryPerNode": 0,
+                "test.0.parameters.tasksPerNode": 1,
+                "test.0.parameters.repetitions": 1,
+                "test.0.parameters.multiFile": 0,
+                "test.0.parameters.interTestDelay": 0,
+                "test.0.parameters.fsync": 0,
+                "test.0.parameters.fsyncperwrite": 0,
+                "test.0.parameters.useExistingTestFile": 0,
+                "test.0.parameters.uniqueDir": 0,
+                "test.0.parameters.singleXferAttempt": 0,
+                "test.0.parameters.readFile": 1,
+                "test.0.parameters.writeFile": 1,
+                "test.0.parameters.filePerProc": 0,
+                "test.0.parameters.reorderTasks": 0,
+                "test.0.parameters.reorderTasksRandom": 0,
+                "test.0.parameters.reorderTasksRandomSeed": 0,
+                "test.0.parameters.randomOffset": 0,
+                "test.0.parameters.checkWrite": 0,
+                "test.0.parameters.checkRead": 0,
+                "test.0.parameters.dataPacketType": 0,
+                "test.0.parameters.keepFile": 0,
+                "test.0.parameters.keepFileWithError": 0,
+                "test.0.parameters.warningAsErrors": 0,
+                "test.0.parameters.verbose": 0,
+                "test.0.parameters.data packet type": "g",
+                "test.0.parameters.setTimeStampSignature/incompressibleSeed": 0,
+                "test.0.parameters.collective": 0,
+                "test.0.parameters.segmentCount": 1,
+                "test.0.parameters.transferSize": 262144,
+                "test.0.parameters.blockSize": 1048576,
+                "test.0.options.api": "POSIX",
+                "test.0.options.apiVersion": "",
+                "test.0.options.test filename": "testFile",
+                "test.0.options.access": "single-shared-file",
+                "test.0.options.type": "independent",
+                "test.0.options.segments": 1,
+                "test.0.options.ordering in a file": "sequential",
+                "test.0.options.ordering inter file": "no tasks offsets",
+                "test.0.options.nodes": 1,
+                "test.0.options.tasks": 1,
+                "test.0.options.clients per node": 1,
+                "test.0.options.repetitions": 1,
+                "test.0.options.xfersize": "262144 bytes",
+                "test.0.options.blocksize": "1 MiB",
+                "test.0.options.aggregate filesize": "1 MiB",
+                "test.0.results.0.access": "write",
+                "test.0.results.0.bwMiB": 904.92,
+                "test.0.results.0.blockKiB": 1024.0,
+                "test.0.results.0.xferKiB": 256.0,
+                "test.0.results.0.iops": 3842.6972,
+                "test.0.results.0.latency": 0.0003,
+                "test.0.results.0.openTime": 0.0001,
+                "test.0.results.0.wrRdTime": 0.001,
+                "test.0.results.0.closeTime": 0.0,
+                "test.0.results.0.totalTime": 0.0011,
+                "test.0.results.1.access": "read",
+                "test.0.results.1.bwMiB": 6615.6215,
+                "test.0.results.1.blockKiB": 1024.0,
+                "test.0.results.1.xferKiB": 256.0,
+                "test.0.results.1.iops": 27962.0267,
+                "test.0.results.1.latency": 0.0,
+                "test.0.results.1.openTime": 0.0,
+                "test.0.results.1.wrRdTime": 0.0001,
+                "test.0.results.1.closeTime": 0.0,
+                "test.0.results.1.totalTime": 0.0002
+            }
+        }
+    ]
+}
+```
+
+</details>
+
+And how to save to file
+
+```bash
+compspec  extract --outfile ior-test.json ior
+```
+
+And run the example to see how to use compspec-ior directly in Python to generate the same
+artifact.
+
+```bash
+python ./examples/singleton-run.py
+```
+
+
+### Development
+
+If you open the [Development container](.devcontainer) in VSCode, you'll find ior on the path:
 
 ```bash
-compspec extract ior ...
+$ which ior
+/usr/bin/ior
 ```
 
-More coming soon!
+This allows us to easily develop and test the compatibility plugin. You can
 
 ## TODO
 
-- Developer environment with IOR installed (for others and me too)
-- testing, etc with pre-commit and spell checking
-- implement run functionality
- - use reasonable defaults for when nothing provided
- - outputs should map to new schema.json attributes
- - main library compspec should have support for oras push, etc.
+- how to handle adding lists (with indices) to schema?
 
 ## License
 

compspec_ior/__init__.py

@@ -1,2 +1,2 @@
-from .plugin import ExtractorPlugin
+from .plugin import Plugin
 from .version import __version__

compspec_ior/defaults.py

@@ -0,0 +1,4 @@
+# These are required to be given to compspec
+spec_version = "0.0.0"
+schema_url = "https://raw.githubusercontent.com/compspec/compspec-ior/main/compspec_ior/schema.json"
+namespace = "io.compspec.ior"