diff --git a/.github/workflows/code.yml b/.github/workflows/code.yml
index 54d35e5..340cbdc 100644
--- a/.github/workflows/code.yml
+++ b/.github/workflows/code.yml
@@ -33,12 +33,12 @@ jobs:
       matrix:
         os: ["ubuntu-latest"] # can add windows-latest, macos-latest
         python: ["3.9", "3.10"]
-        install: ["-e .[dev]"]
+        install: ["-e .[dev,ioc,plotting]"]
         # Make one version be non-editable to test both paths of version code
         include:
           - os: "ubuntu-latest"
             python: "3.9"
-            install: ".[dev]"
+            install: ".[dev,ioc,plotting]"
 
     runs-on: ${{ matrix.os }}
     env:
@@ -147,7 +147,7 @@ jobs:
         uses: docker/build-push-action@v3
         with:
           build-args: |
-            PIP_OPTIONS=-r lockfiles/requirements.txt dist/*.whl
+            PIP_OPTIONS=-r lockfiles/requirements.txt ${ls dist/*.whl}[plotting]
           push: ${{ github.event_name != 'pull_request' && endsWith(github.ref, 'main') }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
diff --git a/pyproject.toml b/pyproject.toml
index 4145734..3824be6 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -20,11 +20,7 @@ dependencies = [
     "ipython",
     "pyepics",
     "click",
-    "PyQt6",
-    "PyQt5",
     "pydantic",
-    "softioc",
-    "pandablocks",
     "scipy",
 ]
 dynamic = ["version"]
@@ -33,6 +29,8 @@ readme = "README.rst"
 requires-python = ">=3.9"
 
 [project.optional-dependencies]
+plotting = ["PyQt6", "PyQt5", "matplotlib"]
+ioc = ["softioc", "pandablocks"]
 dev = [
     "black",
     "mypy",