From 9fcc9e310bbce4bb41a0942551513e6263459c46 Mon Sep 17 00:00:00 2001 From: Paulo Rego <832830+paulovmr@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:53:30 -0300 Subject: [PATCH] Migrate Elyra extensions to support JupyterLab 4.2.5 (#2) --------- Signed-off-by: Harshad Reddy Nalla Signed-off-by: shalberd <21118431+shalberd@users.noreply.github.com> Co-authored-by: Guilherme Caponetto <638737+caponetto@users.noreply.github.com> Co-authored-by: Eder Ignatowicz Co-authored-by: Edson Tirelli Co-authored-by: Harshad Reddy Nalla Co-authored-by: Sven Thoms <21118431+shalberd@users.noreply.github.com> --- .eslintrc.json | 14 +- .github/workflows/build.yml | 129 +- .gitignore | 3 + .prettierrc | 3 +- .yarnrc.yml | 2 - Makefile | 27 +- build_requirements.txt | 2 +- docs/requirements.txt | 2 +- elyra/metadata/schemas/airflow.json | 8 +- elyra/metadata/schemas/code-snippet.json | 6 +- elyra/metadata/schemas/kfp.json | 8 +- elyra/metadata/schemas/metadata-test.json | 4 +- elyra/metadata/schemas/metadata-test2.json | 4 +- elyra/metadata/schemas/runtime-image.json | 4 +- elyra/metadata/schemas/url-catalog.json | 2 +- .../airflow-package-catalog.json | 2 +- .../airflow-provider-package-catalog.json | 2 +- elyra/pipeline/kfp/PipelineConf.py | 158 + elyra/pipeline/kfp/kfp_authentication.py | 8 +- elyra/pipeline/kfp/processor_kfp.py | 97 +- elyra/pipeline/pipeline_definition.py | 3 + ...neric_component_definition_template.jinja2 | 24 + .../kubeflow/v2/python_dsl_template.jinja2 | 152 + .../pipelines/kf_inputpath_parameter.pipeline | 6 +- elyra/tests/cli/test_pipeline_app.py | 9 + elyra/tests/metadata/test_metadata.py | 2 +- .../tests/pipeline/kfp/test_processor_kfp.py | 490 +- .../resources/components/download_data.yaml | 4 +- .../resources/components/filter_text.yaml | 16 +- .../kf_inputpath_parameter.pipeline | 6 +- ...alid_inputpath_missing_connection.pipeline | 4 +- .../kf_invalid_inputpath_parameter.pipeline | 6 +- ...kf_supernode_invalid_single_cycle.pipeline | 4 +- .../kf_supernode_valid.pipeline | 4 +- .../kf_with_parameters.pipeline | 2 +- etc/docker/elyra_development/requirements.yml | 4 +- etc/generic/requirements-elyra.txt | 26 +- etc/scripts/generate-make-graph.ts | 8 +- .../elyra_code_snippet_extension/__init__.py | 14 + .../elyra_metadata_common/__init__.py | 14 + .../elyra_metadata_extension/__init__.py | 14 + .../__init__.py | 14 + .../elyra_python_editor_extension/__init__.py | 14 + .../__init__.py | 14 + labextensions/elyra_script_editor/__init__.py | 14 + labextensions/elyra_services/__init__.py | 14 + .../elyra_theme_extension/__init__.py | 14 + labextensions/elyra_ui_components/__init__.py | 14 + package.json | 15 +- packages/code-snippet/install.json | 5 + packages/code-snippet/package.json | 62 +- packages/code-snippet/setup.py | 1 + .../code-snippet/src/CodeSnippetService.ts | 4 +- .../code-snippet/src/CodeSnippetWidget.tsx | 127 +- packages/code-snippet/src/index.ts | 47 +- packages/code-viewer/package.json | 2 +- packages/code-viewer/src/CodeViewerWidget.ts | 4 +- packages/code-viewer/src/index.ts | 20 +- packages/metadata-common/install.json | 5 + packages/metadata-common/package.json | 43 +- packages/metadata-common/setup.py | 1 + packages/metadata-common/src/FilterTools.tsx | 30 +- .../src/MetadataCommonService.tsx | 6 +- .../metadata-common/src/MetadataEditor.tsx | 8 +- .../src/MetadataEditorWidget.tsx | 22 +- .../metadata-common/src/MetadataWidget.tsx | 56 +- packages/metadata/install.json | 5 + packages/metadata/package.json | 47 +- packages/metadata/setup.py | 1 + packages/metadata/src/index.ts | 76 +- packages/pipeline-editor/install.json | 5 + packages/pipeline-editor/package.json | 72 +- .../schema/src/ComponentCatalogsWidget.tsx | 222 - .../schema/src/EmptyPipelineContent.tsx | 121 - .../schema/src/FileSubmissionDialog.tsx | 144 - .../schema/src/ParameterInputForm.tsx | 124 - .../schema/src/PipelineEditorWidget.tsx | 1215 -- .../schema/src/PipelineExportDialog.tsx | 98 - .../schema/src/PipelineService.tsx | 359 - .../schema/src/PipelineSubmissionDialog.tsx | 56 - .../schema/src/RuntimeConfigSelect.tsx | 99 - .../schema/src/RuntimeImagesWidget.tsx | 112 - .../schema/src/RuntimesWidget.tsx | 239 - .../schema/src/SubmitFileButtonExtension.tsx | 174 - .../pipeline-editor/schema/src/dialogs.tsx | 68 - .../schema/src/formDialogWidget.ts | 49 - packages/pipeline-editor/schema/src/index.ts | 395 - .../schema/src/pipeline-hooks.ts | 297 - .../schema/src/runtime-utils.ts | 100 - .../schema/src/test/pipeline-hooks.spec.ts | 183 - .../schema/src/test/pipeline-service.spec.ts | 48 - packages/pipeline-editor/schema/src/theme.tsx | 121 - packages/pipeline-editor/schema/src/utils.ts | 125 - packages/pipeline-editor/setup.py | 1 + .../src/ComponentCatalogsWidget.tsx | 20 +- .../src/EmptyPipelineContent.tsx | 2 +- .../src/FileSubmissionDialog.tsx | 2 +- .../src/ParameterInputForm.tsx | 8 +- .../src/PipelineEditorWidget.tsx | 280 +- .../src/PipelineExportDialog.tsx | 2 +- .../pipeline-editor/src/PipelineService.tsx | 48 +- .../src/PipelineSubmissionDialog.tsx | 2 +- .../src/RuntimeConfigSelect.tsx | 6 +- .../src/RuntimeImagesWidget.tsx | 2 +- .../pipeline-editor/src/RuntimesWidget.tsx | 16 +- .../src/SubmitFileButtonExtension.tsx | 30 +- packages/pipeline-editor/src/dialogs.tsx | 12 +- .../pipeline-editor/src/formDialogWidget.ts | 4 +- packages/pipeline-editor/src/index.ts | 421 +- .../pipeline-editor/src/pipeline-hooks.ts | 88 +- packages/pipeline-editor/src/runtime-utils.ts | 16 +- .../src/test/pipeline-hooks.spec.ts | 58 +- .../src/test/pipeline-service.spec.ts | 4 +- packages/pipeline-editor/src/theme.tsx | 28 +- packages/pipeline-editor/src/utils.ts | 28 +- packages/python-editor/install.json | 5 + packages/python-editor/package.json | 59 +- packages/python-editor/setup.py | 1 + packages/python-editor/src/PythonEditor.tsx | 2 +- packages/python-editor/src/index.ts | 62 +- packages/r-editor/package.json | 2 +- packages/r-editor/src/REditor.tsx | 2 +- packages/r-editor/src/index.ts | 44 +- packages/scala-editor/package.json | 2 +- packages/scala-editor/src/ScalaEditor.tsx | 2 +- packages/scala-editor/src/index.ts | 44 +- packages/script-debugger/install.json | 5 + packages/script-debugger/package.json | 45 +- packages/script-debugger/setup.py | 1 + packages/script-debugger/src/index.ts | 20 +- packages/script-editor/install.json | 5 + packages/script-editor/package.json | 61 +- packages/script-editor/setup.py | 1 + packages/script-editor/src/KernelDropdown.tsx | 19 +- packages/script-editor/src/ScriptEditor.tsx | 60 +- .../src/ScriptEditorController.ts | 4 +- .../src/ScriptEditorWidgetFactory.tsx | 8 +- packages/script-editor/src/ScriptRunner.ts | 55 +- .../src/test/script-editor.spec.ts | 4 +- packages/script-editor/style/index.css | 18 - packages/services/install.json | 5 + packages/services/jest.config.js | 2 +- packages/services/package.json | 49 +- packages/services/setup.py | 1 + packages/services/src/metadata.ts | 16 +- packages/services/src/parsing.ts | 2 +- packages/services/src/requests.ts | 22 +- packages/services/src/test/services.spec.ts | 34 +- packages/theme/install.json | 5 + packages/theme/package.json | 49 +- packages/theme/setup.py | 1 + packages/theme/src/index.ts | 35 +- packages/theme/src/launcher.tsx | 10 +- packages/ui-components/install.json | 5 + packages/ui-components/package.json | 44 +- packages/ui-components/setup.py | 1 + .../ui-components/src/BrowseFileDialog.tsx | 30 +- packages/ui-components/src/Dropzone.tsx | 6 +- .../ui-components/src/ExpandableComponent.tsx | 4 +- .../src/ExpandableErrorDialog.tsx | 6 +- .../src/FormComponents/CodeBlock.tsx | 56 +- .../src/FormComponents/DropDown.tsx | 41 +- .../src/FormComponents/PasswordField.tsx | 23 +- .../ui-components/src/FormComponents/Tags.tsx | 48 +- packages/ui-components/src/FormDialog.tsx | 12 +- packages/ui-components/src/FormEditor.tsx | 49 +- packages/ui-components/src/JSONComponent.tsx | 6 +- packages/ui-components/src/RequestErrors.tsx | 8 +- packages/ui-components/src/icons.tsx | 48 +- pyproject.toml | 46 +- scripts/start-test-server.ts | 2 +- tests/.eslintrc.js | 6 +- tests/integration/01-scriptdebugger.ts | 25 +- tests/integration/codesnippet.ts | 36 +- .../codesnippetfromselectedcells.ts | 72 +- tests/integration/launcher.ts | 14 +- tests/integration/lsp.ts | 2 +- tests/integration/pipeline.ts | 130 +- tests/integration/pythoneditor.ts | 48 +- tests/integration/reditor.ts | 8 +- tests/integration/submitnotebookbutton.ts | 18 +- .../matches-complex-pipeline-snapshot.1.snap | 498 +- .../matches-empty-pipeline-snapshot.1.snap | 36 +- .../matches-simple-pipeline-snapshot.1.snap | 158 +- tests/support/commands.ts | 103 +- tests/tsconfig.json | 8 +- tests/utils/snapshots/plugin.ts | 16 +- testutils/babel.config.js | 17 - testutils/jest.config.js | 57 +- testutils/jest.setup.js | 5 +- testutils/transform.js | 18 - yarn.lock | 11713 +++++++++------- 192 files changed, 9683 insertions(+), 11988 deletions(-) create mode 100644 elyra/pipeline/kfp/PipelineConf.py create mode 100644 elyra/templates/kubeflow/v2/generic_component_definition_template.jinja2 create mode 100644 elyra/templates/kubeflow/v2/python_dsl_template.jinja2 create mode 100644 labextensions/elyra_code_snippet_extension/__init__.py create mode 100644 labextensions/elyra_metadata_common/__init__.py create mode 100644 labextensions/elyra_metadata_extension/__init__.py create mode 100644 labextensions/elyra_pipeline_editor_extension/__init__.py create mode 100644 labextensions/elyra_python_editor_extension/__init__.py create mode 100644 labextensions/elyra_script_debugger_extension/__init__.py create mode 100644 labextensions/elyra_script_editor/__init__.py create mode 100644 labextensions/elyra_services/__init__.py create mode 100644 labextensions/elyra_theme_extension/__init__.py create mode 100644 labextensions/elyra_ui_components/__init__.py create mode 100644 packages/code-snippet/install.json create mode 100644 packages/code-snippet/setup.py create mode 100644 packages/metadata-common/install.json create mode 100644 packages/metadata-common/setup.py create mode 100644 packages/metadata/install.json create mode 100644 packages/metadata/setup.py create mode 100644 packages/pipeline-editor/install.json delete mode 100644 packages/pipeline-editor/schema/src/ComponentCatalogsWidget.tsx delete mode 100644 packages/pipeline-editor/schema/src/EmptyPipelineContent.tsx delete mode 100644 packages/pipeline-editor/schema/src/FileSubmissionDialog.tsx delete mode 100644 packages/pipeline-editor/schema/src/ParameterInputForm.tsx delete mode 100644 packages/pipeline-editor/schema/src/PipelineEditorWidget.tsx delete mode 100644 packages/pipeline-editor/schema/src/PipelineExportDialog.tsx delete mode 100644 packages/pipeline-editor/schema/src/PipelineService.tsx delete mode 100644 packages/pipeline-editor/schema/src/PipelineSubmissionDialog.tsx delete mode 100644 packages/pipeline-editor/schema/src/RuntimeConfigSelect.tsx delete mode 100644 packages/pipeline-editor/schema/src/RuntimeImagesWidget.tsx delete mode 100644 packages/pipeline-editor/schema/src/RuntimesWidget.tsx delete mode 100644 packages/pipeline-editor/schema/src/SubmitFileButtonExtension.tsx delete mode 100644 packages/pipeline-editor/schema/src/dialogs.tsx delete mode 100644 packages/pipeline-editor/schema/src/formDialogWidget.ts delete mode 100644 packages/pipeline-editor/schema/src/index.ts delete mode 100644 packages/pipeline-editor/schema/src/pipeline-hooks.ts delete mode 100644 packages/pipeline-editor/schema/src/runtime-utils.ts delete mode 100644 packages/pipeline-editor/schema/src/test/pipeline-hooks.spec.ts delete mode 100644 packages/pipeline-editor/schema/src/test/pipeline-service.spec.ts delete mode 100644 packages/pipeline-editor/schema/src/theme.tsx delete mode 100644 packages/pipeline-editor/schema/src/utils.ts create mode 100644 packages/pipeline-editor/setup.py create mode 100644 packages/python-editor/install.json create mode 100644 packages/python-editor/setup.py create mode 100644 packages/script-debugger/install.json create mode 100644 packages/script-debugger/setup.py create mode 100644 packages/script-editor/install.json create mode 100644 packages/script-editor/setup.py create mode 100644 packages/services/install.json create mode 100644 packages/services/setup.py create mode 100644 packages/theme/install.json create mode 100644 packages/theme/setup.py create mode 100644 packages/ui-components/install.json create mode 100644 packages/ui-components/setup.py delete mode 100644 testutils/babel.config.js delete mode 100644 testutils/transform.js diff --git a/.eslintrc.json b/.eslintrc.json index d6f5bf646..42c8f3e3e 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -73,5 +73,17 @@ "react": { "version": "detect" } - } + }, + "ignorePatterns": [ + "packages/pipeline-editor/elyra_pipeline_editor_extension/*", + "packages/theme/elyra_theme_extension/*", + "packages/script-debugger/elyra_script_debugger_extension/*", + "packages/script-editor/elyra_script_editor/*", + "packages/code-snippet/elyra_code_snippet_extension/*", + "packages/metadata/elyra_metadata_extension/*", + "packages/metadata-common/elyra_metadata_common/*", + "packages/python-editor/elyra_python_editor_extension/*", + "packages/services/elyra_services/*", + "packages/ui-components/elyra_ui_components/*" + ] } diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 591651a10..d0277b98f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,38 +19,43 @@ on: pull_request: # all branches schedule: # once a day at 3 am (PST) (10 am (UTC)) - - cron: '0 10 * * *' + - cron: '0 10 * * *' env: FORCE_COLOR: true + NODE_VERSION: 20.11.0 + YARN_VERSION: 3.5.0 + PYTHON_VERSION: 3.11 jobs: prepare-yarn-cache: name: Prepare Cache runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: - node-version: "20.11.0" - - run: npm install -g yarn@3.5.0 - - uses: actions/cache@v3 + node-version: ${{ env.NODE_VERSION }} + - uses: actions/cache@v4 with: path: | node_modules */*/node_modules /home/runner/.cache/Cypress key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - - name: Install + - name: Install Yarn ${{ env.YARN_VERSION }} run: | + corepack prepare yarn@${{ env.YARN_VERSION }} --activate + yarn set version ${{ env.YARN_VERSION }} yarn --version - yarn install && tsc -v - # --frozen-lockfile + - name: Install dependencies + run: | + yarn install lint-server: name: Lint Server runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Lint run: make lint-server @@ -59,18 +64,24 @@ jobs: needs: prepare-yarn-cache runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: - node-version: "*" - - uses: actions/cache@v3 + node-version: ${{ env.NODE_VERSION }} + - uses: actions/cache@v4 with: path: | node_modules */*/node_modules key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - - name: Install - run: make yarn-install + - name: Install Yarn ${{ env.YARN_VERSION }} + run: | + corepack prepare yarn@${{ env.YARN_VERSION }} --activate + yarn set version ${{ env.YARN_VERSION }} + yarn --version + - name: Install dependencies + run: | + make yarn-install - name: Lint run: make eslint-check-ui - name: Check format @@ -81,10 +92,10 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8, 3.9, "3.10"] + python-version: [3.8, 3.9, '3.10', '3.11'] steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install @@ -110,19 +121,24 @@ jobs: needs: prepare-yarn-cache runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: - node-version: "*" - - uses: actions/setup-python@v4 + node-version: ${{ env.NODE_VERSION }} + - uses: actions/setup-python@v5 with: - python-version: "3.10" - - uses: actions/cache@v3 + python-version: ${{ env.PYTHON_VERSION }} + - uses: actions/cache@v4 with: path: | node_modules */*/node_modules key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + - name: Install Yarn ${{ env.YARN_VERSION }} + run: | + corepack prepare yarn@${{ env.YARN_VERSION }} --activate + yarn set version ${{ env.YARN_VERSION }} + yarn --version - name: Build run: | make build-dependencies @@ -138,20 +154,25 @@ jobs: needs: prepare-yarn-cache runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: - node-version: "*" - - uses: actions/setup-python@v4 + node-version: ${{ env.NODE_VERSION }} + - uses: actions/setup-python@v5 with: - python-version: "3.10" - - uses: actions/cache@v3 + python-version: ${{ env.PYTHON_VERSION }} + - uses: actions/cache@v4 with: path: | node_modules */*/node_modules /home/runner/.cache/Cypress key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + - name: Install Yarn ${{ env.YARN_VERSION }} + run: | + corepack prepare yarn@${{ env.YARN_VERSION }} --activate + yarn set version ${{ env.YARN_VERSION }} + yarn --version - name: Build run: | make build-dependencies @@ -164,9 +185,10 @@ jobs: - name: Cypress run: make test-integration - name: Collect logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: failure() with: + name: elyra_test_artifacts path: | ${{ github.workspace }}/build/cypress-tests/*.log ${{ github.workspace }}/build/cypress-tests/screenshots//**/* @@ -177,7 +199,7 @@ jobs: name: Test documentation build runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build run: make docs @@ -186,17 +208,50 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8, 3.9, "3.10"] + python-version: [3.8, 3.9, '3.10', '3.11'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Validate image environment run: make PYTHON_VERSION=${{ matrix.python-version }} elyra-image-env - validate-images: name: Validate Images runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Validate runtime images run: make REMOVE_RUNTIME_IMAGE=1 validate-runtime-images + + upload-artifacts: + name: Upload Artifacts + needs: prepare-yarn-cache + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - uses: actions/cache@v4 + with: + path: | + node_modules + */*/node_modules + /home/runner/.cache/Cypress + key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} + - name: Install Yarn ${{ env.YARN_VERSION }} + run: | + corepack prepare yarn@${{ env.YARN_VERSION }} --activate + yarn set version ${{ env.YARN_VERSION }} + yarn --version + - name: Build + run: | + make clean install + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: elyra_build_artifacts + path: | + ${{ github.workspace }}/dist diff --git a/.gitignore b/.gitignore index 6a73ca5e2..73f87233d 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,7 @@ htmlcov/ nosetests.xml coverage.xml *,cover +**/coverage/ # Translations *.mo @@ -92,3 +93,5 @@ node_modules .vscode/ .yarn/ + +labextensions/**/labextension/* \ No newline at end of file diff --git a/.prettierrc b/.prettierrc index 544138be4..4c112c074 100644 --- a/.prettierrc +++ b/.prettierrc @@ -1,3 +1,4 @@ { - "singleQuote": true + "singleQuote": true, + "trailingComma" : "none" } diff --git a/.yarnrc.yml b/.yarnrc.yml index 436b801f4..fe1125f54 100644 --- a/.yarnrc.yml +++ b/.yarnrc.yml @@ -1,5 +1,3 @@ enableImmutableInstalls: false nodeLinker: node-modules - -yarnPath: .yarn/releases/yarn-3.5.0.cjs diff --git a/Makefile b/Makefile index 9a68fd52c..38a86f0bc 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ # limitations under the License. # -.PHONY: help purge uninstall-src uninstall clean +.PHONY: help purge uninstall clean .PHONY: lint-dependencies lint-server black-format prettier-check-ui eslint-check-ui prettier-ui eslint-ui lint-ui lint .PHONY: dev-link dev-unlink .PHONY: build-dependencies dev-dependencies yarn-install build-ui package-ui package-ui-dev @@ -35,7 +35,7 @@ CONTAINER_OUTPUT_OPTION?=--output=type=docker # Python execs PYTHON?=python3 PYTHON_PIP=$(PYTHON) -m pip -PYTHON_VERSION?=3.9 +PYTHON_VERSION?=3.11 CONDA_ACTIVATE = source $$(conda info --base)/etc/profile.d/conda.sh ; conda activate @@ -75,24 +75,7 @@ purge: rm -rf $$(find . -name .pytest_cache) rm -rf $(yarn cache dir) -uninstall-src: # Uninstalls source extensions if they're still installed - - jupyter labextension unlink --no-build @elyra/services - - jupyter labextension unlink --no-build @elyra/ui-components - - jupyter labextension unlink --no-build @elyra/metadata-common - - jupyter labextension unlink --no-build @elyra/script-editor - - jupyter labextension uninstall --no-build @elyra/theme-extension - - jupyter labextension uninstall --no-build @elyra/code-snippet-extension - - jupyter labextension uninstall --no-build @elyra/metadata-extension - - jupyter labextension uninstall --no-build @elyra/pipeline-editor-extension - - jupyter labextension uninstall --no-build @elyra/python-editor-extension - - jupyter labextension uninstall --no-build @elyra/r-editor-extension - - jupyter labextension uninstall --no-build @elyra/scala-editor-extension - - jupyter labextension uninstall --no-build @elyra/code-viewer-extension - - jupyter labextension uninstall --no-build @elyra/script-debugger-extension - - jupyter labextension unlink --no-build @elyra/pipeline-services - - jupyter labextension unlink --no-build @elyra/pipeline-editor - -uninstall: uninstall-src +uninstall: $(PYTHON_PIP) uninstall -y jupyterlab-git $(PYTHON_PIP) uninstall -y nbdime $(PYTHON_PIP) uninstall -y jupyter-lsp @@ -179,7 +162,7 @@ uninstall-server-package: @$(PYTHON_PIP) uninstall elyra -y install-server-package: uninstall-server-package - $(PYTHON_PIP) install --upgrade --upgrade-strategy $(UPGRADE_STRATEGY) "$(shell find dist -name "elyra-*-py3-none-any.whl")[kfp-tekton]" + $(PYTHON_PIP) install --upgrade --upgrade-strategy $(UPGRADE_STRATEGY) "$(shell find dist -name "elyra-*-py3-none-any.whl")" install-server: build-dependencies lint-server build-server install-server-package ## Build and install backend @@ -189,7 +172,7 @@ install-all: package-ui install-server install-examples install-gitlab-dependenc install-dev: package-ui-dev install-server install-examples install-gitlab-dependency check-install -install-examples: ## Install example pipeline components +install-examples: ## Install example pipeline components # install Kubeflow Pipelines example components # -> https://github.com/elyra-ai/examples/tree/main/component-catalog-connectors/kfp-example-components-connector - $(PYTHON_PIP) install --upgrade elyra-examples-kfp-catalog diff --git a/build_requirements.txt b/build_requirements.txt index 2307f89b6..d16f92197 100644 --- a/build_requirements.txt +++ b/build_requirements.txt @@ -1,3 +1,3 @@ -jupyterlab>=4.0,<4.1 +jupyterlab==4.2.5 jupyter-packaging>=0.10 build diff --git a/docs/requirements.txt b/docs/requirements.txt index c92959e15..9ae7be90e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ sphinx>=1.8 -sphinx_rtd_theme +sphinx_rtd_theme==2.0.0 recommonmark>=0.6.0 sphinx-markdown-tables>=0.0.16 sphinx-version-warning diff --git a/elyra/metadata/schemas/airflow.json b/elyra/metadata/schemas/airflow.json index cbf967672..8ae7f329c 100644 --- a/elyra/metadata/schemas/airflow.json +++ b/elyra/metadata/schemas/airflow.json @@ -107,7 +107,7 @@ "description": "Token that has permission to push to the DAG repository", "type": "string", "uihints": { - "ui:field": "password", + "ui:field": "@elyra/metadata-extension:plugin.password", "category": "Apache Airflow" } }, @@ -161,7 +161,7 @@ "description": "Kubernetes secret that's defined in the specified user namespace, containing the Cloud Object Storage username and password. This property is required for authentication type KUBERNETES_SECRET.", "type": "string", "uihints": { - "ui:field": "password", + "ui:field": "@elyra/metadata-extension:plugin.password", "category": "Cloud Object Storage" } }, @@ -179,7 +179,7 @@ "type": "string", "minLength": 8, "uihints": { - "ui:field": "password", + "ui:field": "@elyra/metadata-extension:plugin.password", "category": "Cloud Object Storage" } }, @@ -193,7 +193,7 @@ "pattern": "^[^ \t]+([ \t]+[^ \t]+)*$" }, "uihints": { - "ui:field": "tags" + "ui:field": "@elyra/metadata-extension:plugin.tags" } } }, diff --git a/elyra/metadata/schemas/code-snippet.json b/elyra/metadata/schemas/code-snippet.json index aeeaa9a8b..c43e64caa 100644 --- a/elyra/metadata/schemas/code-snippet.json +++ b/elyra/metadata/schemas/code-snippet.json @@ -42,7 +42,7 @@ "pattern": "^[^ \t]+([ \t]+[^ \t]+)*$" }, "uihints": { - "ui:field": "tags" + "ui:field": "@elyra/metadata-extension:plugin.tags" } }, "language": { @@ -50,7 +50,7 @@ "description": "Code snippet implementation language", "type": "string", "uihints": { - "ui:field": "dropdown", + "ui:field": "@elyra/metadata-extension:plugin.dropdown", "default_choices": ["Python", "Java", "R", "Scala", "Markdown"], "category": "Source" }, @@ -64,7 +64,7 @@ "type": "string" }, "uihints": { - "ui:field": "code", + "ui:field": "@elyra/metadata-extension:plugin.code", "category": "Source" } } diff --git a/elyra/metadata/schemas/kfp.json b/elyra/metadata/schemas/kfp.json index bafe8b864..701549b44 100644 --- a/elyra/metadata/schemas/kfp.json +++ b/elyra/metadata/schemas/kfp.json @@ -106,7 +106,7 @@ "description": "Password or token to be used for authentication. This property is required for all authentication types, except NO_AUTHENTICATION and KUBERNETES_SERVICE_ACCOUNT_TOKEN.", "type": "string", "uihints": { - "ui:field": "password", + "ui:field": "@elyra/metadata-extension:plugin.password", "category": "Kubeflow Pipelines" } }, @@ -160,7 +160,7 @@ "description": "Kubernetes secret that's defined in the specified user namespace, containing the Cloud Object Storage username and password. This property is required for authentication type KUBERNETES_SECRET.", "type": "string", "uihints": { - "ui:field": "password", + "ui:field": "@elyra/metadata-extension:plugin.password", "category": "Cloud Object Storage" } }, @@ -178,7 +178,7 @@ "type": "string", "minLength": 8, "uihints": { - "ui:field": "password", + "ui:field": "@elyra/metadata-extension:plugin.password", "category": "Cloud Object Storage" } }, @@ -192,7 +192,7 @@ "pattern": "^[^ \t]+([ \t]+[^ \t]+)*$" }, "uihints": { - "ui:field": "tags" + "ui:field": "@elyra/metadata-extension:plugin.tags" } } }, diff --git a/elyra/metadata/schemas/metadata-test.json b/elyra/metadata/schemas/metadata-test.json index 084fd0420..7571df9ee 100644 --- a/elyra/metadata/schemas/metadata-test.json +++ b/elyra/metadata/schemas/metadata-test.json @@ -96,7 +96,7 @@ "type": "string", "enum": ["elyra", "rocks"], "uihints": { - "ui:field": "dropdown", + "ui:field": "@elyra/metadata-extension:plugin.dropdown", "default_choices": ["elyra"] } }, @@ -108,7 +108,7 @@ "maxItems": 10, "uniqueItems": true, "uihints": { - "ui:field": "code" + "ui:field": "@elyra/metadata-extension:plugin.code" } }, "object_test": { diff --git a/elyra/metadata/schemas/metadata-test2.json b/elyra/metadata/schemas/metadata-test2.json index dd6293624..5c01718f7 100644 --- a/elyra/metadata/schemas/metadata-test2.json +++ b/elyra/metadata/schemas/metadata-test2.json @@ -86,7 +86,7 @@ "type": "string", "enum": ["elyra", "rocks"], "uihints": { - "ui:field": "dropdown", + "ui:field": "@elyra/metadata-extension:plugin.dropdown", "default_choices": ["elyra"] } }, @@ -98,7 +98,7 @@ "maxItems": 10, "uniqueItems": true, "uihints": { - "ui:field": "code" + "ui:field": "@elyra/metadata-extension:plugin.code" } }, "object_test": { diff --git a/elyra/metadata/schemas/runtime-image.json b/elyra/metadata/schemas/runtime-image.json index 49c3bba00..93b997dcd 100644 --- a/elyra/metadata/schemas/runtime-image.json +++ b/elyra/metadata/schemas/runtime-image.json @@ -42,7 +42,7 @@ "pattern": "^[^ \t]+([ \t]+[^ \t]+)*$" }, "uihints": { - "ui:field": "tags" + "ui:field": "@elyra/metadata-extension:plugin.tags" } }, "image_name": { @@ -72,7 +72,7 @@ "maxLength": 253, "uihints": { "category": "Source", - "ui:field": "password" + "ui:field": "@elyra/metadata-extension:plugin.password" } } }, diff --git a/elyra/metadata/schemas/url-catalog.json b/elyra/metadata/schemas/url-catalog.json index 6532bf948..dcdd4d93c 100644 --- a/elyra/metadata/schemas/url-catalog.json +++ b/elyra/metadata/schemas/url-catalog.json @@ -84,7 +84,7 @@ "type": "string", "minLength": 1, "uihints": { - "ui:field": "password", + "ui:field": "@elyra/metadata-extension:plugin.password", "category": "Source credentials" } } diff --git a/elyra/pipeline/airflow/package_catalog_connector/airflow-package-catalog.json b/elyra/pipeline/airflow/package_catalog_connector/airflow-package-catalog.json index e2d89c96f..504a490ce 100644 --- a/elyra/pipeline/airflow/package_catalog_connector/airflow-package-catalog.json +++ b/elyra/pipeline/airflow/package_catalog_connector/airflow-package-catalog.json @@ -87,7 +87,7 @@ "type": "string", "minLength": 1, "uihints": { - "ui:field": "password", + "ui:field": "@elyra/metadata-extension:plugin.password", "category": "Source credentials" } } diff --git a/elyra/pipeline/airflow/provider_package_catalog_connector/airflow-provider-package-catalog.json b/elyra/pipeline/airflow/provider_package_catalog_connector/airflow-provider-package-catalog.json index 34cbbc92c..a57dc74da 100644 --- a/elyra/pipeline/airflow/provider_package_catalog_connector/airflow-provider-package-catalog.json +++ b/elyra/pipeline/airflow/provider_package_catalog_connector/airflow-provider-package-catalog.json @@ -79,7 +79,7 @@ "type": "string", "minLength": 1, "uihints": { - "ui:field": "password", + "ui:field": "@elyra/metadata-extension:plugin.password", "category": "Source credentials" } } diff --git a/elyra/pipeline/kfp/PipelineConf.py b/elyra/pipeline/kfp/PipelineConf.py new file mode 100644 index 000000000..ddf3fea38 --- /dev/null +++ b/elyra/pipeline/kfp/PipelineConf.py @@ -0,0 +1,158 @@ +from typing import Union + +from kubernetes.client.models import V1PodDNSConfig + + +class PipelineConf: + """PipelineConf contains pipeline level settings.""" + + def __init__(self): + self.image_pull_secrets = [] + self.timeout = 0 + self.ttl_seconds_after_finished = -1 + self._pod_disruption_budget_min_available = None + self.op_transformers = [] + self.default_pod_node_selector = {} + self.image_pull_policy = None + self.parallelism = None + self._data_passing_method = None + self.dns_config = None + + def set_image_pull_secrets(self, image_pull_secrets): + """Configures the pipeline level imagepullsecret. + + Args: + image_pull_secrets: a list of Kubernetes V1LocalObjectReference For + detailed description, check Kubernetes V1LocalObjectReference definition + https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V1LocalObjectReference.md + """ + self.image_pull_secrets = image_pull_secrets + return self + + def set_timeout(self, seconds: int): + """Configures the pipeline level timeout. + + Args: + seconds: number of seconds for timeout + """ + self.timeout = seconds + return self + + def set_parallelism(self, max_num_pods: int): + """Configures the max number of total parallel pods that can execute at + the same time in a workflow. + + Args: + max_num_pods: max number of total parallel pods. + """ + if max_num_pods < 1: + raise ValueError("Pipeline max_num_pods set to < 1, allowed values are > 0") + + self.parallelism = max_num_pods + return self + + def set_ttl_seconds_after_finished(self, seconds: int): + """Configures the ttl after the pipeline has finished. + + Args: + seconds: number of seconds for the workflow to be garbage collected after + it is finished. + """ + self.ttl_seconds_after_finished = seconds + return self + + def set_pod_disruption_budget(self, min_available: Union[int, str]): + """PodDisruptionBudget holds the number of concurrent disruptions that + you allow for pipeline Pods. + + Args: + min_available (Union[int, str]): An eviction is allowed if at least + "minAvailable" pods selected by "selector" will still be available after + the eviction, i.e. even in the absence of the evicted pod. So for + example you can prevent all voluntary evictions by specifying "100%". + "minAvailable" can be either an absolute number or a percentage. + """ + self._pod_disruption_budget_min_available = min_available + return self + + def set_default_pod_node_selector(self, label_name: str, value: str): + """Add a constraint for nodeSelector for a pipeline. + + Each constraint is a key-value pair label. + + For the container to be eligible to run on a node, the node must have each + of the constraints appeared as labels. + + Args: + label_name: The name of the constraint label. + value: The value of the constraint label. + """ + self.default_pod_node_selector[label_name] = value + return self + + def set_image_pull_policy(self, policy: str): + """Configures the default image pull policy. + + Args: + policy: the pull policy, has to be one of: Always, Never, IfNotPresent. + For more info: + https://github.com/kubernetes-client/python/blob/10a7f95435c0b94a6d949ba98375f8cc85a70e5a/kubernetes/docs/V1Container.md + """ + self.image_pull_policy = policy + return self + + def add_op_transformer(self, transformer): + """Configures the op_transformers which will be applied to all ops in + the pipeline. The ops can be ResourceOp, VolumeOp, or ContainerOp. + + Args: + transformer: A function that takes a kfp Op as input and returns a kfp Op + """ + self.op_transformers.append(transformer) + + def set_dns_config(self, dns_config: V1PodDNSConfig): + """Set the dnsConfig to be given to each pod. + + Args: + dns_config: Kubernetes V1PodDNSConfig For detailed description, check + Kubernetes V1PodDNSConfig definition + https://github.com/kubernetes-client/python/blob/master/kubernetes/docs/V1PodDNSConfig.md + + Example: + :: + + import kfp + from kubernetes.client.models import V1PodDNSConfig, V1PodDNSConfigOption + pipeline_conf = kfp.dsl.PipelineConf() + pipeline_conf.set_dns_config(dns_config=V1PodDNSConfig( + nameservers=["1.2.3.4"], + options=[V1PodDNSConfigOption(name="ndots", value="2")], + )) + """ + self.dns_config = dns_config + + @property + def data_passing_method(self): + return self._data_passing_method + + @data_passing_method.setter + def data_passing_method(self, value): + """Sets the object representing the method used for intermediate data + passing. + + Example: + :: + + from kfp.dsl import PipelineConf, data_passing_methods + from kubernetes.client.models import V1Volume, V1PersistentVolumeClaimVolumeSource + pipeline_conf = PipelineConf() + pipeline_conf.data_passing_method = + data_passing_methods.KubernetesVolume( + volume=V1Volume( + name='data', + persistent_volume_claim=V1PersistentVolumeClaimVolumeSource('data-volume'), + ), + path_prefix='artifact_data/', + ) + """ + self._data_passing_method = value diff --git a/elyra/pipeline/kfp/kfp_authentication.py b/elyra/pipeline/kfp/kfp_authentication.py index 3ca44519a..84b8b15e4 100644 --- a/elyra/pipeline/kfp/kfp_authentication.py +++ b/elyra/pipeline/kfp/kfp_authentication.py @@ -27,9 +27,9 @@ from typing import Tuple from urllib.parse import urlsplit -from kfp.auth import KF_PIPELINES_SA_TOKEN_ENV -from kfp.auth import KF_PIPELINES_SA_TOKEN_PATH -from kfp.auth import ServiceAccountTokenVolumeCredentials +from kfp.client import KF_PIPELINES_SA_TOKEN_ENV +from kfp.client import KF_PIPELINES_SA_TOKEN_PATH +from kfp.client import ServiceAccountTokenVolumeCredentials import requests @@ -230,6 +230,7 @@ def authenticate( """ kf_url = urlsplit(api_endpoint)._replace(path="").geturl() + kf_pipelines_ssl_sa_cert = os.getenv("PIPELINES_SSL_SA_CERTS", None) # return data structure for successful requests auth_info = { @@ -239,6 +240,7 @@ def authenticate( "cookies": None, # passed to KFP SDK client as "cookies" param value "credentials": None, # passed to KFP SDK client as "credentials" param value "existing_token": None, # passed to KFP SDK client as "existing_token" param value + "ssl_ca_cert": kf_pipelines_ssl_sa_cert, # passed to KFP SDK Client as "ssl_ca_cert" param value } try: diff --git a/elyra/pipeline/kfp/processor_kfp.py b/elyra/pipeline/kfp/processor_kfp.py index a0d32da53..33208ce83 100644 --- a/elyra/pipeline/kfp/processor_kfp.py +++ b/elyra/pipeline/kfp/processor_kfp.py @@ -39,19 +39,11 @@ from kfp import Client as ArgoClient from kfp import compiler as kfp_argo_compiler from kfp import components as components -from kfp.dsl import PipelineConf -from kfp.dsl import RUN_ID_PLACEHOLDER from kubernetes import client as k8s_client from traitlets import default from traitlets import Unicode -try: - from kfp_tekton import compiler as kfp_tekton_compiler - from kfp_tekton import TektonClient -except ImportError: - # We may not have kfp-tekton available and that's okay! - kfp_tekton_compiler = None - TektonClient = None +RUN_ID_PLACEHOLDER = "{{workflow.uid}}" from elyra._version import __version__ from elyra.metadata.schemaspaces import RuntimeImages @@ -61,6 +53,7 @@ from elyra.pipeline.kfp.kfp_authentication import AuthenticationError from elyra.pipeline.kfp.kfp_authentication import KFPAuthenticator from elyra.pipeline.kfp.kfp_properties import KfpPipelineParameter +from elyra.pipeline.kfp.PipelineConf import PipelineConf from elyra.pipeline.pipeline import Operation from elyra.pipeline.pipeline import Pipeline from elyra.pipeline.processor import PipelineProcessor @@ -173,11 +166,6 @@ def process(self, pipeline): api_password = runtime_configuration.metadata.get("api_password") user_namespace = runtime_configuration.metadata.get("user_namespace") workflow_engine = WorkflowEngineType.get_instance_by_value(runtime_configuration.metadata.get("engine", "argo")) - if workflow_engine == WorkflowEngineType.TEKTON and not TektonClient: - raise ValueError( - "Python package `kfp-tekton` is not installed. " - "Please install using `elyra[kfp-tekton]` to use Tekton engine." - ) # unpack Cloud Object Storage configs cos_endpoint = runtime_configuration.metadata["cos_endpoint"] @@ -206,22 +194,14 @@ def process(self, pipeline): # Create Kubeflow Client ############# try: - if workflow_engine == WorkflowEngineType.TEKTON: - client = TektonClient( - host=api_endpoint, - cookies=auth_info.get("cookies", None), - credentials=auth_info.get("credentials", None), - existing_token=auth_info.get("existing_token", None), - namespace=user_namespace, - ) - else: - client = ArgoClient( - host=api_endpoint, - cookies=auth_info.get("cookies", None), - credentials=auth_info.get("credentials", None), - existing_token=auth_info.get("existing_token", None), - namespace=user_namespace, - ) + client = ArgoClient( + host=api_endpoint, + cookies=auth_info.get("cookies", None), + credentials=auth_info.get("credentials", None), + existing_token=auth_info.get("existing_token", None), + namespace=user_namespace, + ssl_ca_cert=auth_info.get("ssl_ca_cert", None), + ) except Exception as ex: # a common cause of these errors is forgetting to include `/pipeline` or including it with an 's' api_endpoint_obj = urlsplit(api_endpoint) @@ -275,7 +255,7 @@ def process(self, pipeline): with tempfile.TemporaryDirectory() as temp_dir: self.log.debug(f"Created temporary directory at: {temp_dir}") - pipeline_path = os.path.join(temp_dir, f"{pipeline_name}.tar.gz") + pipeline_path = os.path.join(temp_dir, f"{pipeline_name}.yaml") ############# # Get Pipeline ID @@ -351,11 +331,15 @@ def process(self, pipeline): ) # extract the ID of the pipeline we created - pipeline_id = kfp_pipeline.id + pipeline_id = kfp_pipeline.pipeline_id # the initial "pipeline version" has the same id as the pipeline itself - version_id = pipeline_id - + version_details = client.list_pipeline_versions(pipeline_id=pipeline_id) + version_list = version_details.pipeline_versions + if isinstance(version_list, list): + version_id = version_list[0].pipeline_version_id + else: + version_id = None # CASE 2: pipeline already exists else: # upload the "pipeline version" @@ -366,7 +350,7 @@ def process(self, pipeline): ) # extract the id of the "pipeline version" that was created - version_id = kfp_pipeline.id + version_id = kfp_pipeline.pipeline_version_id except Exception as ex: # a common cause of these errors is forgetting to include `/pipeline` or including it with an 's' @@ -416,7 +400,10 @@ def process(self, pipeline): # create pipeline run (or specified pipeline version) run = client.run_pipeline( - experiment_id=experiment.id, job_name=job_name, pipeline_id=pipeline_id, version_id=version_id + experiment_id=experiment.experiment_id, + job_name=job_name, + pipeline_id=pipeline_id, + version_id=version_id, ) except Exception as ex: @@ -435,7 +422,7 @@ def process(self, pipeline): self.log_pipeline_info( pipeline_name, - f"pipeline submitted: {public_api_endpoint}/#/runs/details/{run.id}", + f"pipeline submitted: {public_api_endpoint}/{experiment.experiment_id}/runs/{run.run_id}", duration=time.time() - t0, ) @@ -450,8 +437,8 @@ def process(self, pipeline): object_storage_path = None return KfpPipelineProcessorResponse( - run_id=run.id, - run_url=f"{public_api_endpoint}/#/runs/details/{run.id}", + run_id=run.run_id, + run_url=f"{public_api_endpoint}/{experiment.experiment_id}/runs/{run.run_id}", object_storage_url=object_storage_url, object_storage_path=object_storage_path, ) @@ -494,8 +481,6 @@ def export( ) workflow_engine = WorkflowEngineType.get_instance_by_value(runtime_configuration.metadata.get("engine", "argo")) - if workflow_engine == WorkflowEngineType.TEKTON and not TektonClient: - raise ValueError("kfp-tekton not installed. Please install using elyra[kfp-tekton] to use Tekton engine.") if Path(absolute_pipeline_export_path).exists() and not overwrite: raise ValueError("File " + absolute_pipeline_export_path + " already exists.") @@ -565,7 +550,7 @@ def _generate_pipeline_dsl( code_generation_options = {} # Load Kubeflow Pipelines Python DSL template - loader = PackageLoader("elyra", "templates/kubeflow/v1") + loader = PackageLoader("elyra", "templates/kubeflow/v2") template_env = Environment(loader=loader) # Add filter that produces a Python-safe variable name template_env.filters["python_safe"] = lambda x: re.sub(r"[" + re.escape(string.punctuation) + "\\s]", "_", x) @@ -668,12 +653,7 @@ def _compile_pipeline_dsl( # in the generated Python DSL "generated_pipeline" pipeline_function = getattr(mod, "generated_pipeline") # compile the DSL - if workflow_engine == WorkflowEngineType.TEKTON: - kfp_tekton_compiler.TektonCompiler().compile( - pipeline_function, output_file, pipeline_conf=pipeline_conf - ) - else: - kfp_argo_compiler.Compiler().compile(pipeline_function, output_file, pipeline_conf=pipeline_conf) + kfp_argo_compiler.Compiler().compile(pipeline_function, output_file) except Exception as ex: raise RuntimeError( f"Failed to compile pipeline with workflow_engine '{workflow_engine.value}' to '{output_file}'" @@ -729,7 +709,7 @@ def _generate_workflow_tasks( pipeline.pipeline_properties.get(pipeline_constants.COS_OBJECT_PREFIX), pipeline_instance_id ) # - load the generic component definition template - template_env = Environment(loader=PackageLoader("elyra", "templates/kubeflow/v1")) + template_env = Environment(loader=PackageLoader("elyra", "templates/kubeflow/v2")) generic_component_template = template_env.get_template("generic_component_definition_template.jinja2") # Add filter that escapes the " character in strings template_env.filters["string_delimiter_safe"] = lambda string: re.sub('"', '\\\\\\\\"', string) @@ -843,6 +823,11 @@ def _generate_workflow_tasks( "size": operation.memory, "units": "G", } + workflow_task["task_modifiers"]["cpu_limit"] = operation.cpu_limit + workflow_task["task_modifiers"]["memory_limit"] = { + "size": operation.memory_limit, + "units": "G", + } gpu_vendor = "nvidia.com/gpu" if operation.gpu_vendor: gpu_vendor = operation.gpu_vendor @@ -920,13 +905,13 @@ def _generate_workflow_tasks( # Identify task inputs and outputs using the component spec # If no data type was specified, string is assumed factory_function = components.load_component_from_text(component.definition) - for input in factory_function.component_spec.inputs or []: - sanitized_input_name = self._sanitize_param_name(input.name) + for input_key, input_value in (factory_function.component_spec.inputs or {}).items(): + sanitized_input_name = self._sanitize_param_name(input_key) workflow_task["task_inputs"][sanitized_input_name] = { "value": None, "task_output_reference": None, "pipeline_parameter_reference": None, - "data_type": (input.type or "string").lower(), + "data_type": (input_value.type or "string").lower(), } # Determine whether the value needs to be rendered in quotes # in the generated DSL code. For example "my name" (string), and 34 (integer). @@ -938,9 +923,9 @@ def _generate_workflow_tasks( "bool", ] - for output in factory_function.component_spec.outputs or []: - workflow_task["task_outputs"][self._sanitize_param_name(output.name)] = { - "data_type": output.type, + for output_key, output_value in (factory_function.component_spec.outputs or {}).items(): + workflow_task["task_outputs"][self._sanitize_param_name(output_key)] = { + "data_type": output_value.type, } # Iterate over component properties and assign values to @@ -1073,7 +1058,7 @@ def _compose_container_command_args( account whether the container will run in a CRI-O environment. """ elyra_github_org = os.getenv("ELYRA_GITHUB_ORG", "elyra-ai") - elyra_github_branch = os.getenv("ELYRA_GITHUB_BRANCH", "main" if "dev" in __version__ else "v" + __version__) + elyra_github_branch = os.getenv("ELYRA_GITHUB_BRANCH", "main" if "dev" in __version__ else f"v{__version__}") elyra_bootstrap_script_url = os.getenv( "ELYRA_BOOTSTRAP_SCRIPT_URL", f"https://raw.githubusercontent.com/{elyra_github_org}/elyra/{elyra_github_branch}/elyra/kfp/bootstrapper.py", # noqa E501 diff --git a/elyra/pipeline/pipeline_definition.py b/elyra/pipeline/pipeline_definition.py index c49c0c29b..6c90c810f 100644 --- a/elyra/pipeline/pipeline_definition.py +++ b/elyra/pipeline/pipeline_definition.py @@ -265,6 +265,9 @@ def convert_pipeline_parameters(self, runtime_type_name: str) -> None: if parameter_class is None: return None # runtime type does not support parameters, skip + if not ElyraProperty.subclass_exists_for_property(parameter_class.property_id): + ElyraProperty.build_property_map() + # Convert pipeline parameters to runtime-specific instances converted_value = ElyraProperty.create_instance(parameter_class.property_id, self.pipeline_parameters) if converted_value is not None: diff --git a/elyra/templates/kubeflow/v2/generic_component_definition_template.jinja2 b/elyra/templates/kubeflow/v2/generic_component_definition_template.jinja2 new file mode 100644 index 000000000..85c8eb086 --- /dev/null +++ b/elyra/templates/kubeflow/v2/generic_component_definition_template.jinja2 @@ -0,0 +1,24 @@ +name: Run a file +description: Run a Jupyter notebook or Python/R script +{% if task_parameters %} +inputs: +{%- for parameter in task_parameters %} +- {name: {{ parameter.name }}, type: {{ parameter.input_type.component_input_type }}{% if parameter.description %}, description: "{{ parameter.description | string_delimiter_safe}}"{% endif %}{% if parameter.default_value is not none %}, default: {% if parameter.selected_type == 'String' %} "{{ parameter.default_value|string_delimiter_safe }}"{% else %}{{ parameter.default_value }}{% endif %}{% endif %}, optional: {{ (not parameter.required)|tojson }}} +{%- endfor %} +{% endif %} +implementation: + container: + image: {{ container_image }} + command: [sh, -c] + args: + - | + {%- for parameter in task_parameters %} + {{ parameter.name }}="${{ loop.index0 }}" + {%- endfor %} + {%- for command in command_args %} + sh -c "{{command}}" + {%- endfor %} + + {%- for parameter in task_parameters %} + - {inputValue: {{ parameter.name }}} + {%- endfor %} diff --git a/elyra/templates/kubeflow/v2/python_dsl_template.jinja2 b/elyra/templates/kubeflow/v2/python_dsl_template.jinja2 new file mode 100644 index 000000000..c5c812130 --- /dev/null +++ b/elyra/templates/kubeflow/v2/python_dsl_template.jinja2 @@ -0,0 +1,152 @@ +# +# Generated by Elyra {{ elyra_version }} +# +import kfp +from kubernetes.client import * +from kubernetes.client.models import * +from kfp.kubernetes import secret, volume + +from typing import Optional + +# ------------------------------------------------------------------ +# kfp-kubernetes 1.1.0 is misisng these function, explicity using them, +# TODO: remove these function once a new release of kfp-kubernetes is made. +# ------------------------------------------------------------------ + +from google.protobuf import json_format +from kfp.dsl import PipelineTask +from kfp.kubernetes import common +from kfp.kubernetes import kubernetes_executor_config_pb2 as pb + +from kfp.kubernetes import add_toleration, add_pod_label, add_pod_annotation + +{# Load statements for custom components -#} +{# component_hash = """""" -#} +{# factory_hash = kfp.components.load_component_from_text(component_hash) -#} +{% for hash, component_definition in component_definitions.items() %} +component_def_{{ hash | python_safe }} = """ +{{ component_definition }} +""" + +factory_{{ hash | python_safe }} = kfp.components.load_component_from_text(component_def_{{ hash | python_safe }}) +{% endfor %} + +{# Define pipeline -#} +{% if pipeline_description %} +@kfp.dsl.pipeline(name="{{ pipeline_name }}", description="{{ pipeline_description | string_delimiter_safe }}") +{% else %} +@kfp.dsl.pipeline(name="{{ pipeline_name }}") +{% endif %} +def generated_pipeline( +{% if pipeline_parameters %} +{% for parameter in pipeline_parameters %} + {{ parameter.name }}{% if parameter.input_type.type_hint %}: {{ parameter.input_type.type_hint }}{% endif %} = {{ parameter|param_val_to_python_var }}, +{% endfor %} +{% endif %} +): +{% for workflow_task in workflow_tasks.values() %} + {% set task_name = "task_" + workflow_task.escaped_task_id %} + # Task for node '{{ workflow_task.name }}' + {{ task_name }} = factory_{{ workflow_task.component_definition_hash | python_safe }}( +{% for task_input_name, task_input_spec in workflow_task.task_inputs.items() %} +{% if task_input_spec.task_output_reference %} + {{ task_input_name }}=task_{{ task_input_spec.task_output_reference.task_id }}.outputs["{{ task_input_spec.task_output_reference.output_id }}"], +{% elif task_input_spec.pipeline_parameter_reference %} + {{ task_input_name }}={{ task_input_spec.pipeline_parameter_reference }}, +{% elif task_input_spec.requires_quoted_rendering %} + {{ task_input_name }}="""{{ task_input_spec.value | string_delimiter_safe }}""", +{% else %} + {{ task_input_name }}={{ task_input_spec.value }}, +{% endif %} +{% endfor %} + ) +{% if workflow_task.task_modifiers.object_storage_secret %} + secret.use_secret_as_env({{ task_name }}, "{{ workflow_task.task_modifiers.object_storage_secret }}", { "AWS_ACCESS_KEY_ID": "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY": "AWS_SECRET_ACCESS_KEY" }) +{% endif %} + {{ task_name }}.set_display_name("{{ workflow_task.name | string_delimiter_safe }}") +{% if workflow_task.doc %} + add_pod_annotation({{ task_name }}, "elyra/node-user-doc","""{{ workflow_task.doc| string_delimiter_safe }}""") +{% endif %} +{% if workflow_task.task_modifiers.cpu_request %} + {{ task_name }}.set_cpu_request(cpu="{{ workflow_task.task_modifiers.cpu_request }}") +{% endif %} +{% if workflow_task.task_modifiers.mem_request and workflow_task.task_modifiers.mem_request.size %} + {{ task_name }}.set_memory_request(memory="{{ workflow_task.task_modifiers.mem_request.size }}{{ workflow_task.task_modifiers.mem_request.units }}") +{% endif %} +{% if workflow_task.task_modifiers.cpu_limit %} + {{ task_name }}.set_cpu_limit(cpu="{{ workflow_task.task_modifiers.cpu_limit }}") +{% endif %} +{% if workflow_task.task_modifiers.memory_limit and workflow_task.task_modifiers.memory_limit.size %} + {{ task_name }}.set_memory_limit(memory="{{ workflow_task.task_modifiers.memory_limit.size }}{{ workflow_task.task_modifiers.memory_limit.units }}") +{% endif %} +{% if workflow_task.task_modifiers.gpu_limit and workflow_task.task_modifiers.gpu_limit.size %} + {{ task_name }}.set_accelerator_limit("{{ workflow_task.task_modifiers.gpu_limit.size }}").set_accelerator_type("{{ workflow_task.task_modifiers.gpu_limit.vendor }}") +{% endif %} +{% if workflow_task.task_modifiers.env_variables %} +{% for env_var_name, env_var_value in workflow_task.task_modifiers.env_variables.items() %} + {{ task_name }}.set_env_variable(name="{{ env_var_name }}", value="{{ env_var_value | string_delimiter_safe }}") +{% endfor %} +{% endif %} +{% if workflow_task.task_modifiers.set_run_name %} + {{ task_name }}.set_env_variable(name="ELYRA_RUN_NAME", value="{{ workflow_task.task_modifiers.set_run_name }}") +{% endif %} +{% if workflow_task.task_modifiers.disable_node_caching %} + {{ task_name }}.execution_options.caching_strategy.max_cache_staleness = "P0D" +{% endif %} +{% if workflow_task.task_modifiers.pod_labels %} +{% for pod_label_key, pod_label_value in workflow_task.task_modifiers.pod_labels.items() %} + add_pod_label({{ task_name }}, "{{ pod_label_key }}", "{{ pod_label_value }}") +{% endfor %} +{% endif %} +{% if workflow_task.task_modifiers.pod_annotations %} +{% for pod_annotation_key, pod_annotation_value in workflow_task.task_modifiers.pod_annotations.items() %} + add_pod_annotation({{ task_name }}, "{{ pod_annotation_key }}" , """{{ pod_annotation_value | string_delimiter_safe }}""") +{% endfor %} +{% endif %} +{% if workflow_task.task_modifiers.kubernetes_secrets %} +{% for env_var, secret_dict in workflow_task.task_modifiers.kubernetes_secrets.items() %} + secret.use_secret_as_env({{ task_name }}, "{{ secret_dict.name }}", { "{{ secret_dict.key }}" : "{{ env_var }}" }) +{% endfor %} +{% endif %} +{% if workflow_task.task_modifiers.kubernetes_volumes %} +{% for volume_path, volume_dict in workflow_task.task_modifiers.kubernetes_volumes.items() %} + volume.mount_pvc({{ task_name }}, "{{ volume_dict.pvc_name }}", "{{ volume_path }}") +{% endfor %} +{% endif %} +{% if workflow_task.task_modifiers.kubernetes_tolerations %} +{% for toleration_dict in workflow_task.task_modifiers.kubernetes_tolerations.values() %} + add_toleration( + {{ task_name }}, + {% if toleration_dict.key %} + key="{{ toleration_dict.key }}", + {% else %} + key=None, + {% endif %} + operator="{{ toleration_dict.operator }}", + {% if toleration_dict.value %} + value="{{ toleration_dict.value | string_delimiter_safe }}", + {% else %} + value=None, + {% endif %} + {% if toleration_dict.effect %} + effect="{{ toleration_dict.effect }}", + {% else %} + effect=None, + {% endif %} + ) +{% endfor %} +{% endif %} +{# declare upstream dependencies -#} +{% if workflow_task.upstream_workflow_task_ids %} +{% for upstream_workflow_task_id in workflow_task.upstream_workflow_task_ids %} + {{ task_name }}.after(task_{{ upstream_workflow_task_id | python_safe }}) +{% endfor %} +{% endif %} +{% endfor %} + +if __name__ == "__main__": + from pathlib import Path + kfp.compiler.Compiler().compile( + pipeline_func=generated_pipeline, + package_path=Path(__file__).with_suffix(".yaml").name, + ) \ No newline at end of file diff --git a/elyra/tests/cli/resources/pipelines/kf_inputpath_parameter.pipeline b/elyra/tests/cli/resources/pipelines/kf_inputpath_parameter.pipeline index 36a693701..9ddfc3247 100644 --- a/elyra/tests/cli/resources/pipelines/kf_inputpath_parameter.pipeline +++ b/elyra/tests/cli/resources/pipelines/kf_inputpath_parameter.pipeline @@ -16,7 +16,7 @@ "component_parameters": { "url": { "activeControl": "StringControl", - "StringControl": "https://raw.githubusercontent.com/kubeflow/pipelines/93fc34474bf989998cf19445149aca2847eee763/components/notebooks/samples/test_notebook.ipynb" + "StringControl": "https://raw.githubusercontent.com/kubeflow/pipelines/sdk-2.8.0/components/contrib/notebooks/samples/test_notebook.ipynb" }, "curl_options": { "activeControl": "StringControl", @@ -25,7 +25,7 @@ "output_data": "" }, "label": "", - "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/1.6.0/components/web/Download/component.yaml", + "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/sdk-2.8.0/components/contrib/web/Download/component-sdk-v2.yaml", "ui_data": { "label": "Download data", "image": "data:image/svg+xml;utf8,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20276.93%20274.55%22%3E%3Cg%20id%3D%22Layer_2%22%20data-name%3D%22Layer%202%22%3E%3Cg%20id%3D%22Layer_1-2%22%20data-name%3D%22Layer%201%22%3E%3Cpath%20d%3D%22M95.9%2C62.15%2C100%2C164.25l73.75-94.12a6.79%2C6.79%2C0%2C0%2C1%2C9.6-1.11l46%2C36.92-15-65.61Z%22%20fill%3D%22%234279f4%22%2F%3E%3Cpolygon%20points%3D%22102.55%20182.98%20167.97%20182.98%20127.8%20150.75%20102.55%20182.98%22%20fill%3D%22%230028aa%22%2F%3E%3Cpolygon%20points%3D%22180.18%2083.92%20136.18%20140.06%20183.06%20177.67%20227.53%20121.91%20180.18%2083.92%22%20fill%3D%22%23014bd1%22%2F%3E%3Cpolygon%20points%3D%2283.56%2052.3%2083.57%2052.29%20122.26%203.77%2059.87%2033.82%2044.46%20101.33%2083.56%2052.3%22%20fill%3D%22%23bedcff%22%2F%3E%3Cpolygon%20points%3D%2245.32%20122.05%2086.76%20174.01%2082.81%2075.03%2045.32%20122.05%22%20fill%3D%22%236ca1ff%22%2F%3E%3Cpolygon%20points%3D%22202.31%2028.73%20142.65%200%20105.52%2046.56%20202.31%2028.73%22%20fill%3D%22%23a1c3ff%22%2F%3E%3Cpath%20d%3D%22M1.6%2C272V227.22H7.34v23.41l20.48-23.41h6.4l-17.39%2C19.7%2C19%2C25.07H29.1l-15.92-20.8-5.84%2C6.65V272Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M41.62%2C262.21V240h5.43v22.39a4.67%2C4.67%2C0%2C0%2C0%2C2.35%2C4.19%2C11%2C11%2C0%2C0%2C0%2C11%2C0%2C4.69%2C4.69%2C0%2C0%2C0%2C2.33-4.19V240h5.43v22.19a9.08%2C9.08%2C0%2C0%2C1-4.1%2C7.87%2C16.2%2C16.2%2C0%2C0%2C1-18.37%2C0A9.07%2C9.07%2C0%2C0%2C1%2C41.62%2C262.21Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M77.46%2C272V224h5.43v16.81a29.29%2C29.29%2C0%2C0%2C1%2C9.32-1.73%2C13.1%2C13.1%2C0%2C0%2C1%2C6.2%2C1.41%2C10.71%2C10.71%2C0%2C0%2C1%2C4.18%2C3.74%2C18.07%2C18.07%2C0%2C0%2C1%2C2.23%2C5.06%2C21.26%2C21.26%2C0%2C0%2C1%2C.73%2C5.58q0%2C8.43-4.38%2C12.79T87.35%2C272Zm5.43-4.87h4.55q6.77%2C0%2C9.72-2.95t3-9.51a14.21%2C14.21%2C0%2C0%2C0-2-7.52%2C6.55%2C6.55%2C0%2C0%2C0-6-3.22%2C24.73%2C24.73%2C0%2C0%2C0-9.25%2C1.54Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M112.36%2C255.94q0-7.71%2C4.09-12.3a13.75%2C13.75%2C0%2C0%2C1%2C10.8-4.59q13.35%2C0%2C13.36%2C18.86H117.79a12.3%2C12.3%2C0%2C0%2C0%2C2.9%2C7.07q2.59%2C3.11%2C7.9%2C3.1a24.92%2C24.92%2C0%2C0%2C0%2C10.55-2v5a27.74%2C27.74%2C0%2C0%2C1-9.86%2C1.87%2C19.83%2C19.83%2C0%2C0%2C1-7.7-1.37%2C13.31%2C13.31%2C0%2C0%2C1-5.28-3.76%2C16.21%2C16.21%2C0%2C0%2C1-3-5.38A20.84%2C20.84%2C0%2C0%2C1%2C112.36%2C255.94Zm5.62-2.12h17.26a14.91%2C14.91%2C0%2C0%2C0-2.37-7.12%2C6.44%2C6.44%2C0%2C0%2C0-5.62-2.78%2C8.2%2C8.2%2C0%2C0%2C0-6.21%2C2.72A12.07%2C12.07%2C0%2C0%2C0%2C118%2C253.82Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M147.32%2C244.89V240h5v-7.59a8.14%2C8.14%2C0%2C0%2C1%2C2.31-6.05%2C7.79%2C7.79%2C0%2C0%2C1%2C5.69-2.28h7.86V229h-5c-2.21%2C0-3.67.45-4.37%2C1.34s-1.06%2C2.55-1.06%2C5V240h8.46v4.87h-8.46V272h-5.44v-27.1Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M175.26%2C272V224h5.43v48Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M194.41%2C268.05a17.86%2C17.86%2C0%2C1%2C1%2C12.33%2C4.9A16.57%2C16.57%2C0%2C0%2C1%2C194.41%2C268.05Zm3.84-20.65a13.16%2C13.16%2C0%2C0%2C0%2C0%2C17.2%2C12.07%2C12.07%2C0%2C0%2C0%2C17%2C0%2C13.09%2C13.09%2C0%2C0%2C0%2C0-17.2%2C12.07%2C12.07%2C0%2C0%2C0-17%2C0Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M228.45%2C240h5.75l7.3%2C25.32L248.93%2C240h5.36l7.34%2C25.34L269%2C240h5.74L264.7%2C272h-6.12l-6.83-24.58L245%2C272h-6.47Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E", @@ -150,7 +150,7 @@ "output_hash": "" }, "label": "", - "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/1.6.0/components/basics/Calculate_hash/component.yaml", + "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/sdk-2.8.0/components/contrib/basics/Calculate_hash/component.yaml", "ui_data": { "label": "Calculate data hash", "image": "data:image/svg+xml;utf8,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20276.93%20274.55%22%3E%3Cg%20id%3D%22Layer_2%22%20data-name%3D%22Layer%202%22%3E%3Cg%20id%3D%22Layer_1-2%22%20data-name%3D%22Layer%201%22%3E%3Cpath%20d%3D%22M95.9%2C62.15%2C100%2C164.25l73.75-94.12a6.79%2C6.79%2C0%2C0%2C1%2C9.6-1.11l46%2C36.92-15-65.61Z%22%20fill%3D%22%234279f4%22%2F%3E%3Cpolygon%20points%3D%22102.55%20182.98%20167.97%20182.98%20127.8%20150.75%20102.55%20182.98%22%20fill%3D%22%230028aa%22%2F%3E%3Cpolygon%20points%3D%22180.18%2083.92%20136.18%20140.06%20183.06%20177.67%20227.53%20121.91%20180.18%2083.92%22%20fill%3D%22%23014bd1%22%2F%3E%3Cpolygon%20points%3D%2283.56%2052.3%2083.57%2052.29%20122.26%203.77%2059.87%2033.82%2044.46%20101.33%2083.56%2052.3%22%20fill%3D%22%23bedcff%22%2F%3E%3Cpolygon%20points%3D%2245.32%20122.05%2086.76%20174.01%2082.81%2075.03%2045.32%20122.05%22%20fill%3D%22%236ca1ff%22%2F%3E%3Cpolygon%20points%3D%22202.31%2028.73%20142.65%200%20105.52%2046.56%20202.31%2028.73%22%20fill%3D%22%23a1c3ff%22%2F%3E%3Cpath%20d%3D%22M1.6%2C272V227.22H7.34v23.41l20.48-23.41h6.4l-17.39%2C19.7%2C19%2C25.07H29.1l-15.92-20.8-5.84%2C6.65V272Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M41.62%2C262.21V240h5.43v22.39a4.67%2C4.67%2C0%2C0%2C0%2C2.35%2C4.19%2C11%2C11%2C0%2C0%2C0%2C11%2C0%2C4.69%2C4.69%2C0%2C0%2C0%2C2.33-4.19V240h5.43v22.19a9.08%2C9.08%2C0%2C0%2C1-4.1%2C7.87%2C16.2%2C16.2%2C0%2C0%2C1-18.37%2C0A9.07%2C9.07%2C0%2C0%2C1%2C41.62%2C262.21Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M77.46%2C272V224h5.43v16.81a29.29%2C29.29%2C0%2C0%2C1%2C9.32-1.73%2C13.1%2C13.1%2C0%2C0%2C1%2C6.2%2C1.41%2C10.71%2C10.71%2C0%2C0%2C1%2C4.18%2C3.74%2C18.07%2C18.07%2C0%2C0%2C1%2C2.23%2C5.06%2C21.26%2C21.26%2C0%2C0%2C1%2C.73%2C5.58q0%2C8.43-4.38%2C12.79T87.35%2C272Zm5.43-4.87h4.55q6.77%2C0%2C9.72-2.95t3-9.51a14.21%2C14.21%2C0%2C0%2C0-2-7.52%2C6.55%2C6.55%2C0%2C0%2C0-6-3.22%2C24.73%2C24.73%2C0%2C0%2C0-9.25%2C1.54Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M112.36%2C255.94q0-7.71%2C4.09-12.3a13.75%2C13.75%2C0%2C0%2C1%2C10.8-4.59q13.35%2C0%2C13.36%2C18.86H117.79a12.3%2C12.3%2C0%2C0%2C0%2C2.9%2C7.07q2.59%2C3.11%2C7.9%2C3.1a24.92%2C24.92%2C0%2C0%2C0%2C10.55-2v5a27.74%2C27.74%2C0%2C0%2C1-9.86%2C1.87%2C19.83%2C19.83%2C0%2C0%2C1-7.7-1.37%2C13.31%2C13.31%2C0%2C0%2C1-5.28-3.76%2C16.21%2C16.21%2C0%2C0%2C1-3-5.38A20.84%2C20.84%2C0%2C0%2C1%2C112.36%2C255.94Zm5.62-2.12h17.26a14.91%2C14.91%2C0%2C0%2C0-2.37-7.12%2C6.44%2C6.44%2C0%2C0%2C0-5.62-2.78%2C8.2%2C8.2%2C0%2C0%2C0-6.21%2C2.72A12.07%2C12.07%2C0%2C0%2C0%2C118%2C253.82Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M147.32%2C244.89V240h5v-7.59a8.14%2C8.14%2C0%2C0%2C1%2C2.31-6.05%2C7.79%2C7.79%2C0%2C0%2C1%2C5.69-2.28h7.86V229h-5c-2.21%2C0-3.67.45-4.37%2C1.34s-1.06%2C2.55-1.06%2C5V240h8.46v4.87h-8.46V272h-5.44v-27.1Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M175.26%2C272V224h5.43v48Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M194.41%2C268.05a17.86%2C17.86%2C0%2C1%2C1%2C12.33%2C4.9A16.57%2C16.57%2C0%2C0%2C1%2C194.41%2C268.05Zm3.84-20.65a13.16%2C13.16%2C0%2C0%2C0%2C0%2C17.2%2C12.07%2C12.07%2C0%2C0%2C0%2C17%2C0%2C13.09%2C13.09%2C0%2C0%2C0%2C0-17.2%2C12.07%2C12.07%2C0%2C0%2C0-17%2C0Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M228.45%2C240h5.75l7.3%2C25.32L248.93%2C240h5.36l7.34%2C25.34L269%2C240h5.74L264.7%2C272h-6.12l-6.83-24.58L245%2C272h-6.47Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E", diff --git a/elyra/tests/cli/test_pipeline_app.py b/elyra/tests/cli/test_pipeline_app.py index c043abe9e..3ae1b25e8 100644 --- a/elyra/tests/cli/test_pipeline_app.py +++ b/elyra/tests/cli/test_pipeline_app.py @@ -1079,6 +1079,9 @@ def test_export_incompatible_runtime_config(kubeflow_pipelines_runtime_instance, @pytest.mark.parametrize("catalog_instance_no_server_process", [KFP_COMPONENT_CACHE_INSTANCE], indirect=True) +@pytest.mark.skip( + reason="This test is not compatible with KFP v2: It relies on incompatible assets from elyra-examples-kfp-catalog lib. See https://github.com/elyra-ai/examples/issues/115 and https://github.com/opendatahub-io/elyra-examples/pull/1" # noqa: E501 +) def test_export_kubeflow_output_option( jp_environ, kubeflow_pipelines_runtime_instance, catalog_instance_no_server_process ): @@ -1229,6 +1232,9 @@ def test_export_airflow_output_option(airflow_runtime_instance): @pytest.mark.parametrize("catalog_instance_no_server_process", [KFP_COMPONENT_CACHE_INSTANCE], indirect=True) +@pytest.mark.skip( + reason="This test is not compatible with KFP v2: It relies on incompatible assets from elyra-examples-kfp-catalog lib. See https://github.com/elyra-ai/examples/issues/115 and https://github.com/opendatahub-io/elyra-examples/pull/1" # noqa: E501 +) def test_export_kubeflow_overwrite_option( jp_environ, kubeflow_pipelines_runtime_instance, catalog_instance_no_server_process ): @@ -1322,6 +1328,9 @@ def test_export_airflow_format_option(airflow_runtime_instance): @pytest.mark.parametrize("catalog_instance_no_server_process", [KFP_COMPONENT_CACHE_INSTANCE], indirect=True) +@pytest.mark.skip( + reason="This test is not compatible with KFP v2: It relies on incompatible assets from elyra-examples-kfp-catalog lib. See https://github.com/elyra-ai/examples/issues/115 and https://github.com/opendatahub-io/elyra-examples/pull/1" # noqa: E501 +) def test_export_kubeflow_format_option( jp_environ, kubeflow_pipelines_runtime_instance, catalog_instance_no_server_process ): diff --git a/elyra/tests/metadata/test_metadata.py b/elyra/tests/metadata/test_metadata.py index 71728dd3a..d431d8428 100644 --- a/elyra/tests/metadata/test_metadata.py +++ b/elyra/tests/metadata/test_metadata.py @@ -704,7 +704,7 @@ def test_validation_performance(): print( f"\nMemory: {diff:,} kb, Start: {memory_start.rss / 1024 / 1024:,.3f} mb, " f"End: {memory_end.rss / 1024 / 1024:,.3f} mb., " - f"Elapsed time: {t1-t0:.3f}s over {iterations} iterations." + f"Elapsed time: {t1 - t0:.3f}s over {iterations} iterations." ) diff --git a/elyra/tests/pipeline/kfp/test_processor_kfp.py b/elyra/tests/pipeline/kfp/test_processor_kfp.py index f57040b78..6ab857d2e 100644 --- a/elyra/tests/pipeline/kfp/test_processor_kfp.py +++ b/elyra/tests/pipeline/kfp/test_processor_kfp.py @@ -15,37 +15,25 @@ # from datetime import datetime import hashlib -import json import os from pathlib import Path import re from typing import Any from typing import Dict -from kfp.dsl import RUN_ID_PLACEHOLDER import pytest import yaml from elyra.pipeline.catalog_connector import FilesystemComponentCatalogConnector from elyra.pipeline.component import Component from elyra.pipeline.kfp.kfp_properties import KfpPipelineParameter -from elyra.pipeline.kfp.processor_kfp import CRIO_VOL_DEF_MEDIUM -from elyra.pipeline.kfp.processor_kfp import CRIO_VOL_DEF_NAME -from elyra.pipeline.kfp.processor_kfp import CRIO_VOL_DEF_SIZE -from elyra.pipeline.kfp.processor_kfp import CRIO_VOL_MOUNT_PATH -from elyra.pipeline.kfp.processor_kfp import CRIO_VOL_PYTHON_PATH -from elyra.pipeline.kfp.processor_kfp import CRIO_VOL_WORKDIR_PATH from elyra.pipeline.kfp.processor_kfp import KfpPipelineProcessor +from elyra.pipeline.kfp.processor_kfp import RUN_ID_PLACEHOLDER from elyra.pipeline.kfp.processor_kfp import WorkflowEngineType from elyra.pipeline.pipeline import GenericOperation from elyra.pipeline.pipeline import Operation from elyra.pipeline.pipeline import Pipeline -from elyra.pipeline.pipeline_constants import COS_OBJECT_PREFIX -from elyra.pipeline.pipeline_constants import KUBERNETES_POD_ANNOTATIONS -from elyra.pipeline.pipeline_constants import KUBERNETES_POD_LABELS -from elyra.pipeline.pipeline_constants import KUBERNETES_SECRETS from elyra.pipeline.pipeline_constants import KUBERNETES_SHARED_MEM_SIZE -from elyra.pipeline.pipeline_constants import KUBERNETES_TOLERATIONS from elyra.pipeline.pipeline_constants import MOUNTED_VOLUMES from elyra.pipeline.processor import PipelineProcessor from elyra.pipeline.properties import ComponentProperty @@ -57,7 +45,6 @@ from elyra.pipeline.properties import KubernetesSecret from elyra.pipeline.properties import KubernetesToleration from elyra.pipeline.properties import VolumeMount -from elyra.util.cos import join_paths from elyra.util.kubernetes import sanitize_label_value PIPELINE_FILE_COMPLEX = str((Path("resources") / "sample_pipelines" / "pipeline_dependency_complex.json").as_posix()) @@ -491,40 +478,8 @@ def test_generate_pipeline_dsl_compile_pipeline_dsl_custom_component_pipeline( with open(compiled_argo_output_file) as fh: argo_spec = yaml.safe_load(fh.read()) - assert "argoproj.io/" in argo_spec["apiVersion"] - pipeline_spec_annotations = json.loads(argo_spec["metadata"]["annotations"]["pipelines.kubeflow.org/pipeline_spec"]) - assert ( - pipeline_spec_annotations["name"] == pipeline.name - ), f"DSL input: {generated_argo_dsl}\nArgo output: {argo_spec}" - assert pipeline_spec_annotations["description"] == pipeline.description, pipeline_spec_annotations - - # generate Python DSL for the Tekton workflow engine - generated_tekton_dsl = processor._generate_pipeline_dsl( - pipeline=pipeline, pipeline_name=pipeline.name, workflow_engine=WorkflowEngineType.TEKTON - ) - - assert generated_tekton_dsl is not None - # Generated DSL includes workflow engine specific code in the _main_ function - assert "compiler.TektonCompiler().compile(" in generated_tekton_dsl - - compiled_tekton_output_file = Path(tmpdir) / "compiled_kfp_test_tekton.yaml" - - # if the compiler discovers an issue with the generated DSL this call fails - processor._compile_pipeline_dsl( - dsl=generated_tekton_dsl, - workflow_engine=WorkflowEngineType.TEKTON, - output_file=compiled_tekton_output_file.as_posix(), - pipeline_conf=None, - ) - - # verify that the output file exists - assert compiled_tekton_output_file.is_file() - - # verify the file content - with open(compiled_tekton_output_file) as fh: - tekton_spec = yaml.safe_load(fh.read()) - - assert "tekton.dev/" in tekton_spec["apiVersion"] + assert argo_spec["pipelineInfo"]["name"] == pipeline.name + assert argo_spec["pipelineInfo"]["description"] == pipeline.description @pytest.mark.parametrize( @@ -539,15 +494,6 @@ def test_generate_pipeline_dsl_compile_pipeline_dsl_custom_component_pipeline( / "kfp-one-node-generic.pipeline", "workflow_engine": WorkflowEngineType.ARGO, }, - { - "pipeline_file": Path(__file__).parent - / ".." - / "resources" - / "test_pipelines" - / "kfp" - / "kfp-one-node-generic.pipeline", - "workflow_engine": WorkflowEngineType.TEKTON, - }, ], indirect=True, ) @@ -614,13 +560,11 @@ def test_generate_pipeline_dsl_compile_pipeline_dsl_workflow_engine_test( # Load compiled workflow with open(compiled_output_file_name) as f: - workflow_spec = yaml.safe_load(f.read()) + workflow_spec_docs = list(yaml.safe_load_all(f.read())) - # Verify that the output is for the specified workflow engine - if workflow_engine == WorkflowEngineType.TEKTON: - assert "tekton.dev/" in workflow_spec["apiVersion"] - else: - assert "argoproj.io/" in workflow_spec["apiVersion"] + assert len(workflow_spec_docs) == 2 + assert "components" in workflow_spec_docs[0] + assert "platforms" in workflow_spec_docs[1] @pytest.mark.parametrize( @@ -634,17 +578,6 @@ def test_generate_pipeline_dsl_compile_pipeline_dsl_workflow_engine_test( / "kfp" / "kfp-one-node-generic.pipeline", "workflow_engine": WorkflowEngineType.ARGO, - "use_cos_credentials_secret": True, - }, - { - "pipeline_file": Path(__file__).parent - / ".." - / "resources" - / "test_pipelines" - / "kfp" - / "kfp-one-node-generic.pipeline", - "workflow_engine": WorkflowEngineType.ARGO, - "use_cos_credentials_secret": False, }, ], indirect=True, @@ -672,7 +605,6 @@ def test_generate_pipeline_dsl_compile_pipeline_dsl_one_generic_node_pipeline_te runtime_config = metadata_dependencies["runtime_config"] assert runtime_config is not None assert runtime_config.name == pipeline.runtime_config - runtime_image_configs = metadata_dependencies["runtime_image_configs"] workflow_engine = WorkflowEngineType.get_instance_by_value(runtime_config.metadata["engine"]) @@ -714,28 +646,26 @@ def test_generate_pipeline_dsl_compile_pipeline_dsl_one_generic_node_pipeline_te pipeline_conf=None, ) - # Load generated Argo workflow + # Load generated workflow with open(compiled_argo_output_file_name) as f: - argo_spec = yaml.safe_load(f.read()) + spec_docs = list(yaml.safe_load_all(f.read())) - # verify that this is an argo specification - assert "argoproj.io" in argo_spec["apiVersion"] + assert len(spec_docs) == 2 + components, platforms = spec_docs[0], spec_docs[1]["platforms"] - pipeline_meta_annotations = json.loads(argo_spec["metadata"]["annotations"]["pipelines.kubeflow.org/pipeline_spec"]) - assert pipeline_meta_annotations["name"] == pipeline.name - assert pipeline_meta_annotations["description"] == pipeline.description + assert components["pipelineInfo"]["name"] == pipeline.name + assert components["pipelineInfo"]["description"] == pipeline.description # There should be two templates, one for the DAG and one for the generic node. # Locate the one for the generic node and inspect its properties. - assert len(argo_spec["spec"]["templates"]) == 2 - if argo_spec["spec"]["templates"][0]["name"] == argo_spec["spec"]["entrypoint"]: - node_template = argo_spec["spec"]["templates"][1] - else: - node_template = argo_spec["spec"]["templates"][0] + assert components["root"]["dag"] + assert len(components["components"]) == 1 + executors = components["deploymentSpec"]["executors"] + assert len(executors) == 1 + + node_template = components["deploymentSpec"]["executors"]["exec-run-a-file"] # Verify component definition information (see generic_component_definition_template.jinja2) - # - property 'name' - assert node_template["name"] == "run-a-file" # - property 'implementation.container.command' assert node_template["container"]["command"] == ["sh", "-c"] # - property 'implementation.container.args' @@ -749,11 +679,7 @@ def test_generate_pipeline_dsl_compile_pipeline_dsl_one_generic_node_pipeline_te # - the object storage bucket name that this node uses for file I/O assert f"--cos-bucket '{runtime_config.metadata['cos_bucket']}'" in node_template["container"]["args"][0] # - the directory within that object storage bucket - if pipeline.pipeline_properties.get(COS_OBJECT_PREFIX): - expected_directory_value = join_paths(pipeline.pipeline_properties.get(COS_OBJECT_PREFIX), pipeline_instance_id) - assert f"--cos-directory '{expected_directory_value}' " in node_template["container"]["args"][0] - else: - assert f"--cos-directory '{pipeline_instance_id}" in node_template["container"]["args"][0] + assert f"--cos-directory '{pipeline_instance_id}" in node_template["container"]["args"][0] # - the name of the archive in that directory expected_archive_name = processor._get_dependency_archive_name(op) assert f"--cos-dependencies-archive '{expected_archive_name}' " in node_template["container"]["args"][0] @@ -770,33 +696,26 @@ def test_generate_pipeline_dsl_compile_pipeline_dsl_one_generic_node_pipeline_te # - property 'implementation.container.image' assert node_template["container"]["image"] == op.runtime_image - # - property 'implementation.container.imagePullPolicy' - # The image pull policy is defined in the the runtime image - # configuration. Look it up and verified it is properly applied. - for runtime_image_config in runtime_image_configs: - if runtime_image_config.metadata["image_name"] == op.runtime_image: - if runtime_image_config.metadata.get("pull_policy"): - assert node_template["container"]["imagePullPolicy"] == runtime_image_config.metadata["pull_policy"] - else: - assert node_template["container"].get("imagePullPolicy") is None - break + + pod_metadata = platforms["kubernetes"]["deploymentSpec"]["executors"]["exec-run-a-file"]["podMetadata"] + assert pod_metadata # Verify Kubernetes labels and annotations that Elyra attaches to pods that # execute generic nodes or custom nodes if op.doc: # only set if a comment is attached to the node - assert node_template["metadata"]["annotations"].get("elyra/node-user-doc") == op.doc + assert pod_metadata["annotations"]["elyra/node-user-doc"] == op.doc # Verify Kubernetes labels and annotations that Elyra attaches to pods that # execute generic nodes - assert node_template["metadata"]["annotations"]["elyra/node-file-name"] == op.filename + assert pod_metadata["annotations"]["elyra/node-file-name"] == op.filename if pipeline.source: - assert node_template["metadata"]["annotations"]["elyra/pipeline-source"] == pipeline.source - assert node_template["metadata"]["labels"]["elyra/node-name"] == sanitize_label_value(op.name) - assert node_template["metadata"]["labels"]["elyra/node-type"] == sanitize_label_value("notebook-script") - assert node_template["metadata"]["labels"]["elyra/pipeline-name"] == sanitize_label_value(pipeline.name) - assert node_template["metadata"]["labels"]["elyra/pipeline-version"] == sanitize_label_value(pipeline_version) - assert node_template["metadata"]["labels"]["elyra/experiment-name"] == sanitize_label_value(experiment_name) + assert pod_metadata["annotations"]["elyra/pipeline-source"] == pipeline.source + assert pod_metadata["labels"]["elyra/node-name"] == sanitize_label_value(op.name) + assert pod_metadata["labels"]["elyra/node-type"] == sanitize_label_value("notebook-script") + assert pod_metadata["labels"]["elyra/pipeline-name"] == sanitize_label_value(pipeline.name) + assert pod_metadata["labels"]["elyra/pipeline-version"] == sanitize_label_value(pipeline_version) + assert pod_metadata["labels"]["elyra/experiment-name"] == sanitize_label_value(experiment_name) # Verify environment variables that Elyra attaches to pods that # execute generic nodes. All values are hard-coded in the template, with the @@ -828,20 +747,6 @@ def test_generate_pipeline_dsl_compile_pipeline_dsl_one_generic_node_pipeline_te else: assert env_var["value"] == runtime_config.metadata["cos_password"] - # Verify that the mlpipeline specific outputs are declared - assert node_template.get("outputs") is not None, node_template - assert node_template["outputs"]["artifacts"] is not None, node_template["container"]["outputs"] - assert node_template["outputs"]["artifacts"][0]["name"] == "mlpipeline-metrics" - assert ( - node_template["outputs"]["artifacts"][0]["path"] - == (Path(KfpPipelineProcessor.WCD) / "mlpipeline-metrics.json").as_posix() - ) - assert node_template["outputs"]["artifacts"][1]["name"] == "mlpipeline-ui-metadata" - assert ( - node_template["outputs"]["artifacts"][1]["path"] - == (Path(KfpPipelineProcessor.WCD) / "mlpipeline-ui-metadata.json").as_posix() - ) - @pytest.mark.parametrize( "metadata_dependencies", @@ -928,29 +833,34 @@ def test_generate_pipeline_dsl_compile_pipeline_dsl_one_generic_node_pipeline_te # Load generated Argo workflow with open(compiled_argo_output_file_name) as f: - argo_spec = yaml.safe_load(f.read()) + spec_docs = list(yaml.safe_load_all(f.read())) + + assert len(spec_docs) == 2 + components = spec_docs[0] - # verify that this is an argo specification - assert "argoproj.io" in argo_spec["apiVersion"] + assert components["pipelineInfo"]["name"] == pipeline.name + assert components["pipelineInfo"]["description"] == pipeline.description # There should be two templates, one for the DAG and one for the generic node. # Locate the one for the generic node and inspect its properties. - assert len(argo_spec["spec"]["templates"]) == 2 - if argo_spec["spec"]["templates"][0]["name"] == argo_spec["spec"]["entrypoint"]: - node_template = argo_spec["spec"]["templates"][1] - else: - node_template = argo_spec["spec"]["templates"][0] + assert components["root"]["dag"] + assert len(components["components"]) == 1 + executors = components["deploymentSpec"]["executors"] + assert len(executors) == 1 op = list(pipeline.operations.values())[0] - if op.gpu or op.cpu or op.memory: - assert node_template["container"].get("resources") is not None + node_template = components["deploymentSpec"]["executors"]["exec-run-a-file"] + + if op.gpu or op.cpu or op.memory or op.cpu_limit or op.memory_limit: + assert node_template["container"]["resources"] if op.gpu: - assert node_template["container"]["resources"]["limits"]["nvidia.com/gpu"] == str(op.gpu) + assert node_template["container"]["resources"]["accelerator"]["type"] == "nvidia.com/gpu" + assert node_template["container"]["resources"]["accelerator"]["count"] == str(op.gpu) if op.cpu: - assert node_template["container"]["resources"]["requests"]["cpu"] == str(op.cpu) + assert node_template["container"]["resources"]["cpuRequest"] == op.cpu if op.memory: - assert node_template["container"]["resources"]["requests"]["memory"] == f"{op.memory}G" + assert node_template["container"]["resources"]["memoryRequest"] == op.memory @pytest.fixture(autouse=False) @@ -969,156 +879,6 @@ def enable_and_disable_crio(request): del os.environ["CRIO_RUNTIME"] -@pytest.mark.parametrize("enable_and_disable_crio", [False, True], indirect=True) -@pytest.mark.parametrize( - "metadata_dependencies", - [ - { - "pipeline_file": Path(__file__).parent - / ".." - / "resources" - / "test_pipelines" - / "kfp" - / "kfp-one-node-generic.pipeline", - "workflow_engine": WorkflowEngineType.ARGO, - }, - ], - indirect=True, -) -def test_generate_pipeline_dsl_compile_pipeline_dsl_generic_component_crio( - monkeypatch, processor: KfpPipelineProcessor, metadata_dependencies: Dict[str, Any], tmpdir, enable_and_disable_crio -): - """ - This test validates that the output of _generate_pipeline_dsl and _compile_pipeline_dsl - yields the expected results for a generic node when the CRIO_RUNTIME environment variable - is set to a valid string representation of the boolean value True (/true/i). - Test assumptions: - - Enabling CRIO_RUNTIME has the same effect for all supported workflow engines - - The test pipeline contains at least one generic node - - With CRIO_RUNTIME enabled, the compiled output must include the following properties: - - in spec.templates[].volumes: - - emptyDir: {medium: '', sizeLimit: 20Gi} - name: workspace - """ - crio_runtime_enabled = os.environ.get("CRIO_RUNTIME", "").lower() == "true" - - # Obtain artifacts from metadata_dependencies fixture - test_pipeline_file = metadata_dependencies["pipeline_file"] - pipeline = metadata_dependencies["pipeline_object"] - assert pipeline is not None - runtime_config = metadata_dependencies["runtime_config"] - assert runtime_config is not None - - workflow_engine = WorkflowEngineType.get_instance_by_value(runtime_config.metadata["engine"]) - - # Mock calls that require access to object storage, because their side effects - # have no bearing on the outcome of this test. - monkeypatch.setattr(processor, "_upload_dependencies_to_object_store", lambda w, x, y, prefix: True) - monkeypatch.setattr(processor, "_verify_cos_connectivity", lambda x: True) - - # Mock pipeline to not include any parameters - monkeypatch.setattr(pipeline, "_pipeline_parameters", ElyraPropertyList([])) - - # Test begins here - - compiled_output_file = Path(tmpdir) / test_pipeline_file.with_suffix(".yaml").name - compiled_output_file_name = str(compiled_output_file.absolute()) - - # generate Python DSL for the specified workflow engine - pipeline_version = f"{pipeline.name}-test-0" - pipeline_instance_id = f"{pipeline.name}-{datetime.now().strftime('%m%d%H%M%S')}" - experiment_name = f"{pipeline.name}-test-0" - - generated_dsl = processor._generate_pipeline_dsl( - pipeline=pipeline, - pipeline_name=pipeline.name, - workflow_engine=workflow_engine, - pipeline_version=pipeline_version, - pipeline_instance_id=pipeline_instance_id, - experiment_name=experiment_name, - ) - - # Compile the DSL - processor._compile_pipeline_dsl( - dsl=generated_dsl, - workflow_engine=workflow_engine, - output_file=compiled_output_file_name, - pipeline_conf=None, - ) - - # Load compiled workflow - with open(compiled_output_file_name) as f: - compiled_spec = yaml.safe_load(f.read()) - - # There should be multiple templates, one for the DAG and one for every generic node. - assert len(compiled_spec["spec"]["templates"]) >= 2 - if crio_runtime_enabled: - for template in compiled_spec["spec"]["templates"]: - if template["name"] == compiled_spec["spec"]["entrypoint"]: - continue - # Check volume definition - assert template.get("volumes") is not None, template - entry_found = False - for volume_entry in template["volumes"]: - if volume_entry["name"] != CRIO_VOL_DEF_NAME: - continue - assert ( - volume_entry.get("emptyDir") is not None - ), f"Unexpected volume entry '{CRIO_VOL_DEF_NAME}': {volume_entry} " - assert volume_entry["emptyDir"]["sizeLimit"] == CRIO_VOL_DEF_SIZE - assert volume_entry["emptyDir"]["medium"] == CRIO_VOL_DEF_MEDIUM - entry_found = True - assert entry_found, f"Missing volume entry '{CRIO_VOL_DEF_NAME}' for CRI-O in {template['volumes']}" - # Check volume mount definition - assert template["container"].get("volumeMounts") is not None, template["container"] - for volumemount_entry in template["container"]["volumeMounts"]: - entry_found = False - if volumemount_entry["name"] != CRIO_VOL_DEF_NAME: - continue - assert volumemount_entry["mountPath"] == CRIO_VOL_MOUNT_PATH - entry_found = True - break - assert ( - entry_found - ), f"Missing volume mount entry '{CRIO_VOL_DEF_NAME}' for CRI-O in {template['container']['volumeMounts']}" - # Check PYTHONPATH environment variable (python_user_lib_path) - assert template["container"].get("env") is not None, template["container"] - for env_entry in template["container"]["env"]: - entry_found = False - if env_entry["name"] != "PYTHONPATH": - continue - assert env_entry["value"] == CRIO_VOL_PYTHON_PATH - entry_found = True - break - assert entry_found, f"Missing env variable entry 'PYTHONPATH' for CRI-O in {template['container']['env']}" - # Check the container command argument list - assert len(template["container"]["args"]) == 1 - assert f"mkdir -p {CRIO_VOL_WORKDIR_PATH}" in template["container"]["args"][0] - assert f"--target={CRIO_VOL_PYTHON_PATH}" in template["container"]["args"][0] - assert f"--user-volume-path '{CRIO_VOL_PYTHON_PATH}' " in template["container"]["args"][0] - else: - for template in compiled_spec["spec"]["templates"]: - if template["name"] == compiled_spec["spec"]["entrypoint"]: - continue - # Check if a volume was defined - for volume_entry in template.get("volumes", []): - if volume_entry["name"] == CRIO_VOL_DEF_NAME: - # if a volume with the 'reserved' name exist there could be a problem - assert volume_entry.get("emptyDir") is None - # Check volume mount definition - for volumemount_entry in template["container"].get("volumeMounts", []): - if volumemount_entry["name"] == CRIO_VOL_DEF_NAME: - assert volumemount_entry["mountPath"] != CRIO_VOL_MOUNT_PATH - # Check PYTHONPATH environment variable - for env_entry in template["container"].get("env", []): - assert env_entry["name"] != "PYTHONPATH" - # Check the container command argument list - assert "mkdir -p ./jupyter-work-dir" in template["container"]["args"][0] - assert f"--target={CRIO_VOL_PYTHON_PATH}" not in template["container"]["args"][0] - assert "--user-volume-path" not in template["container"]["args"][0] - - @pytest.mark.parametrize( "metadata_dependencies", [ @@ -1134,6 +894,10 @@ def test_generate_pipeline_dsl_compile_pipeline_dsl_generic_component_crio( ], indirect=True, ) +@pytest.mark.skip( + reason="This test is not compatible with KFP v2 as the generated YAML is ignoring \ + attributes from the source pipeline file" +) def test_generate_pipeline_dsl_compile_pipeline_dsl_optional_elyra_properties( monkeypatch, processor: KfpPipelineProcessor, metadata_dependencies: Dict[str, Any], tmpdir ): @@ -1174,6 +938,8 @@ def test_generate_pipeline_dsl_compile_pipeline_dsl_optional_elyra_properties( compiled_output_file = Path(tmpdir) / test_pipeline_file.with_suffix(".yaml").name compiled_output_file_name = str(compiled_output_file.absolute()) + print(f">>>> compiled_output_file_name: {compiled_output_file_name}") + # generate Python DSL pipeline_version = f"{pipeline.name}-0815" pipeline_instance_id = f"{pipeline.name}-{datetime.now().strftime('%m%d%H%M%S')}" @@ -1197,76 +963,65 @@ def test_generate_pipeline_dsl_compile_pipeline_dsl_optional_elyra_properties( # Load compiled output with open(compiled_output_file_name) as fh: - compiled_spec = yaml.safe_load(fh.read()) + spec_docs = list(yaml.safe_load_all(fh.read())) - # There should be two templates, one for the DAG and one for the generic node. - # Locate the one for the generic node and inspect its properties. - assert len(compiled_spec["spec"]["templates"]) == 2 - if compiled_spec["spec"]["templates"][0]["name"] == compiled_spec["spec"]["entrypoint"]: - node_template = compiled_spec["spec"]["templates"][1] - else: - node_template = compiled_spec["spec"]["templates"][0] + assert len(spec_docs) == 2 + assert "components" in spec_docs[0] + assert "platforms" in spec_docs[1] # # validate data volumes, if applicable expected_volume_mounts = op.elyra_props.get(MOUNTED_VOLUMES) if len(expected_volume_mounts) > 0: # There must be one or more 'volumeMounts' entry and one or more 'volumes' entry - assert node_template["container"].get("volumeMounts") is not None, node_template["container"] - assert node_template.get("volumes") is not None, compiled_spec["spec"] + assert ( + spec_docs[1]["platforms"]["kubernetes"]["deploymentSpec"]["executors"]["exec-run-a-file"].get("pvcMount") + is not None + ), spec_docs[1]["platforms"]["kubernetes"] + pvc_mounts = spec_docs[1]["platforms"]["kubernetes"]["deploymentSpec"]["executors"]["exec-run-a-file"][ + "pvcMount" + ] - assert len(node_template["container"]["volumeMounts"]) >= len(expected_volume_mounts) + assert len(pvc_mounts) >= len(expected_volume_mounts) for volume_mount in expected_volume_mounts: - for volumemount_entry in node_template["container"]["volumeMounts"]: + for volumemount_entry in pvc_mounts: entry_found = False if volumemount_entry["mountPath"] == volume_mount.path: - assert volumemount_entry["name"] == volume_mount.pvc_name - assert volumemount_entry.get("subPath", None) == volume_mount.sub_path - assert volumemount_entry.get("readOnly", None) == volume_mount.read_only - entry_found = True - break - assert ( - entry_found - ), f"Cannot find volume mount entry '{volume_mount.path}' in {node_template['container']['volumeMounts']}" - for volume_entry in node_template["volumes"]: - entry_found = False - if volume_entry["name"] == volume_mount.pvc_name: - assert volume_entry["persistentVolumeClaim"]["claimName"] == volume_mount.pvc_name + assert volumemount_entry["constant"] == volume_mount.pvc_name + # the following attributes are currently ignored in KFP v2. + # Once they are implemented, the code below needs to be updated accordingly. + # Reference: https://github.com/kubeflow/pipelines/blob/master/ + # kubernetes_platform/proto/kubernetes_executor_config.proto#L84 + # + # assert volumemount_entry.get("subPath", None) == volume_mount.sub_path + # assert volumemount_entry.get("readOnly", False) == volume_mount.read_only entry_found = True break - assert ( - entry_found - ), f"Cannot find volume entry '{volume_mount.path}' in {node_template['container']['volumeMounts']}" + assert entry_found, f"Cannot find volume mount entry '{volume_mount.path}' in {pvc_mounts}" # # validate custom shared memory size, if applicable custom_shared_mem_size = op.elyra_props.get(KUBERNETES_SHARED_MEM_SIZE) if custom_shared_mem_size: # There must be one 'volumeMounts' entry and one 'volumes' entry - assert node_template["container"].get("volumeMounts") is not None, node_template["container"] - assert node_template.get("volumes") is not None, compiled_spec["spec"] - for volumemount_entry in node_template["container"]["volumeMounts"]: + assert ( + spec_docs[1]["platforms"]["kubernetes"]["deploymentSpec"]["executors"]["exec-run-a-file"].get("pvcMount") + is not None + ), spec_docs[1]["platforms"]["kubernetes"] + pvc_mounts = spec_docs[1]["platforms"]["kubernetes"]["deploymentSpec"]["executors"]["exec-run-a-file"][ + "pvcMount" + ] + + for volumemount_entry in pvc_mounts: entry_found = False if volumemount_entry["mountPath"] == "/dev/shm": assert volumemount_entry["name"] == "shm" entry_found = True break - assert ( - entry_found - ), "Missing volume mount entry for shared memory size in {node_template['container']['volumeMounts']}" - for volume_entry in node_template["volumes"]: - entry_found = False - if volume_entry["name"] == "shm": - assert volume_entry["emptyDir"]["medium"] == "Memory" - assert ( - volume_entry["emptyDir"]["sizeLimit"] - == f"{custom_shared_mem_size.size}{custom_shared_mem_size.units}" - ) - entry_found = True - break - assert ( - entry_found - ), f"Missing volume entry for shm size '{volume_mount.path}' in {node_template['container']['volumeMounts']}" + assert entry_found, "Missing volume mount entry for shared memory size in {pvc_mounts}" + + """ + IMPORTANT: TODO: The following code needs to be updated to the KFP v2 once the feature is implemented. # # validate Kubernetes secrets, if applicable @@ -1332,6 +1087,7 @@ def test_generate_pipeline_dsl_compile_pipeline_dsl_optional_elyra_properties( f"in {node_template['tolerations']}" ) assert entry_found, not_found_msg + """ @pytest.mark.parametrize( @@ -1407,20 +1163,23 @@ def test_generate_pipeline_dsl_compile_pipeline_dsl_generic_components_data_exch # Load compiled output with open(compiled_output_file_name) as fh: - compiled_spec = yaml.safe_load(fh.read()) + compiled_spec_docs = list(yaml.safe_load_all(fh.read())) + + assert len(compiled_spec_docs) == 2 + assert "components" in compiled_spec_docs[0] + assert "platforms" in compiled_spec_docs[1] # There should be at least four templates, one for the DAG and three # for generic nodes. Each template spec for generic nodes is named # "run-a-file[-index]". The "-index" is added by the compiler to # guarantee uniqueness. - assert len(compiled_spec["spec"]["templates"]) >= 3 + executors = compiled_spec_docs[0]["deploymentSpec"]["executors"] + assert len(executors) >= 3 template_specs = {} - for node_template in compiled_spec["spec"]["templates"]: - if node_template["name"] == compiled_spec["spec"]["entrypoint"] or not node_template["name"].startswith( - "run-a-file" - ): + for node_template_name, node_template in executors.items(): + if node_template_name == executors or not node_template_name.startswith("exec-run-a-file"): continue - template_specs[node_template["name"]] = node_template + template_specs[node_template_name] = node_template # Iterate through sorted operations and verify that their inputs # and outputs are properly represented in their respective template @@ -1431,9 +1190,9 @@ def test_generate_pipeline_dsl_compile_pipeline_dsl_generic_components_data_exch # ignore custom nodes continue if template_index == 1: - template_name = "run-a-file" + template_name = "exec-run-a-file" else: - template_name = f"run-a-file-{template_index}" + template_name = f"exec-run-a-file-{template_index}" template_index = template_index + 1 # compare outputs if len(op.outputs) > 0: @@ -1473,6 +1232,9 @@ def test_generate_pipeline_dsl_compile_pipeline_dsl_generic_components_data_exch ], indirect=True, ) +@pytest.mark.skip( + reason="This test is not compatible with KFP v2: There is no `imagePullSecrets` in the generated YAML to be verified." # noqa: E501 +) def test_generate_pipeline_dsl_compile_pipeline_dsl_generic_components_pipeline_conf( monkeypatch, processor: KfpPipelineProcessor, metadata_dependencies: Dict[str, Any], tmpdir ): @@ -1627,24 +1389,20 @@ def test_generate_pipeline_dsl_compile_pipeline_dsl_generic_components_with_para # Load compiled workflow with open(compiled_output_file_name) as f: - compiled_spec = yaml.safe_load(f.read()) + compiled_spec_docs = list(yaml.safe_load_all(f.read())) + + assert len(compiled_spec_docs) == 2 + assert "components" in compiled_spec_docs[0] + assert "platforms" in compiled_spec_docs[1] # Test parameters appear as expected - yaml_pipeline_params = compiled_spec["spec"]["arguments"]["parameters"] + yaml_pipeline_params = compiled_spec_docs[0]["root"]["inputDefinitions"]["parameters"] # Only two parameters are referenced by a node in the pipeline, so only 2 should be present in YAML assert len(yaml_pipeline_params) == 2 # Assert params defined in YAML correspond to those defined by the Pipeline object - for param_from_yaml in yaml_pipeline_params: - param_name, param_value = param_from_yaml.get("name"), param_from_yaml.get("value") - assert any(param.name == param_name and str(param.value) == param_value for param in pipeline.parameters) - - yaml_node_params = compiled_spec["spec"]["templates"][1]["inputs"]["parameters"] - # Only two parameters are referenced by this node, so only 2 should be present as inputs - assert len(yaml_node_params) == 2 - # Assert params defined in YAML correspond to those defined by the Pipeline object - for param_from_yaml in yaml_node_params: - param_name = param_from_yaml.get("name") - assert any(param.name == param_name for param in pipeline.parameters) + for param_name, param_info in yaml_pipeline_params.items(): + param_value = param_info["defaultValue"] + assert any(param.name == param_name and param.value == param_value for param in pipeline.parameters) def test_generate_pipeline_dsl_compile_pipeline_dsl_custom_components_with_parameters( @@ -1772,21 +1530,15 @@ def test_generate_pipeline_dsl_compile_pipeline_dsl_custom_components_with_param print(compiled_spec) # Test parameters appear as expected - yaml_pipeline_params = compiled_spec["spec"]["arguments"]["parameters"] + yaml_pipeline_params = compiled_spec["root"]["inputDefinitions"]["parameters"] # Only two parameters are referenced by a node in the pipeline, so only 1 should be present in YAML assert len(yaml_pipeline_params) == 1 # Assert params defined in YAML correspond to those defined by the Pipeline object - for param_from_yaml in yaml_pipeline_params: - param_name, param_value = param_from_yaml.get("name"), param_from_yaml.get("value") - assert any(param.name == param_name and str(param.value) == param_value for param in pipeline.parameters) - - yaml_node_params = compiled_spec["spec"]["templates"][0]["inputs"]["parameters"] - # Only two parameters are referenced by this node, so only 1 should be present as input - assert len(yaml_node_params) == 1 - # Assert params defined in YAML correspond to those defined by the Pipeline object - for param_from_yaml in yaml_node_params: - param_name = param_from_yaml.get("name") - assert any(param.name == param_name for param in pipeline.parameters) + for param_from_yaml_key, param_from_yaml_value in yaml_pipeline_params.items(): + assert any( + param.name == param_from_yaml_key and str(param.value) == param_from_yaml_value["defaultValue"] + for param in pipeline.parameters + ) def test_kfp_invalid_pipeline_parameter_type(): diff --git a/elyra/tests/pipeline/resources/components/download_data.yaml b/elyra/tests/pipeline/resources/components/download_data.yaml index c8e8aab1b..74ccd263b 100644 --- a/elyra/tests/pipeline/resources/components/download_data.yaml +++ b/elyra/tests/pipeline/resources/components/download_data.yaml @@ -14,8 +14,8 @@ # name: Download data inputs: -- {name: Url, type: URI} -- {name: curl options, type: string, default: '--location', description: 'Additional options given to the curl program. See https://curl.haxx.se/docs/manpage.html'} +- {name: Url, type: String} +- {name: curl options, type: String, default: '--location', description: 'Additional options given to the curl program. See https://curl.haxx.se/docs/manpage.html'} outputs: - {name: Data} metadata: diff --git a/elyra/tests/pipeline/resources/components/filter_text.yaml b/elyra/tests/pipeline/resources/components/filter_text.yaml index 815d6711d..22259806f 100644 --- a/elyra/tests/pipeline/resources/components/filter_text.yaml +++ b/elyra/tests/pipeline/resources/components/filter_text.yaml @@ -12,17 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Component source location: https://raw.githubusercontent.com/kubeflow/pipelines/master/components/sample/Shell_script/component.yaml +# Component source location: https://raw.githubusercontent.com/kubeflow/pipelines/sdk-2.8.0/components/contrib/sample/Shell_script/component.yaml # Component details: Takes a text file and a regex pattern filter to produce a filtered text file -name: Filter text +name: Filter text using shell and grep inputs: -- {name: Text, optional: false, description: 'Path to file to be filtered'} -- {name: Pattern, optional: true, default: '.*', description: 'Regex pattern'} +- {name: Text, type: String} +- {name: Pattern, default: '.*', type: String} outputs: -- {name: Filtered text} +- {name: Filtered text, type: String} metadata: annotations: author: Alexey Volkov + canonical_location: 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/sample/Shell_script/component.yaml' implementation: container: image: alpine @@ -34,8 +35,5 @@ implementation: pattern=$1 filtered_text_path=$2 mkdir -p "$(dirname "$filtered_text_path")" - + grep "$pattern" < "$text_path" > "$filtered_text_path" - - {inputPath: Text} - - {inputValue: Pattern} - - {outputPath: Filtered text} diff --git a/elyra/tests/pipeline/resources/validation_pipelines/kf_inputpath_parameter.pipeline b/elyra/tests/pipeline/resources/validation_pipelines/kf_inputpath_parameter.pipeline index c9b3982dc..2dbde813d 100644 --- a/elyra/tests/pipeline/resources/validation_pipelines/kf_inputpath_parameter.pipeline +++ b/elyra/tests/pipeline/resources/validation_pipelines/kf_inputpath_parameter.pipeline @@ -16,7 +16,7 @@ "component_parameters": { "url": { "widget": "string", - "value": "https://raw.githubusercontent.com/kubeflow/pipelines/93fc34474bf989998cf19445149aca2847eee763/components/notebooks/samples/test_notebook.ipynb" + "value": "https://raw.githubusercontent.com/kubeflow/pipelines/sdk-2.8.0/components/contrib/notebooks/samples/test_notebook.ipynb" }, "curl_options": { "widget": "string", @@ -25,7 +25,7 @@ "output_data": "" }, "label": "", - "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/1.6.0/components/web/Download/component.yaml", + "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/sdk-2.8.0/components/contrib/web/Download/component-sdk-v2.yaml", "ui_data": { "label": "Download data", "image": "/static/elyra/kubeflow.svg", @@ -156,7 +156,7 @@ "output_hash": "" }, "label": "", - "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/1.6.0/components/basics/Calculate_hash/component.yaml", + "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/sdk-2.8.0/components/contrib/basics/Calculate_hash/component.yaml", "ui_data": { "label": "Calculate data hash", "image": "/static/elyra/kubeflow.svg", diff --git a/elyra/tests/pipeline/resources/validation_pipelines/kf_invalid_inputpath_missing_connection.pipeline b/elyra/tests/pipeline/resources/validation_pipelines/kf_invalid_inputpath_missing_connection.pipeline index 3082695ff..a320599d4 100644 --- a/elyra/tests/pipeline/resources/validation_pipelines/kf_invalid_inputpath_missing_connection.pipeline +++ b/elyra/tests/pipeline/resources/validation_pipelines/kf_invalid_inputpath_missing_connection.pipeline @@ -91,7 +91,7 @@ } }, "label": "", - "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/1.6.0/components/web/Download/component.yaml", + "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/sdk-2.8.0/components/contrib/web/Download/component-sdk-v2.yaml", "ui_data": { "label": "Download data", "image": "/static/elyra/kubeflow.svg", @@ -148,7 +148,7 @@ } }, "label": "", - "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/1.6.0/components/basics/Calculate_hash/component.yaml", + "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/sdk-2.8.0/components/contrib/basics/Calculate_hash/component.yaml", "ui_data": { "label": "Calculate data hash", "image": "/static/elyra/kubeflow.svg", diff --git a/elyra/tests/pipeline/resources/validation_pipelines/kf_invalid_inputpath_parameter.pipeline b/elyra/tests/pipeline/resources/validation_pipelines/kf_invalid_inputpath_parameter.pipeline index 2a7f1bde2..90d431cc7 100644 --- a/elyra/tests/pipeline/resources/validation_pipelines/kf_invalid_inputpath_parameter.pipeline +++ b/elyra/tests/pipeline/resources/validation_pipelines/kf_invalid_inputpath_parameter.pipeline @@ -16,7 +16,7 @@ "component_parameters": { "url": { "widget": "string", - "value": "https://raw.githubusercontent.com/kubeflow/pipelines/93fc34474bf989998cf19445149aca2847eee763/components/notebooks/samples/test_notebook.ipynb" + "value": "https://raw.githubusercontent.com/kubeflow/pipelines/sdk-2.8.0/components/contrib/notebooks/samples/test_notebook.ipynb" }, "curl_options": { "widget": "string", @@ -25,7 +25,7 @@ "output_data": "" }, "label": "", - "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/1.6.0/components/web/Download/component.yaml", + "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/sdk-2.8.0/components/contrib/web/Download/component-sdk-v2.yaml", "ui_data": { "label": "Download data", "image": "/static/elyra/kubeflow.svg", @@ -155,7 +155,7 @@ "output_hash": "" }, "label": "", - "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/1.6.0/components/basics/Calculate_hash/component.yaml", + "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/sdk-2.8.0/components/contrib/basics/Calculate_hash/component.yaml", "ui_data": { "label": "Calculate data hash", "image": "/static/elyra/kubeflow.svg", diff --git a/elyra/tests/pipeline/resources/validation_pipelines/kf_supernode_invalid_single_cycle.pipeline b/elyra/tests/pipeline/resources/validation_pipelines/kf_supernode_invalid_single_cycle.pipeline index 08b0febac..712b26176 100644 --- a/elyra/tests/pipeline/resources/validation_pipelines/kf_supernode_invalid_single_cycle.pipeline +++ b/elyra/tests/pipeline/resources/validation_pipelines/kf_supernode_invalid_single_cycle.pipeline @@ -19,7 +19,7 @@ "output_data": "" }, "label": "", - "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/1.6.0/components/web/Download/component.yaml", + "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/sdk-2.8.0/components/contrib/web/Download/component-sdk-v2.yaml", "ui_data": { "label": "Download data", "image": "data:image/svg+xml;utf8,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20276.93%20274.55%22%3E%3Cg%20id%3D%22Layer_2%22%20data-name%3D%22Layer%202%22%3E%3Cg%20id%3D%22Layer_1-2%22%20data-name%3D%22Layer%201%22%3E%3Cpath%20d%3D%22M95.9%2C62.15%2C100%2C164.25l73.75-94.12a6.79%2C6.79%2C0%2C0%2C1%2C9.6-1.11l46%2C36.92-15-65.61Z%22%20fill%3D%22%234279f4%22%2F%3E%3Cpolygon%20points%3D%22102.55%20182.98%20167.97%20182.98%20127.8%20150.75%20102.55%20182.98%22%20fill%3D%22%230028aa%22%2F%3E%3Cpolygon%20points%3D%22180.18%2083.92%20136.18%20140.06%20183.06%20177.67%20227.53%20121.91%20180.18%2083.92%22%20fill%3D%22%23014bd1%22%2F%3E%3Cpolygon%20points%3D%2283.56%2052.3%2083.57%2052.29%20122.26%203.77%2059.87%2033.82%2044.46%20101.33%2083.56%2052.3%22%20fill%3D%22%23bedcff%22%2F%3E%3Cpolygon%20points%3D%2245.32%20122.05%2086.76%20174.01%2082.81%2075.03%2045.32%20122.05%22%20fill%3D%22%236ca1ff%22%2F%3E%3Cpolygon%20points%3D%22202.31%2028.73%20142.65%200%20105.52%2046.56%20202.31%2028.73%22%20fill%3D%22%23a1c3ff%22%2F%3E%3Cpath%20d%3D%22M1.6%2C272V227.22H7.34v23.41l20.48-23.41h6.4l-17.39%2C19.7%2C19%2C25.07H29.1l-15.92-20.8-5.84%2C6.65V272Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M41.62%2C262.21V240h5.43v22.39a4.67%2C4.67%2C0%2C0%2C0%2C2.35%2C4.19%2C11%2C11%2C0%2C0%2C0%2C11%2C0%2C4.69%2C4.69%2C0%2C0%2C0%2C2.33-4.19V240h5.43v22.19a9.08%2C9.08%2C0%2C0%2C1-4.1%2C7.87%2C16.2%2C16.2%2C0%2C0%2C1-18.37%2C0A9.07%2C9.07%2C0%2C0%2C1%2C41.62%2C262.21Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M77.46%2C272V224h5.43v16.81a29.29%2C29.29%2C0%2C0%2C1%2C9.32-1.73%2C13.1%2C13.1%2C0%2C0%2C1%2C6.2%2C1.41%2C10.71%2C10.71%2C0%2C0%2C1%2C4.18%2C3.74%2C18.07%2C18.07%2C0%2C0%2C1%2C2.23%2C5.06%2C21.26%2C21.26%2C0%2C0%2C1%2C.73%2C5.58q0%2C8.43-4.38%2C12.79T87.35%2C272Zm5.43-4.87h4.55q6.77%2C0%2C9.72-2.95t3-9.51a14.21%2C14.21%2C0%2C0%2C0-2-7.52%2C6.55%2C6.55%2C0%2C0%2C0-6-3.22%2C24.73%2C24.73%2C0%2C0%2C0-9.25%2C1.54Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M112.36%2C255.94q0-7.71%2C4.09-12.3a13.75%2C13.75%2C0%2C0%2C1%2C10.8-4.59q13.35%2C0%2C13.36%2C18.86H117.79a12.3%2C12.3%2C0%2C0%2C0%2C2.9%2C7.07q2.59%2C3.11%2C7.9%2C3.1a24.92%2C24.92%2C0%2C0%2C0%2C10.55-2v5a27.74%2C27.74%2C0%2C0%2C1-9.86%2C1.87%2C19.83%2C19.83%2C0%2C0%2C1-7.7-1.37%2C13.31%2C13.31%2C0%2C0%2C1-5.28-3.76%2C16.21%2C16.21%2C0%2C0%2C1-3-5.38A20.84%2C20.84%2C0%2C0%2C1%2C112.36%2C255.94Zm5.62-2.12h17.26a14.91%2C14.91%2C0%2C0%2C0-2.37-7.12%2C6.44%2C6.44%2C0%2C0%2C0-5.62-2.78%2C8.2%2C8.2%2C0%2C0%2C0-6.21%2C2.72A12.07%2C12.07%2C0%2C0%2C0%2C118%2C253.82Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M147.32%2C244.89V240h5v-7.59a8.14%2C8.14%2C0%2C0%2C1%2C2.31-6.05%2C7.79%2C7.79%2C0%2C0%2C1%2C5.69-2.28h7.86V229h-5c-2.21%2C0-3.67.45-4.37%2C1.34s-1.06%2C2.55-1.06%2C5V240h8.46v4.87h-8.46V272h-5.44v-27.1Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M175.26%2C272V224h5.43v48Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M194.41%2C268.05a17.86%2C17.86%2C0%2C1%2C1%2C12.33%2C4.9A16.57%2C16.57%2C0%2C0%2C1%2C194.41%2C268.05Zm3.84-20.65a13.16%2C13.16%2C0%2C0%2C0%2C0%2C17.2%2C12.07%2C12.07%2C0%2C0%2C0%2C17%2C0%2C13.09%2C13.09%2C0%2C0%2C0%2C0-17.2%2C12.07%2C12.07%2C0%2C0%2C0-17%2C0Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M228.45%2C240h5.75l7.3%2C25.32L248.93%2C240h5.36l7.34%2C25.34L269%2C240h5.74L264.7%2C272h-6.12l-6.83-24.58L245%2C272h-6.47Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E", @@ -277,7 +277,7 @@ } }, "label": "", - "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/1.6.0/components/basics/Calculate_hash/component.yaml", + "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/sdk-2.8.0/components/contrib/basics/Calculate_hash/component.yaml", "ui_data": { "label": "Calculate data hash", "image": "data:image/svg+xml;utf8,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20276.93%20274.55%22%3E%3Cg%20id%3D%22Layer_2%22%20data-name%3D%22Layer%202%22%3E%3Cg%20id%3D%22Layer_1-2%22%20data-name%3D%22Layer%201%22%3E%3Cpath%20d%3D%22M95.9%2C62.15%2C100%2C164.25l73.75-94.12a6.79%2C6.79%2C0%2C0%2C1%2C9.6-1.11l46%2C36.92-15-65.61Z%22%20fill%3D%22%234279f4%22%2F%3E%3Cpolygon%20points%3D%22102.55%20182.98%20167.97%20182.98%20127.8%20150.75%20102.55%20182.98%22%20fill%3D%22%230028aa%22%2F%3E%3Cpolygon%20points%3D%22180.18%2083.92%20136.18%20140.06%20183.06%20177.67%20227.53%20121.91%20180.18%2083.92%22%20fill%3D%22%23014bd1%22%2F%3E%3Cpolygon%20points%3D%2283.56%2052.3%2083.57%2052.29%20122.26%203.77%2059.87%2033.82%2044.46%20101.33%2083.56%2052.3%22%20fill%3D%22%23bedcff%22%2F%3E%3Cpolygon%20points%3D%2245.32%20122.05%2086.76%20174.01%2082.81%2075.03%2045.32%20122.05%22%20fill%3D%22%236ca1ff%22%2F%3E%3Cpolygon%20points%3D%22202.31%2028.73%20142.65%200%20105.52%2046.56%20202.31%2028.73%22%20fill%3D%22%23a1c3ff%22%2F%3E%3Cpath%20d%3D%22M1.6%2C272V227.22H7.34v23.41l20.48-23.41h6.4l-17.39%2C19.7%2C19%2C25.07H29.1l-15.92-20.8-5.84%2C6.65V272Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M41.62%2C262.21V240h5.43v22.39a4.67%2C4.67%2C0%2C0%2C0%2C2.35%2C4.19%2C11%2C11%2C0%2C0%2C0%2C11%2C0%2C4.69%2C4.69%2C0%2C0%2C0%2C2.33-4.19V240h5.43v22.19a9.08%2C9.08%2C0%2C0%2C1-4.1%2C7.87%2C16.2%2C16.2%2C0%2C0%2C1-18.37%2C0A9.07%2C9.07%2C0%2C0%2C1%2C41.62%2C262.21Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M77.46%2C272V224h5.43v16.81a29.29%2C29.29%2C0%2C0%2C1%2C9.32-1.73%2C13.1%2C13.1%2C0%2C0%2C1%2C6.2%2C1.41%2C10.71%2C10.71%2C0%2C0%2C1%2C4.18%2C3.74%2C18.07%2C18.07%2C0%2C0%2C1%2C2.23%2C5.06%2C21.26%2C21.26%2C0%2C0%2C1%2C.73%2C5.58q0%2C8.43-4.38%2C12.79T87.35%2C272Zm5.43-4.87h4.55q6.77%2C0%2C9.72-2.95t3-9.51a14.21%2C14.21%2C0%2C0%2C0-2-7.52%2C6.55%2C6.55%2C0%2C0%2C0-6-3.22%2C24.73%2C24.73%2C0%2C0%2C0-9.25%2C1.54Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M112.36%2C255.94q0-7.71%2C4.09-12.3a13.75%2C13.75%2C0%2C0%2C1%2C10.8-4.59q13.35%2C0%2C13.36%2C18.86H117.79a12.3%2C12.3%2C0%2C0%2C0%2C2.9%2C7.07q2.59%2C3.11%2C7.9%2C3.1a24.92%2C24.92%2C0%2C0%2C0%2C10.55-2v5a27.74%2C27.74%2C0%2C0%2C1-9.86%2C1.87%2C19.83%2C19.83%2C0%2C0%2C1-7.7-1.37%2C13.31%2C13.31%2C0%2C0%2C1-5.28-3.76%2C16.21%2C16.21%2C0%2C0%2C1-3-5.38A20.84%2C20.84%2C0%2C0%2C1%2C112.36%2C255.94Zm5.62-2.12h17.26a14.91%2C14.91%2C0%2C0%2C0-2.37-7.12%2C6.44%2C6.44%2C0%2C0%2C0-5.62-2.78%2C8.2%2C8.2%2C0%2C0%2C0-6.21%2C2.72A12.07%2C12.07%2C0%2C0%2C0%2C118%2C253.82Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M147.32%2C244.89V240h5v-7.59a8.14%2C8.14%2C0%2C0%2C1%2C2.31-6.05%2C7.79%2C7.79%2C0%2C0%2C1%2C5.69-2.28h7.86V229h-5c-2.21%2C0-3.67.45-4.37%2C1.34s-1.06%2C2.55-1.06%2C5V240h8.46v4.87h-8.46V272h-5.44v-27.1Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M175.26%2C272V224h5.43v48Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M194.41%2C268.05a17.86%2C17.86%2C0%2C1%2C1%2C12.33%2C4.9A16.57%2C16.57%2C0%2C0%2C1%2C194.41%2C268.05Zm3.84-20.65a13.16%2C13.16%2C0%2C0%2C0%2C0%2C17.2%2C12.07%2C12.07%2C0%2C0%2C0%2C17%2C0%2C13.09%2C13.09%2C0%2C0%2C0%2C0-17.2%2C12.07%2C12.07%2C0%2C0%2C0-17%2C0Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M228.45%2C240h5.75l7.3%2C25.32L248.93%2C240h5.36l7.34%2C25.34L269%2C240h5.74L264.7%2C272h-6.12l-6.83-24.58L245%2C272h-6.47Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E", diff --git a/elyra/tests/pipeline/resources/validation_pipelines/kf_supernode_valid.pipeline b/elyra/tests/pipeline/resources/validation_pipelines/kf_supernode_valid.pipeline index 3429ecb4b..bdf4d2c63 100644 --- a/elyra/tests/pipeline/resources/validation_pipelines/kf_supernode_valid.pipeline +++ b/elyra/tests/pipeline/resources/validation_pipelines/kf_supernode_valid.pipeline @@ -19,7 +19,7 @@ "output_data": "" }, "label": "", - "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/1.6.0/components/web/Download/component.yaml", + "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/sdk-2.8.0/components/contrib/web/Download/component-sdk-v2.yaml", "ui_data": { "label": "Download data", "image": "data:image/svg+xml;utf8,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20276.93%20274.55%22%3E%3Cg%20id%3D%22Layer_2%22%20data-name%3D%22Layer%202%22%3E%3Cg%20id%3D%22Layer_1-2%22%20data-name%3D%22Layer%201%22%3E%3Cpath%20d%3D%22M95.9%2C62.15%2C100%2C164.25l73.75-94.12a6.79%2C6.79%2C0%2C0%2C1%2C9.6-1.11l46%2C36.92-15-65.61Z%22%20fill%3D%22%234279f4%22%2F%3E%3Cpolygon%20points%3D%22102.55%20182.98%20167.97%20182.98%20127.8%20150.75%20102.55%20182.98%22%20fill%3D%22%230028aa%22%2F%3E%3Cpolygon%20points%3D%22180.18%2083.92%20136.18%20140.06%20183.06%20177.67%20227.53%20121.91%20180.18%2083.92%22%20fill%3D%22%23014bd1%22%2F%3E%3Cpolygon%20points%3D%2283.56%2052.3%2083.57%2052.29%20122.26%203.77%2059.87%2033.82%2044.46%20101.33%2083.56%2052.3%22%20fill%3D%22%23bedcff%22%2F%3E%3Cpolygon%20points%3D%2245.32%20122.05%2086.76%20174.01%2082.81%2075.03%2045.32%20122.05%22%20fill%3D%22%236ca1ff%22%2F%3E%3Cpolygon%20points%3D%22202.31%2028.73%20142.65%200%20105.52%2046.56%20202.31%2028.73%22%20fill%3D%22%23a1c3ff%22%2F%3E%3Cpath%20d%3D%22M1.6%2C272V227.22H7.34v23.41l20.48-23.41h6.4l-17.39%2C19.7%2C19%2C25.07H29.1l-15.92-20.8-5.84%2C6.65V272Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M41.62%2C262.21V240h5.43v22.39a4.67%2C4.67%2C0%2C0%2C0%2C2.35%2C4.19%2C11%2C11%2C0%2C0%2C0%2C11%2C0%2C4.69%2C4.69%2C0%2C0%2C0%2C2.33-4.19V240h5.43v22.19a9.08%2C9.08%2C0%2C0%2C1-4.1%2C7.87%2C16.2%2C16.2%2C0%2C0%2C1-18.37%2C0A9.07%2C9.07%2C0%2C0%2C1%2C41.62%2C262.21Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M77.46%2C272V224h5.43v16.81a29.29%2C29.29%2C0%2C0%2C1%2C9.32-1.73%2C13.1%2C13.1%2C0%2C0%2C1%2C6.2%2C1.41%2C10.71%2C10.71%2C0%2C0%2C1%2C4.18%2C3.74%2C18.07%2C18.07%2C0%2C0%2C1%2C2.23%2C5.06%2C21.26%2C21.26%2C0%2C0%2C1%2C.73%2C5.58q0%2C8.43-4.38%2C12.79T87.35%2C272Zm5.43-4.87h4.55q6.77%2C0%2C9.72-2.95t3-9.51a14.21%2C14.21%2C0%2C0%2C0-2-7.52%2C6.55%2C6.55%2C0%2C0%2C0-6-3.22%2C24.73%2C24.73%2C0%2C0%2C0-9.25%2C1.54Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M112.36%2C255.94q0-7.71%2C4.09-12.3a13.75%2C13.75%2C0%2C0%2C1%2C10.8-4.59q13.35%2C0%2C13.36%2C18.86H117.79a12.3%2C12.3%2C0%2C0%2C0%2C2.9%2C7.07q2.59%2C3.11%2C7.9%2C3.1a24.92%2C24.92%2C0%2C0%2C0%2C10.55-2v5a27.74%2C27.74%2C0%2C0%2C1-9.86%2C1.87%2C19.83%2C19.83%2C0%2C0%2C1-7.7-1.37%2C13.31%2C13.31%2C0%2C0%2C1-5.28-3.76%2C16.21%2C16.21%2C0%2C0%2C1-3-5.38A20.84%2C20.84%2C0%2C0%2C1%2C112.36%2C255.94Zm5.62-2.12h17.26a14.91%2C14.91%2C0%2C0%2C0-2.37-7.12%2C6.44%2C6.44%2C0%2C0%2C0-5.62-2.78%2C8.2%2C8.2%2C0%2C0%2C0-6.21%2C2.72A12.07%2C12.07%2C0%2C0%2C0%2C118%2C253.82Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M147.32%2C244.89V240h5v-7.59a8.14%2C8.14%2C0%2C0%2C1%2C2.31-6.05%2C7.79%2C7.79%2C0%2C0%2C1%2C5.69-2.28h7.86V229h-5c-2.21%2C0-3.67.45-4.37%2C1.34s-1.06%2C2.55-1.06%2C5V240h8.46v4.87h-8.46V272h-5.44v-27.1Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M175.26%2C272V224h5.43v48Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M194.41%2C268.05a17.86%2C17.86%2C0%2C1%2C1%2C12.33%2C4.9A16.57%2C16.57%2C0%2C0%2C1%2C194.41%2C268.05Zm3.84-20.65a13.16%2C13.16%2C0%2C0%2C0%2C0%2C17.2%2C12.07%2C12.07%2C0%2C0%2C0%2C17%2C0%2C13.09%2C13.09%2C0%2C0%2C0%2C0-17.2%2C12.07%2C12.07%2C0%2C0%2C0-17%2C0Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M228.45%2C240h5.75l7.3%2C25.32L248.93%2C240h5.36l7.34%2C25.34L269%2C240h5.74L264.7%2C272h-6.12l-6.83-24.58L245%2C272h-6.47Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E", @@ -263,7 +263,7 @@ } }, "label": "", - "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/1.6.0/components/basics/Calculate_hash/component.yaml", + "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/sdk-2.8.0/components/contrib/basics/Calculate_hash/component.yaml", "ui_data": { "label": "Calculate data hash", "image": "data:image/svg+xml;utf8,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%20276.93%20274.55%22%3E%3Cg%20id%3D%22Layer_2%22%20data-name%3D%22Layer%202%22%3E%3Cg%20id%3D%22Layer_1-2%22%20data-name%3D%22Layer%201%22%3E%3Cpath%20d%3D%22M95.9%2C62.15%2C100%2C164.25l73.75-94.12a6.79%2C6.79%2C0%2C0%2C1%2C9.6-1.11l46%2C36.92-15-65.61Z%22%20fill%3D%22%234279f4%22%2F%3E%3Cpolygon%20points%3D%22102.55%20182.98%20167.97%20182.98%20127.8%20150.75%20102.55%20182.98%22%20fill%3D%22%230028aa%22%2F%3E%3Cpolygon%20points%3D%22180.18%2083.92%20136.18%20140.06%20183.06%20177.67%20227.53%20121.91%20180.18%2083.92%22%20fill%3D%22%23014bd1%22%2F%3E%3Cpolygon%20points%3D%2283.56%2052.3%2083.57%2052.29%20122.26%203.77%2059.87%2033.82%2044.46%20101.33%2083.56%2052.3%22%20fill%3D%22%23bedcff%22%2F%3E%3Cpolygon%20points%3D%2245.32%20122.05%2086.76%20174.01%2082.81%2075.03%2045.32%20122.05%22%20fill%3D%22%236ca1ff%22%2F%3E%3Cpolygon%20points%3D%22202.31%2028.73%20142.65%200%20105.52%2046.56%20202.31%2028.73%22%20fill%3D%22%23a1c3ff%22%2F%3E%3Cpath%20d%3D%22M1.6%2C272V227.22H7.34v23.41l20.48-23.41h6.4l-17.39%2C19.7%2C19%2C25.07H29.1l-15.92-20.8-5.84%2C6.65V272Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M41.62%2C262.21V240h5.43v22.39a4.67%2C4.67%2C0%2C0%2C0%2C2.35%2C4.19%2C11%2C11%2C0%2C0%2C0%2C11%2C0%2C4.69%2C4.69%2C0%2C0%2C0%2C2.33-4.19V240h5.43v22.19a9.08%2C9.08%2C0%2C0%2C1-4.1%2C7.87%2C16.2%2C16.2%2C0%2C0%2C1-18.37%2C0A9.07%2C9.07%2C0%2C0%2C1%2C41.62%2C262.21Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M77.46%2C272V224h5.43v16.81a29.29%2C29.29%2C0%2C0%2C1%2C9.32-1.73%2C13.1%2C13.1%2C0%2C0%2C1%2C6.2%2C1.41%2C10.71%2C10.71%2C0%2C0%2C1%2C4.18%2C3.74%2C18.07%2C18.07%2C0%2C0%2C1%2C2.23%2C5.06%2C21.26%2C21.26%2C0%2C0%2C1%2C.73%2C5.58q0%2C8.43-4.38%2C12.79T87.35%2C272Zm5.43-4.87h4.55q6.77%2C0%2C9.72-2.95t3-9.51a14.21%2C14.21%2C0%2C0%2C0-2-7.52%2C6.55%2C6.55%2C0%2C0%2C0-6-3.22%2C24.73%2C24.73%2C0%2C0%2C0-9.25%2C1.54Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M112.36%2C255.94q0-7.71%2C4.09-12.3a13.75%2C13.75%2C0%2C0%2C1%2C10.8-4.59q13.35%2C0%2C13.36%2C18.86H117.79a12.3%2C12.3%2C0%2C0%2C0%2C2.9%2C7.07q2.59%2C3.11%2C7.9%2C3.1a24.92%2C24.92%2C0%2C0%2C0%2C10.55-2v5a27.74%2C27.74%2C0%2C0%2C1-9.86%2C1.87%2C19.83%2C19.83%2C0%2C0%2C1-7.7-1.37%2C13.31%2C13.31%2C0%2C0%2C1-5.28-3.76%2C16.21%2C16.21%2C0%2C0%2C1-3-5.38A20.84%2C20.84%2C0%2C0%2C1%2C112.36%2C255.94Zm5.62-2.12h17.26a14.91%2C14.91%2C0%2C0%2C0-2.37-7.12%2C6.44%2C6.44%2C0%2C0%2C0-5.62-2.78%2C8.2%2C8.2%2C0%2C0%2C0-6.21%2C2.72A12.07%2C12.07%2C0%2C0%2C0%2C118%2C253.82Z%22%20fill%3D%22%234279f4%22%20stroke%3D%22%234279f4%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M147.32%2C244.89V240h5v-7.59a8.14%2C8.14%2C0%2C0%2C1%2C2.31-6.05%2C7.79%2C7.79%2C0%2C0%2C1%2C5.69-2.28h7.86V229h-5c-2.21%2C0-3.67.45-4.37%2C1.34s-1.06%2C2.55-1.06%2C5V240h8.46v4.87h-8.46V272h-5.44v-27.1Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M175.26%2C272V224h5.43v48Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M194.41%2C268.05a17.86%2C17.86%2C0%2C1%2C1%2C12.33%2C4.9A16.57%2C16.57%2C0%2C0%2C1%2C194.41%2C268.05Zm3.84-20.65a13.16%2C13.16%2C0%2C0%2C0%2C0%2C17.2%2C12.07%2C12.07%2C0%2C0%2C0%2C17%2C0%2C13.09%2C13.09%2C0%2C0%2C0%2C0-17.2%2C12.07%2C12.07%2C0%2C0%2C0-17%2C0Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3Cpath%20d%3D%22M228.45%2C240h5.75l7.3%2C25.32L248.93%2C240h5.36l7.34%2C25.34L269%2C240h5.74L264.7%2C272h-6.12l-6.83-24.58L245%2C272h-6.47Z%22%20fill%3D%22%230028aa%22%20stroke%3D%22%230028aa%22%20stroke-miterlimit%3D%2210%22%20stroke-width%3D%223.2%22%2F%3E%3C%2Fg%3E%3C%2Fg%3E%3C%2Fsvg%3E", diff --git a/elyra/tests/pipeline/resources/validation_pipelines/kf_with_parameters.pipeline b/elyra/tests/pipeline/resources/validation_pipelines/kf_with_parameters.pipeline index 9e7b0d52b..65dbe1278 100644 --- a/elyra/tests/pipeline/resources/validation_pipelines/kf_with_parameters.pipeline +++ b/elyra/tests/pipeline/resources/validation_pipelines/kf_with_parameters.pipeline @@ -47,7 +47,7 @@ "output_data": "" }, "label": "", - "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/1.6.0/components/web/Download/component.yaml", + "component_source": "https://raw.githubusercontent.com/kubeflow/pipelines/sdk-2.8.0/components/contrib/web/Download/component-sdk-v2.yaml", "ui_data": { "label": "Download data", "image": "/static/elyra/kubeflow.svg", diff --git a/etc/docker/elyra_development/requirements.yml b/etc/docker/elyra_development/requirements.yml index d6e4aa47a..0fc3ccf58 100644 --- a/etc/docker/elyra_development/requirements.yml +++ b/etc/docker/elyra_development/requirements.yml @@ -1,5 +1,5 @@ name: elyra-env dependencies: - - python>=3.8 + - python>=3.11 - pip: - - jupyterlab>=3.4.6 + - jupyterlab==4.2.5 diff --git a/etc/generic/requirements-elyra.txt b/etc/generic/requirements-elyra.txt index 313c6a781..07ab52e72 100644 --- a/etc/generic/requirements-elyra.txt +++ b/etc/generic/requirements-elyra.txt @@ -1,22 +1,22 @@ # This is a comprehensive list of python dependencies that Elyra requires to execute Jupyter notebooks. -ipykernel==6.13.0 -ipython==8.10.0 +ipykernel==6.25.2 +ipython==8.12.3 ipython-genutils==0.2.0 -jinja2==3.0.3 -jupyter-client==7.3.1 -jupyter-core==4.11.2 +jinja2==3.1.4 +jupyter-client==7.4.9 +jupyter-core==5.3.1 MarkupSafe==2.1.1 -minio==7.1.8 +minio==7.1.15 nbclient==0.6.3 -nbconvert==6.5.1 +nbconvert==7.1.0 nbformat==5.4.0 -papermill==2.3.4 +papermill==2.6.0 pyzmq==24.0.1 -prompt-toolkit==3.0.30 -requests==2.31.0 -tornado==6.3.3 -traitlets==5.1.1 -urllib3==1.26.18 +prompt-toolkit==3.0.43 +requests==2.32.3 +tornado==6.4.1 +traitlets==5.10.0 +urllib3==1.26.19 # # These excluded are transitive dependencies of the included python packages. #ansiwrap==0.8.4 diff --git a/etc/scripts/generate-make-graph.ts b/etc/scripts/generate-make-graph.ts index 9068990d1..adc1912e4 100644 --- a/etc/scripts/generate-make-graph.ts +++ b/etc/scripts/generate-make-graph.ts @@ -63,7 +63,7 @@ make.stdout.on('data', (data: Buffer) => { graph.push({ type: 'target', name, - depth: depthString.length / 2, + depth: depthString.length / 2 }); continue; } @@ -79,7 +79,7 @@ make.stdout.on('data', (data: Buffer) => { depth = depthString.length / 2; graph.push({ type: 'end', - depth, + depth }); depth = undefined; continue; @@ -89,7 +89,7 @@ make.stdout.on('data', (data: Buffer) => { graph.push({ type: 'code', value: msg.toString(), - depth, + depth }); continue; } @@ -107,7 +107,7 @@ const printGraph = (): void => { const spacer = ' '.repeat(cellWidth - g.name.length - 2); console.log(`${padLeft}┌${bar}┐${padRight}`); console.log( - `${padLeft}│ ${chalk.cyan.bold(g.name)}${spacer} |${padRight}`, + `${padLeft}│ ${chalk.cyan.bold(g.name)}${spacer} |${padRight}` ); continue; } diff --git a/labextensions/elyra_code_snippet_extension/__init__.py b/labextensions/elyra_code_snippet_extension/__init__.py new file mode 100644 index 000000000..e50fe87fd --- /dev/null +++ b/labextensions/elyra_code_snippet_extension/__init__.py @@ -0,0 +1,14 @@ +try: + from ._version import __version__ +except ImportError: + # Fallback when using the package in dev mode without installing + # in editable mode with pip. It is highly recommended to install + # the package from a stable release or in editable mode: https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs + import warnings + + warnings.warn("Importing 'elyra_code_snippet_extension' outside a proper installation.") + __version__ = "dev" + + +def _jupyter_labextension_paths(): + return [{"src": "labextension", "dest": "@elyra/code-snippet-extension"}] diff --git a/labextensions/elyra_metadata_common/__init__.py b/labextensions/elyra_metadata_common/__init__.py new file mode 100644 index 000000000..ada029c9c --- /dev/null +++ b/labextensions/elyra_metadata_common/__init__.py @@ -0,0 +1,14 @@ +try: + from ._version import __version__ +except ImportError: + # Fallback when using the package in dev mode without installing + # in editable mode with pip. It is highly recommended to install + # the package from a stable release or in editable mode: https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs + import warnings + + warnings.warn("Importing 'elyra_metadata_comomn' outside a proper installation.") + __version__ = "dev" + + +def _jupyter_labextension_paths(): + return [{"src": "labextension", "dest": "@elyra/metadata-common"}] diff --git a/labextensions/elyra_metadata_extension/__init__.py b/labextensions/elyra_metadata_extension/__init__.py new file mode 100644 index 000000000..c9ad80568 --- /dev/null +++ b/labextensions/elyra_metadata_extension/__init__.py @@ -0,0 +1,14 @@ +try: + from ._version import __version__ +except ImportError: + # Fallback when using the package in dev mode without installing + # in editable mode with pip. It is highly recommended to install + # the package from a stable release or in editable mode: https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs + import warnings + + warnings.warn("Importing 'elyra_metadata_extension' outside a proper installation.") + __version__ = "dev" + + +def _jupyter_labextension_paths(): + return [{"src": "labextension", "dest": "@elyra/metadata-extension"}] diff --git a/labextensions/elyra_pipeline_editor_extension/__init__.py b/labextensions/elyra_pipeline_editor_extension/__init__.py new file mode 100644 index 000000000..9321d169b --- /dev/null +++ b/labextensions/elyra_pipeline_editor_extension/__init__.py @@ -0,0 +1,14 @@ +try: + from ._version import __version__ +except ImportError: + # Fallback when using the package in dev mode without installing + # in editable mode with pip. It is highly recommended to install + # the package from a stable release or in editable mode: https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs + import warnings + + warnings.warn("Importing 'elyra_pipeline_editor_extension' outside a proper installation.") + __version__ = "dev" + + +def _jupyter_labextension_paths(): + return [{"src": "labextension", "dest": "@elyra/pipeline-editor-extension"}] diff --git a/labextensions/elyra_python_editor_extension/__init__.py b/labextensions/elyra_python_editor_extension/__init__.py new file mode 100644 index 000000000..6aaf2988f --- /dev/null +++ b/labextensions/elyra_python_editor_extension/__init__.py @@ -0,0 +1,14 @@ +try: + from ._version import __version__ +except ImportError: + # Fallback when using the package in dev mode without installing + # in editable mode with pip. It is highly recommended to install + # the package from a stable release or in editable mode: https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs + import warnings + + warnings.warn("Importing 'elyra_python_editor_extension' outside a proper installation.") + __version__ = "dev" + + +def _jupyter_labextension_paths(): + return [{"src": "labextension", "dest": "@elyra/python-editor-extension"}] diff --git a/labextensions/elyra_script_debugger_extension/__init__.py b/labextensions/elyra_script_debugger_extension/__init__.py new file mode 100644 index 000000000..2e078f4f5 --- /dev/null +++ b/labextensions/elyra_script_debugger_extension/__init__.py @@ -0,0 +1,14 @@ +try: + from ._version import __version__ +except ImportError: + # Fallback when using the package in dev mode without installing + # in editable mode with pip. It is highly recommended to install + # the package from a stable release or in editable mode: https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs + import warnings + + warnings.warn("Importing 'elyra_script_debugger_extension' outside a proper installation.") + __version__ = "dev" + + +def _jupyter_labextension_paths(): + return [{"src": "labextension", "dest": "@elyra/script-debugger-extension"}] diff --git a/labextensions/elyra_script_editor/__init__.py b/labextensions/elyra_script_editor/__init__.py new file mode 100644 index 000000000..da999da45 --- /dev/null +++ b/labextensions/elyra_script_editor/__init__.py @@ -0,0 +1,14 @@ +try: + from ._version import __version__ +except ImportError: + # Fallback when using the package in dev mode without installing + # in editable mode with pip. It is highly recommended to install + # the package from a stable release or in editable mode: https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs + import warnings + + warnings.warn("Importing 'elyra_script_editor' outside a proper installation.") + __version__ = "dev" + + +def _jupyter_labextension_paths(): + return [{"src": "labextension", "dest": "@elyra/script-editor"}] diff --git a/labextensions/elyra_services/__init__.py b/labextensions/elyra_services/__init__.py new file mode 100644 index 000000000..928404622 --- /dev/null +++ b/labextensions/elyra_services/__init__.py @@ -0,0 +1,14 @@ +try: + from ._version import __version__ +except ImportError: + # Fallback when using the package in dev mode without installing + # in editable mode with pip. It is highly recommended to install + # the package from a stable release or in editable mode: https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs + import warnings + + warnings.warn("Importing 'elyra_services' outside a proper installation.") + __version__ = "dev" + + +def _jupyter_labextension_paths(): + return [{"src": "labextension", "dest": "@elyra/services"}] diff --git a/labextensions/elyra_theme_extension/__init__.py b/labextensions/elyra_theme_extension/__init__.py new file mode 100644 index 000000000..bc4647318 --- /dev/null +++ b/labextensions/elyra_theme_extension/__init__.py @@ -0,0 +1,14 @@ +try: + from ._version import __version__ +except ImportError: + # Fallback when using the package in dev mode without installing + # in editable mode with pip. It is highly recommended to install + # the package from a stable release or in editable mode: https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs + import warnings + + warnings.warn("Importing 'elyra_theme_extension' outside a proper installation.") + __version__ = "dev" + + +def _jupyter_labextension_paths(): + return [{"src": "labextension", "dest": "@elyra/theme-extension"}] diff --git a/labextensions/elyra_ui_components/__init__.py b/labextensions/elyra_ui_components/__init__.py new file mode 100644 index 000000000..294cd1806 --- /dev/null +++ b/labextensions/elyra_ui_components/__init__.py @@ -0,0 +1,14 @@ +try: + from ._version import __version__ +except ImportError: + # Fallback when using the package in dev mode without installing + # in editable mode with pip. It is highly recommended to install + # the package from a stable release or in editable mode: https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs + import warnings + + warnings.warn("Importing 'elyra_ui_components' outside a proper installation.") + __version__ = "dev" + + +def _jupyter_labextension_paths(): + return [{"src": "labextension", "dest": "@elyra/ui-components"}] diff --git a/package.json b/package.json index ddeb5d53f..a26501779 100644 --- a/package.json +++ b/package.json @@ -45,11 +45,11 @@ "yjs": "^13.5.40" }, "devDependencies": { - "@4tw/cypress-drag-drop": "^1.3.1", "@cypress/webpack-preprocessor": "^5.5.0", - "@jupyterlab/testutils": "^4.0.6", + "@glen/jest-raw-loader": "^2.0.0", + "@jupyterlab/testutils": "^4.2.5", "@testing-library/cypress": "^7.0.4", - "@types/jest": "^26.0.20", + "@types/jest": "^29.2.0", "@types/lodash": "^4.14.170", "@types/node": "^15.0.1", "@types/react": "^18.0.26", @@ -57,6 +57,7 @@ "@typescript-eslint/eslint-plugin": "~6.13.2", "@typescript-eslint/parser": "~6.13.2", "cypress": "^6.2.0", + "cypress-real-events": "^1.13.0", "eslint": "~8.55.0", "eslint-config-prettier": "~9.1.0", "eslint-plugin-cypress": "^2.15.1", @@ -68,21 +69,19 @@ "eslint-plugin-react": "~7.33.2", "eslint-plugin-react-hooks": "^4.6.0", "husky": "^8.0.3", - "jest": "^29.7.0", - "jest-raw-loader": "^1.0.1", + "jest": "^29.2.0", "lerna": "^8.0.1", "lint-staged": "^15.2.0", "npm-run-all": "^4.1.5", "prettier": "^3.1.1", "rimraf": "~5.0.5", "start-server-and-test": "^2.0.3", - "ts-jest": "^29.1.1", + "ts-jest": "^29.2.5", "ts-loader": "^9.5.1", "ts-node": "^10.9.2", "typedoc": "~0.24.7", "typedoc-plugin-mdn-links": "^3.0.3", "typescript": "~5.1.6", "webpack": "^5.76.1" - }, - "packageManager": "yarn@3.5.0" + } } diff --git a/packages/code-snippet/install.json b/packages/code-snippet/install.json new file mode 100644 index 000000000..622ced017 --- /dev/null +++ b/packages/code-snippet/install.json @@ -0,0 +1,5 @@ +{ + "packageManager": "python", + "packageName": "elyra_code_snippet_extension", + "uninstallInstructions": "Use your Python package manager (pip, conda, etc.) to uninstall the package elyra_code_snippet_extension" +} diff --git a/packages/code-snippet/package.json b/packages/code-snippet/package.json index ec8c59748..eebcf1bc5 100644 --- a/packages/code-snippet/package.json +++ b/packages/code-snippet/package.json @@ -25,40 +25,50 @@ "style/**/*.{css,eot,gif,html,jpg,json,png,svg,woff2,ttf}" ], "scripts": { - "build": "jlpm run build:lib && jlpm run build:labextension:dev", - "build:prod": "jlpm run build:lib && jlpm run build:labextension", - "build:lib": "tsc", + "build": "jlpm build:lib && jlpm build:labextension:dev", + "build:prod": "jlpm clean && jlpm build:lib:prod && jlpm build:labextension", "build:labextension": "jupyter labextension build .", "build:labextension:dev": "jupyter labextension build --development True .", - "clean": "rimraf lib tsconfig.tsbuildinfo ../../build/labextensions/@elyra/code-snippet-extension", - "lab:dev": "jupyter labextension develop --overwrite ../../build/labextensions/@elyra/code-snippet-extension", - "dist": "npm pack .", - "prepare": "npm run build", + "build:lib": "tsc --sourceMap", + "build:lib:prod": "tsc", + "clean": "jlpm clean:lib", + "clean:lib": "rimraf lib tsconfig.tsbuildinfo", + "clean:lintcache": "rimraf .eslintcache .stylelintcache", + "clean:labextension": "rimraf ../../../../labextensions/elyra_code_snippet_extension/labextension ../../../../labextensions/elyra_code_snippet_extension/_version.py", + "clean:all": "jlpm clean:lib && jlpm clean:labextension && jlpm clean:lintcache", + "eslint": "jlpm eslint:check --fix", + "eslint:check": "eslint . --cache --ext .ts,.tsx", + "install:extension": "jlpm build", + "lint": "jlpm stylelint && jlpm prettier && jlpm eslint", + "lint:check": "jlpm stylelint:check && jlpm prettier:check && jlpm eslint:check", + "prettier": "jlpm prettier:base --write --list-different", + "prettier:base": "prettier \"**/*{.ts,.tsx,.js,.jsx,.css,.json,.md}\"", + "prettier:check": "jlpm prettier:base --check", + "stylelint": "jlpm stylelint:check --fix", + "stylelint:check": "stylelint --cache \"style/**/*.css\"", + "test": "jest --coverage --passWithNoTests", "watch": "run-p watch:src watch:labextension", - "watch:src": "tsc -w", - "watch:labextension": "jupyter labextension watch .", - "lab:install": "jupyter labextension install --no-build", - "lab:uninstall": "jupyter labextension uninstall --no-build", - "link:dev": "yarn link @jupyterlab/builder", - "unlink:dev": "yarn unlink @jupyterlab/builder" + "watch:src": "tsc -w --sourceMap", + "watch:labextension": "jupyter labextension watch ." }, "dependencies": { "@elyra/metadata-common": "4.0.0-dev", "@elyra/services": "4.0.0-dev", "@elyra/ui-components": "4.0.0-dev", - "@jupyterlab/application": "^4.0.6", - "@jupyterlab/apputils": "^4.1.6", - "@jupyterlab/builder": "^4.0.6", - "@jupyterlab/cells": "^4.0.6", - "@jupyterlab/codeeditor": "^4.0.6", + "@jupyterlab/application": "^4.2.5", + "@jupyterlab/apputils": "^4.2.5", + "@jupyterlab/builder": "^4.2.5", + "@jupyterlab/cells": "^4.2.5", + "@jupyterlab/codeeditor": "^4.2.5", + "@jupyterlab/codemirror": "^4.2.5", "@jupyterlab/coreutils": "^6.0.6", - "@jupyterlab/docmanager": "^4.0.6", - "@jupyterlab/docregistry": "^4.0.6", - "@jupyterlab/fileeditor": "^4.0.6", - "@jupyterlab/mainmenu": "^4.0.6", - "@jupyterlab/markdownviewer": "^4.0.6", - "@jupyterlab/notebook": "^4.0.6", - "@jupyterlab/ui-components": "^4.0.6", + "@jupyterlab/docmanager": "^4.2.5", + "@jupyterlab/docregistry": "^4.2.5", + "@jupyterlab/fileeditor": "^4.2.5", + "@jupyterlab/mainmenu": "^4.2.5", + "@jupyterlab/markdownviewer": "^4.2.5", + "@jupyterlab/notebook": "^4.2.5", + "@jupyterlab/ui-components": "^4.2.5", "@lumino/algorithm": "*", "@lumino/coreutils": "^2.1.2", "@lumino/messaging": "^2.0.1", @@ -78,6 +88,6 @@ }, "jupyterlab": { "extension": true, - "outputDir": "../../build/labextensions/@elyra/code-snippet-extension" + "outputDir": "../../labextensions/elyra_code_snippet_extension/labextension" } } diff --git a/packages/code-snippet/setup.py b/packages/code-snippet/setup.py new file mode 100644 index 000000000..aefdf20db --- /dev/null +++ b/packages/code-snippet/setup.py @@ -0,0 +1 @@ +__import__("setuptools").setup() diff --git a/packages/code-snippet/src/CodeSnippetService.ts b/packages/code-snippet/src/CodeSnippetService.ts index 3ff85c777..94693b46f 100644 --- a/packages/code-snippet/src/CodeSnippetService.ts +++ b/packages/code-snippet/src/CodeSnippetService.ts @@ -57,13 +57,13 @@ export class CodeSnippetService { static deleteCodeSnippet(codeSnippet: IMetadata): Promise { return showDialog({ title: `Delete snippet '${codeSnippet.display_name}'?`, - buttons: [Dialog.cancelButton(), Dialog.okButton()], + buttons: [Dialog.cancelButton(), Dialog.okButton()] }).then((result: any) => { // Do nothing if the cancel button is pressed if (result.button.accept) { return MetadataService.deleteMetadata( CODE_SNIPPET_SCHEMASPACE, - codeSnippet.name, + codeSnippet.name ).then(() => true); } else { return false; diff --git a/packages/code-snippet/src/CodeSnippetWidget.tsx b/packages/code-snippet/src/CodeSnippetWidget.tsx index f28f06f56..aa07478f9 100644 --- a/packages/code-snippet/src/CodeSnippetWidget.tsx +++ b/packages/code-snippet/src/CodeSnippetWidget.tsx @@ -25,13 +25,13 @@ import { MetadataCommonService, MetadataDisplay, MetadataWidget, - METADATA_ITEM, + METADATA_ITEM } from '@elyra/metadata-common'; import { ExpandableComponent, importIcon, RequestErrors, - trashIcon, + trashIcon } from '@elyra/ui-components'; import { JupyterFrontEnd } from '@jupyterlab/application'; @@ -40,22 +40,23 @@ import { CodeCell, MarkdownCell, ICodeCellModel, - IMarkdownCellModel /*RawCell*/, + IMarkdownCellModel /*RawCell*/ } from '@jupyterlab/cells'; import { CodeEditor, IEditorServices } from '@jupyterlab/codeeditor'; +import { EditorLanguageRegistry } from '@jupyterlab/codemirror'; import { PathExt } from '@jupyterlab/coreutils'; import { DocumentWidget } from '@jupyterlab/docregistry'; import { FileEditor } from '@jupyterlab/fileeditor'; import * as nbformat from '@jupyterlab/nbformat'; import { Notebook, - /*NotebookModel,*/ NotebookPanel /*,NotebookActions*/, + /*NotebookModel,*/ NotebookPanel /*,NotebookActions*/ } from '@jupyterlab/notebook'; import { copyIcon, editIcon, pasteIcon, - LabIcon, + LabIcon } from '@jupyterlab/ui-components'; import { find } from '@lumino/algorithm'; @@ -70,7 +71,7 @@ import React from 'react'; import { CodeSnippetService, CODE_SNIPPET_SCHEMASPACE, - CODE_SNIPPET_SCHEMA, + CODE_SNIPPET_SCHEMA } from './CodeSnippetService'; const METADATA_EDITOR_ID = 'elyra-metadata-editor'; @@ -138,7 +139,7 @@ class CodeSnippetDisplay extends MetadataDisplay { snippetLanguage.toLowerCase() !== 'markdown' ) { fileEditor.replaceSelection?.( - this.addMarkdownCodeBlock(snippetLanguage, codeSnippet), + this.addMarkdownCodeBlock(snippetLanguage, codeSnippet) ); } else if (editorLanguage) { this.verifyLanguageAndInsert(snippet, editorLanguage, fileEditor); @@ -165,14 +166,14 @@ class CodeSnippetDisplay extends MetadataDisplay { this.verifyLanguageAndInsert( snippet, kernelLanguage, - notebookCellEditor, + notebookCellEditor ); } else if ( notebookCell instanceof MarkdownCell && snippetLanguage.toLowerCase() !== 'markdown' ) { notebookCellEditor.replaceSelection?.( - this.addMarkdownCodeBlock(snippetLanguage, codeSnippet), + this.addMarkdownCodeBlock(snippetLanguage, codeSnippet) ); } else { notebookCellEditor.replaceSelection?.(codeSnippet); @@ -183,14 +184,14 @@ class CodeSnippetDisplay extends MetadataDisplay { const contentFactory = new NotebookPanel.ContentFactory({ editorFactory: - this.props.editorServices.factoryService.newInlineEditor, + this.props.editorServices.factoryService.newInlineEditor }); /* interface CodeCellCreatorOption { - model: ICodeCellModel | undefined; - rendermime: RenderMimeRegistry; - contentFactory: any; + model: ICodeCellModel | undefined; + rendermime: RenderMimeRegistry; + contentFactory: any; cell_type: string; } */ @@ -198,7 +199,7 @@ class CodeSnippetDisplay extends MetadataDisplay { const options: CodeCell.IOptions = { model: notebookContent.activeCell?.model as ICodeCellModel, rendermime: notebookContent.rendermime, - contentFactory: contentFactory, + contentFactory: contentFactory }; const codeCell: any = contentFactory.createCodeCell(options); @@ -209,7 +210,7 @@ class CodeSnippetDisplay extends MetadataDisplay { // codeCell: SharedCell.Cell widget.content.model?.sharedModel.insertCell( activeCellIndex, - codeCell as Partial & { cell_type: string }, + codeCell as Partial & { cell_type: string } ); //update the active cell index to the newly inserted cell @@ -224,18 +225,19 @@ class CodeSnippetDisplay extends MetadataDisplay { // Verify if a given widget is a FileEditor private isFileEditor = ( - widget: Widget, + widget: Widget ): widget is DocumentWidget => { return (widget as DocumentWidget).content instanceof FileEditor; }; // Return the language of the editor or empty string private getEditorLanguage = ( - widget: DocumentWidget, + widget: DocumentWidget ): string | undefined => { - const editorLanguage = - widget.context.sessionContext.kernelPreference.language; - return editorLanguage === 'null' ? '' : editorLanguage; + const editorLanguage = EditorLanguageRegistry.getDefaultLanguages().find( + (language) => language.mime.includes(widget.content.model.mimeType) + ); + return editorLanguage?.displayName ?? ''; }; // Return the given code wrapped in a markdown code block @@ -247,7 +249,7 @@ class CodeSnippetDisplay extends MetadataDisplay { private verifyLanguageAndInsert = async ( snippet: IMetadata, editorLanguage: string, - editor: CodeEditor.IEditor, + editor: CodeEditor.IEditor ): Promise => { const codeSnippet: string = snippet.metadata.code.join('\n'); const snippetLanguage = snippet.metadata.language; @@ -257,7 +259,7 @@ class CodeSnippetDisplay extends MetadataDisplay { ) { const result = await this.showWarnDialog( editorLanguage, - snippet.display_name, + snippet.display_name ); if (result.button.accept) { editor.replaceSelection?.(codeSnippet); @@ -271,12 +273,12 @@ class CodeSnippetDisplay extends MetadataDisplay { // Display warning dialog when inserting a code snippet incompatible with editor's language private showWarnDialog = async ( editorLanguage: string, - snippetName: string, + snippetName: string ): Promise> => { return showDialog({ title: 'Warning', body: `Code snippet "${snippetName}" is incompatible with ${editorLanguage}. Continue?`, - buttons: [Dialog.cancelButton(), Dialog.okButton()], + buttons: [Dialog.cancelButton(), Dialog.okButton()] }); }; @@ -285,14 +287,14 @@ class CodeSnippetDisplay extends MetadataDisplay { return showDialog({ title: 'Error', body: errMsg, - buttons: [Dialog.okButton()], + buttons: [Dialog.okButton()] }); }; // Initial setup to handle dragging a code snippet private handleDragSnippet( event: React.MouseEvent, - metadata: IMetadata, + metadata: IMetadata ): void { const { button } = event; @@ -304,7 +306,7 @@ class CodeSnippetDisplay extends MetadataDisplay { this._dragData = { pressX: event.clientX, pressY: event.clientY, - dragImage: null, + dragImage: null }; const mouseUpListener = (event: MouseEvent): void => { @@ -317,7 +319,7 @@ class CodeSnippetDisplay extends MetadataDisplay { const target = event.target as HTMLElement; target.addEventListener('mouseup', mouseUpListener, { once: true, - capture: true, + capture: true }); target.addEventListener('mousemove', mouseMoveListener, true); @@ -328,7 +330,7 @@ class CodeSnippetDisplay extends MetadataDisplay { private _evtMouseUp( event: MouseEvent, metadata: IMetadata, - mouseMoveListener: (event: MouseEvent) => void, + mouseMoveListener: (event: MouseEvent) => void ): void { event.preventDefault(); event.stopPropagation(); @@ -341,7 +343,7 @@ class CodeSnippetDisplay extends MetadataDisplay { event: MouseEvent, metadata: IMetadata, mouseMoveListener: (event: MouseEvent) => void, - mouseUpListener: (event: MouseEvent) => void, + mouseUpListener: (event: MouseEvent) => void ): void { event.preventDefault(); event.stopPropagation(); @@ -354,7 +356,7 @@ class CodeSnippetDisplay extends MetadataDisplay { data.pressX, data.pressY, event.clientX, - event.clientY, + event.clientY ) ) { // Create drag image @@ -372,7 +374,7 @@ class CodeSnippetDisplay extends MetadataDisplay { data.dragImage, metadata, event.clientX, - event.clientY, + event.clientY ); } } @@ -390,7 +392,7 @@ class CodeSnippetDisplay extends MetadataDisplay { prevX: number, prevY: number, nextX: number, - nextY: number, + nextY: number ): boolean { const dx = Math.abs(nextX - prevX); const dy = Math.abs(nextY - prevY); @@ -401,7 +403,7 @@ class CodeSnippetDisplay extends MetadataDisplay { dragImage: HTMLElement, metadata: IMetadata, clientX: number, - clientY: number, + clientY: number ): Promise { const widget: NotebookPanel = this.props.getCurrentWidget() as NotebookPanel; @@ -410,19 +412,19 @@ class CodeSnippetDisplay extends MetadataDisplay { //const activeCellIndex = notebookContent.activeCellIndex ?? -1; const contentFactory = new NotebookPanel.ContentFactory({ - editorFactory: this.props.editorServices.factoryService.newInlineEditor, + editorFactory: this.props.editorServices.factoryService.newInlineEditor }); const options: CodeCell.IOptions = { model: notebookContent.activeCell?.model as ICodeCellModel, rendermime: notebookContent.rendermime, - contentFactory: contentFactory, + contentFactory: contentFactory }; const options2: MarkdownCell.IOptions = { model: notebookContent.activeCell?.model as IMarkdownCellModel, rendermime: notebookContent.rendermime, - contentFactory: contentFactory, + contentFactory: contentFactory }; const codeCell = contentFactory.createCodeCell(options); @@ -455,7 +457,7 @@ class CodeSnippetDisplay extends MetadataDisplay { dragImage: dragImage, supportedActions: 'copy-move', proposedAction: 'copy', - source: this, + source: this }); const selected: nbformat.ICell[] = [model.model.toJSON()]; @@ -476,14 +478,14 @@ class CodeSnippetDisplay extends MetadataDisplay { feedback: 'Copied!', onClick: (): void => { Clipboard.copyToSystem(metadata.metadata.code.join('\n')); - }, + } }, { title: 'Insert', icon: importIcon, onClick: (): void => { this.insertCodeSnippet(metadata); - }, + } }, { title: 'Edit', @@ -493,9 +495,9 @@ class CodeSnippetDisplay extends MetadataDisplay { onSave: this.props.updateMetadata, schemaspace: CODE_SNIPPET_SCHEMASPACE, schema: CODE_SNIPPET_SCHEMA, - name: metadata.name, + name: metadata.name }); - }, + } }, { title: 'Duplicate', @@ -504,13 +506,13 @@ class CodeSnippetDisplay extends MetadataDisplay { MetadataCommonService.duplicateMetadataInstance( CODE_SNIPPET_SCHEMASPACE, metadata, - this.props.metadata, + this.props.metadata ) .then((response: any): void => { this.props.updateMetadata(); }) .catch((error) => RequestErrors.serverError(error)); - }, + } }, { title: 'Delete', @@ -528,7 +530,7 @@ class CodeSnippetDisplay extends MetadataDisplay { value.id === `${METADATA_EDITOR_ID}:${CODE_SNIPPET_SCHEMASPACE}:${CODE_SNIPPET_SCHEMA}:${metadata.name}` ); - }, + } ); if (editorWidget) { editorWidget.dispose(); @@ -536,8 +538,8 @@ class CodeSnippetDisplay extends MetadataDisplay { } }) .catch((error) => RequestErrors.serverError(error)); - }, - }, + } + } ]; }; @@ -547,7 +549,7 @@ class CodeSnippetDisplay extends MetadataDisplay { sortMetadata(): void { this.props.metadata.sort((a, b) => - this.getDisplayName(a).localeCompare(this.getDisplayName(b)), + this.getDisplayName(a).localeCompare(this.getDisplayName(b)) ); } @@ -593,14 +595,12 @@ class CodeSnippetDisplay extends MetadataDisplay { createPreviewEditors = (): void => { const editorFactory = this.props.editorServices.factoryService.newInlineEditor; - const getMimeTypeByLanguage = - this.props.editorServices.mimeTypeService.getMimeTypeByLanguage; this.props.metadata.map((codeSnippet: IMetadata) => { + const content = codeSnippet.metadata.code.join('\n'); + if (codeSnippet.name in this.editors) { // Make sure code is up to date - this.editors[codeSnippet.name].model.selections.has( - codeSnippet.metadata.code.join('\n'), - ); + this.editors[codeSnippet.name].model.sharedModel.setSource(content); } else { // Add new snippets const snippetElement = document.getElementById(codeSnippet.name); @@ -608,17 +608,20 @@ class CodeSnippetDisplay extends MetadataDisplay { return; } - this.editors[codeSnippet.name] = editorFactory({ + const mimeType = + this.props.editorServices.mimeTypeService.getMimeTypeByLanguage({ + value: codeSnippet.metadata.code.join('\n'), + name: codeSnippet.metadata.language, + codemirror_mode: codeSnippet.metadata.language + }); + + const newEditor = editorFactory({ config: { readOnly: true }, host: snippetElement, - model: new CodeEditor.Model({ - mimeType: getMimeTypeByLanguage({ - value: codeSnippet.metadata.code.join('\n'), - name: codeSnippet.metadata.language, - codemirror_mode: codeSnippet.metadata.language, - }), - }), + model: new CodeEditor.Model({ mimeType }) }); + newEditor.model.sharedModel.setSource(content); + this.editors[codeSnippet.name] = newEditor; } }); }; @@ -663,7 +666,7 @@ export class CodeSnippetWidget extends MetadataWidget { // Request code snippets from server async fetchMetadata(): Promise { return CodeSnippetService.findAll().catch((error) => - RequestErrors.serverError(error), + RequestErrors.serverError(error) ); } diff --git a/packages/code-snippet/src/index.ts b/packages/code-snippet/src/index.ts index 70abdfef3..41c969bf9 100644 --- a/packages/code-snippet/src/index.ts +++ b/packages/code-snippet/src/index.ts @@ -21,11 +21,11 @@ import { codeSnippetIcon } from '@elyra/ui-components'; import { JupyterFrontEnd, JupyterFrontEndPlugin, - ILayoutRestorer, + ILayoutRestorer } from '@jupyterlab/application'; import { ICommandPalette } from '@jupyterlab/apputils'; import { Cell } from '@jupyterlab/cells'; -import { IEditorServices } from '@jupyterlab/codeeditor'; +import { CodeEditor, IEditorServices } from '@jupyterlab/codeeditor'; import { DocumentWidget } from '@jupyterlab/docregistry'; import { FileEditor } from '@jupyterlab/fileeditor'; import { MarkdownDocument } from '@jupyterlab/markdownviewer'; @@ -34,14 +34,14 @@ import { Widget } from '@lumino/widgets'; import { CODE_SNIPPET_SCHEMASPACE, - CODE_SNIPPET_SCHEMA, + CODE_SNIPPET_SCHEMA } from './CodeSnippetService'; import { CodeSnippetWidget } from './CodeSnippetWidget'; const CODE_SNIPPET_EXTENSION_ID = 'elyra-code-snippet-extension'; const commandIDs = { - saveAsSnippet: 'codesnippet:save-as-snippet', + saveAsSnippet: 'codesnippet:save-as-snippet' }; /** @@ -55,7 +55,7 @@ export const code_snippet_extension: JupyterFrontEndPlugin = { app: JupyterFrontEnd, palette: ICommandPalette, restorer: ILayoutRestorer, - editorServices: IEditorServices, + editorServices: IEditorServices ) => { console.log('Elyra - code-snippet extension is activated!'); @@ -71,7 +71,8 @@ export const code_snippet_extension: JupyterFrontEndPlugin = { icon: codeSnippetIcon, getCurrentWidget, editorServices, - titleContext: 'code snippet', + titleContext: '', + addLabel: 'code snippet' }); const codeSnippetWidgetId = `elyra-metadata:${CODE_SNIPPET_SCHEMASPACE}`; codeSnippetWidget.id = codeSnippetWidgetId; @@ -126,7 +127,7 @@ export const code_snippet_extension: JupyterFrontEndPlugin = { schemaspace: CODE_SNIPPET_SCHEMASPACE, schema: CODE_SNIPPET_SCHEMA, code: selection.split('\n'), - onSave: codeSnippetWidget.updateMetadata, + onSave: codeSnippetWidget.updateMetadata }); } else { const selectedCells = getSelectedCellContents(); @@ -136,37 +137,38 @@ export const code_snippet_extension: JupyterFrontEndPlugin = { schemaspace: CODE_SNIPPET_SCHEMASPACE, schema: CODE_SNIPPET_SCHEMA, code: code, - onSave: codeSnippetWidget.updateMetadata, + onSave: codeSnippetWidget.updateMetadata }); } - }, + } }); app.contextMenu.addItem({ command: commandIDs.saveAsSnippet, - selector: '.jp-Cell', + selector: '.jp-Cell' }); app.contextMenu.addItem({ command: commandIDs.saveAsSnippet, - selector: '.jp-FileEditor', + selector: '.jp-FileEditor' }); app.contextMenu.addItem({ command: commandIDs.saveAsSnippet, - selector: '.jp-MarkdownViewer', + selector: '.jp-MarkdownViewer' }); const getTextSelection = ( - editor: any, - markdownPreview?: boolean, + editor: CodeEditor.IEditor, + markdownPreview?: boolean ): string => { const selectionObj = editor.getSelection(); const start = editor.getOffsetAt(selectionObj.start); const end = editor.getOffsetAt(selectionObj.end); - const selection = editor.model.value.text.substring(start, end); + const source = editor.model.sharedModel.getSource(); + const selection = source.substring(start, end); - if (!selection && editor.model.value.text) { + if (!selection && source) { // Allow selections from a rendered notebook cell return document.getSelection()?.toString() ?? ''; } @@ -196,22 +198,24 @@ export const code_snippet_extension: JupyterFrontEndPlugin = { return selectedCells; }; - const isFileEditor = (currentWidget: any): boolean => { + const isFileEditor = (currentWidget: Widget | null): boolean => { return ( currentWidget instanceof DocumentWidget && (currentWidget as DocumentWidget).content instanceof FileEditor ); }; - const isNotebookEditor = (currentWidget: any): boolean => { + const isNotebookEditor = (currentWidget: Widget | null): boolean => { return currentWidget instanceof NotebookPanel; }; - const isMarkdownDocument = (currentWidget: any): boolean => { + const isMarkdownDocument = (currentWidget: Widget | null): boolean => { return currentWidget instanceof MarkdownDocument; }; - const getEditor = (currentWidget: any): any => { + const getEditor = ( + currentWidget: Widget | null + ): CodeEditor.IEditor | null | undefined => { if (isFileEditor(currentWidget)) { const documentWidget = currentWidget as DocumentWidget; return (documentWidget.content as FileEditor).editor; @@ -220,8 +224,9 @@ export const code_snippet_extension: JupyterFrontEndPlugin = { const notebookCell = (notebookWidget.content as Notebook).activeCell; return notebookCell?.editor; } + return undefined; }; - }, + } }; export default code_snippet_extension; diff --git a/packages/code-viewer/package.json b/packages/code-viewer/package.json index 805d1a008..a0572fc76 100644 --- a/packages/code-viewer/package.json +++ b/packages/code-viewer/package.json @@ -25,7 +25,7 @@ "style/**/*.{css,eot,gif,html,jpg,json,png,svg,woff2,ttf}" ], "scripts": { - "build": "jlpm run build:lib && jlpm run build:labextension:dev", + "build": "echo Skip the build until the extension is migrated to JL4", "build:prod": "jlpm run build:lib && jlpm run build:labextension", "build:lib": "tsc", "build:labextension": "jupyter labextension build .", diff --git a/packages/code-viewer/src/CodeViewerWidget.ts b/packages/code-viewer/src/CodeViewerWidget.ts index 99ccb66dc..2bba4dc59 100644 --- a/packages/code-viewer/src/CodeViewerWidget.ts +++ b/packages/code-viewer/src/CodeViewerWidget.ts @@ -27,7 +27,7 @@ export class CodeViewerWidget extends Widget { const editorWidget = new CodeEditorWrapper({ factory: options.factory, - model: options.model, + model: options.model }); this.editor = editorWidget.editor; this.editor.setOption('readOnly', true); @@ -37,7 +37,7 @@ export class CodeViewerWidget extends Widget { } static getCodeViewer( - options: CodeViewerWidget.INoModelOptions, + options: CodeViewerWidget.INoModelOptions ): CodeViewerWidget { const model = new CodeEditor.Model({ mimeType: options.mimeType }); model.sharedModel.source = options.content; diff --git a/packages/code-viewer/src/index.ts b/packages/code-viewer/src/index.ts index 549eee55e..c2259b215 100644 --- a/packages/code-viewer/src/index.ts +++ b/packages/code-viewer/src/index.ts @@ -17,7 +17,7 @@ import { ILayoutRestorer, JupyterFrontEnd, - JupyterFrontEndPlugin, + JupyterFrontEndPlugin } from '@jupyterlab/application'; import { MainAreaWidget, WidgetTracker } from '@jupyterlab/apputils'; import { CodeEditor, IEditorServices } from '@jupyterlab/codeeditor'; @@ -32,7 +32,7 @@ const ELYRA_CODE_VIEWER_NAMESPACE = 'elyra-code-viewer-extension'; * The command IDs used by the code-viewer plugin. */ const CommandIDs = { - openViewer: 'elyra-code-viewer:open', + openViewer: 'elyra-code-viewer:open' }; /** @@ -46,12 +46,12 @@ const extension: JupyterFrontEndPlugin = { activate: ( app: JupyterFrontEnd, editorServices: IEditorServices, - restorer: ILayoutRestorer, + restorer: ILayoutRestorer ) => { console.log('Elyra - code-viewer extension is activated!'); const tracker = new WidgetTracker>({ - namespace: ELYRA_CODE_VIEWER_NAMESPACE, + namespace: ELYRA_CODE_VIEWER_NAMESPACE }); // Handle state restoration @@ -62,9 +62,9 @@ const extension: JupyterFrontEndPlugin = { content: widget.content.getContent(), label: widget.content.title.label, mimeType: widget.content.getMimeType(), - widgetId: widget.content.id, + widgetId: widget.content.id }), - name: (widget) => widget.content.id, + name: (widget) => widget.content.id }); } @@ -84,14 +84,14 @@ const extension: JupyterFrontEndPlugin = { let mimetype = args.mimeType; if (!mimetype && args.extension) { mimetype = editorServices.mimeTypeService.getMimeTypeByFilePath( - `temp.${args.extension.replace(/\\.$/, '')}`, + `temp.${args.extension.replace(/\\.$/, '')}` ); } const widget = CodeViewerWidget.getCodeViewer({ factory, content: args.content, - mimeType: mimetype, + mimeType: mimetype }); widget.title.label = args.label || 'Code Viewer'; widget.title.caption = widget.title.label; @@ -114,9 +114,9 @@ const extension: JupyterFrontEndPlugin = { app.commands.addCommand(CommandIDs.openViewer, { execute: (args: any) => { return openCodeViewer(args); - }, + } }); - }, + } }; export default extension; diff --git a/packages/metadata-common/install.json b/packages/metadata-common/install.json new file mode 100644 index 000000000..87e6c4e75 --- /dev/null +++ b/packages/metadata-common/install.json @@ -0,0 +1,5 @@ +{ + "packageManager": "python", + "packageName": "elyra_metadata_common", + "uninstallInstructions": "Use your Python package manager (pip, conda, etc.) to uninstall the package elyra_metadata_common" +} diff --git a/packages/metadata-common/package.json b/packages/metadata-common/package.json index 738d2c150..04148d435 100644 --- a/packages/metadata-common/package.json +++ b/packages/metadata-common/package.json @@ -24,21 +24,39 @@ "style/**/*.{css,eot,gif,html,jpg,json,png,svg,woff2,ttf}" ], "scripts": { - "build": "tsc", - "clean": "rimraf lib", - "dist": "npm pack .", - "prepare": "npm run build", - "watch": "tsc -w", - "lab:install": "jupyter labextension link --no-build", - "lab:uninstall": "jupyter labextension unlink --no-build" + "build": "jlpm build:lib && jlpm build:labextension:dev", + "build:prod": "jlpm clean && jlpm build:lib:prod && jlpm build:labextension", + "build:labextension": "jupyter labextension build .", + "build:labextension:dev": "jupyter labextension build --development True .", + "build:lib": "tsc --sourceMap", + "build:lib:prod": "tsc", + "clean": "jlpm clean:lib", + "clean:lib": "rimraf lib tsconfig.tsbuildinfo", + "clean:lintcache": "rimraf .eslintcache .stylelintcache", + "clean:labextension": "rimraf ../../../../labextensions/elyra_metadata_common/labextension ../../../../labextensions/elyra_metadata_common/_version.py", + "clean:all": "jlpm clean:lib && jlpm clean:labextension && jlpm clean:lintcache", + "eslint": "jlpm eslint:check --fix", + "eslint:check": "eslint . --cache --ext .ts,.tsx", + "install:extension": "jlpm build", + "lint": "jlpm stylelint && jlpm prettier && jlpm eslint", + "lint:check": "jlpm stylelint:check && jlpm prettier:check && jlpm eslint:check", + "prettier": "jlpm prettier:base --write --list-different", + "prettier:base": "prettier \"**/*{.ts,.tsx,.js,.jsx,.css,.json,.md}\"", + "prettier:check": "jlpm prettier:base --check", + "stylelint": "jlpm stylelint:check --fix", + "stylelint:check": "stylelint --cache \"style/**/*.css\"", + "test": "jest --coverage --passWithNoTests", + "watch": "run-p watch:src watch:labextension", + "watch:src": "tsc -w --sourceMap", + "watch:labextension": "jupyter labextension watch ." }, "dependencies": { "@elyra/services": "4.0.0-dev", "@elyra/ui-components": "4.0.0-dev", - "@jupyterlab/application": "^4.0.6", - "@jupyterlab/apputils": "^4.1.6", - "@jupyterlab/codeeditor": "^4.0.6", - "@jupyterlab/ui-components": "^4.0.6", + "@jupyterlab/application": "^4.2.5", + "@jupyterlab/apputils": "^4.2.5", + "@jupyterlab/codeeditor": "^4.2.5", + "@jupyterlab/ui-components": "^4.2.5", "@lumino/algorithm": "*", "@lumino/messaging": "^2.0.1", "@lumino/widgets": "^2.3.1", @@ -46,6 +64,7 @@ "react-dom": "^18.0.9" }, "devDependencies": { + "@jupyterlab/builder": "^4.2.5", "@types/react": "^18.0.26", "@types/react-dom": "^18.0.9", "rimraf": "~5.0.5", @@ -56,6 +75,6 @@ }, "jupyterlab": { "extension": false, - "outputDir": "../../build/labextensions/@elyra/metadata-common" + "outputDir": "../../labextensions/elyra_metadata_common/labextension" } } diff --git a/packages/metadata-common/setup.py b/packages/metadata-common/setup.py new file mode 100644 index 000000000..aefdf20db --- /dev/null +++ b/packages/metadata-common/setup.py @@ -0,0 +1 @@ +__import__("setuptools").setup() diff --git a/packages/metadata-common/src/FilterTools.tsx b/packages/metadata-common/src/FilterTools.tsx index 65e95425c..8deee092a 100644 --- a/packages/metadata-common/src/FilterTools.tsx +++ b/packages/metadata-common/src/FilterTools.tsx @@ -61,7 +61,7 @@ export class FilterTools extends React.Component< componentDidMount(): void { this.setState({ selectedTags: [], - searchValue: '', + searchValue: '' }); } @@ -70,14 +70,14 @@ export class FilterTools extends React.Component< this.setState((state) => ({ selectedTags: state.selectedTags .filter((tag) => this.props.tags.includes(tag)) - .sort(), + .sort() })); } } createFilterBox(): void { const filterOption = document.querySelector( - `#${this.props.schemaspace} .${FILTER_OPTION}`, + `#${this.props.schemaspace} .${FILTER_OPTION}` ); filterOption?.classList.toggle('idle'); @@ -109,7 +109,7 @@ export class FilterTools extends React.Component< renderAppliedTag(tag: string, index: string): JSX.Element { return ( diff --git a/packages/metadata/install.json b/packages/metadata/install.json new file mode 100644 index 000000000..4aa136ba4 --- /dev/null +++ b/packages/metadata/install.json @@ -0,0 +1,5 @@ +{ + "packageManager": "python", + "packageName": "elyra_metadata_extension", + "uninstallInstructions": "Use your Python package manager (pip, conda, etc.) to uninstall the package elyra_metadata_extension" +} diff --git a/packages/metadata/package.json b/packages/metadata/package.json index b9fb0e89c..5ad318f51 100644 --- a/packages/metadata/package.json +++ b/packages/metadata/package.json @@ -25,31 +25,40 @@ "style/**/*.{css,eot,gif,html,jpg,json,png,svg,woff2,ttf}" ], "scripts": { - "build": "jlpm run build:lib && jlpm run build:labextension:dev", - "build:prod": "jlpm run build:lib && jlpm run build:labextension", - "build:lib": "tsc", + "build": "jlpm build:lib && jlpm build:labextension:dev", + "build:prod": "jlpm clean && jlpm build:lib:prod && jlpm build:labextension", "build:labextension": "jupyter labextension build .", "build:labextension:dev": "jupyter labextension build --development True .", - "clean": "rimraf lib tsconfig.tsbuildinfo ../../build/labextensions/@elyra/metadata-extension", - "lab:dev": "jupyter labextension develop --overwrite ../../build/labextensions/@elyra/metadata-extension", - "dist": "npm pack .", - "prepare": "npm run build", + "build:lib": "tsc --sourceMap", + "build:lib:prod": "tsc", + "clean": "jlpm clean:lib", + "clean:lib": "rimraf lib tsconfig.tsbuildinfo", + "clean:lintcache": "rimraf .eslintcache .stylelintcache", + "clean:labextension": "rimraf ../../../../labextensions/elyra_metadata_extension/labextension ../../../../labextensions/elyra_metadata_extension/_version.py", + "clean:all": "jlpm clean:lib && jlpm clean:labextension && jlpm clean:lintcache", + "eslint": "jlpm eslint:check --fix", + "eslint:check": "eslint . --cache --ext .ts,.tsx", + "install:extension": "jlpm build", + "lint": "jlpm stylelint && jlpm prettier && jlpm eslint", + "lint:check": "jlpm stylelint:check && jlpm prettier:check && jlpm eslint:check", + "prettier": "jlpm prettier:base --write --list-different", + "prettier:base": "prettier \"**/*{.ts,.tsx,.js,.jsx,.css,.json,.md}\"", + "prettier:check": "jlpm prettier:base --check", + "stylelint": "jlpm stylelint:check --fix", + "stylelint:check": "stylelint --cache \"style/**/*.css\"", + "test": "jest --coverage --passWithNoTests", "watch": "run-p watch:src watch:labextension", - "watch:src": "tsc -w", - "watch:labextension": "jupyter labextension watch .", - "lab:install": "jupyter labextension install --no-build", - "lab:uninstall": "jupyter labextension uninstall --no-build", - "link:dev": "yarn link @jupyterlab/builder", - "unlink:dev": "yarn unlink @jupyterlab/builder" + "watch:src": "tsc -w --sourceMap", + "watch:labextension": "jupyter labextension watch ." }, "dependencies": { "@elyra/metadata-common": "4.0.0-dev", "@elyra/services": "4.0.0-dev", - "@jupyterlab/application": "^4.0.6", - "@jupyterlab/apputils": "^4.1.6", - "@jupyterlab/builder": "^4.0.6", - "@jupyterlab/codeeditor": "^4.0.6", - "@jupyterlab/ui-components": "^4.0.6", + "@jupyterlab/application": "^4.2.5", + "@jupyterlab/apputils": "^4.2.5", + "@jupyterlab/builder": "^4.2.5", + "@jupyterlab/codeeditor": "^4.2.5", + "@jupyterlab/ui-components": "^4.2.5", "@lumino/algorithm": "*", "@lumino/widgets": "^2.3.1" }, @@ -62,6 +71,6 @@ }, "jupyterlab": { "extension": true, - "outputDir": "../../build/labextensions/@elyra/metadata-extension" + "outputDir": "../../labextensions/elyra_metadata_extension/labextension" } } diff --git a/packages/metadata/setup.py b/packages/metadata/setup.py new file mode 100644 index 000000000..aefdf20db --- /dev/null +++ b/packages/metadata/setup.py @@ -0,0 +1 @@ +__import__("setuptools").setup() diff --git a/packages/metadata/src/index.ts b/packages/metadata/src/index.ts index dba5d1576..e733dc950 100644 --- a/packages/metadata/src/index.ts +++ b/packages/metadata/src/index.ts @@ -18,16 +18,16 @@ import { MetadataWidget, MetadataEditorWidget } from '@elyra/metadata-common'; import { MetadataService } from '@elyra/services'; import { - DropDown, RequestErrors, + PasswordField, CodeBlock, TagsField, - PasswordField, + DropDown } from '@elyra/ui-components'; import { JupyterFrontEnd, JupyterFrontEndPlugin, - ILabStatus, + ILabStatus } from '@jupyterlab/application'; import { ICommandPalette, MainAreaWidget } from '@jupyterlab/apputils'; import { IEditorServices } from '@jupyterlab/codeeditor'; @@ -35,8 +35,7 @@ import { ITranslator } from '@jupyterlab/translation'; import { textEditorIcon, LabIcon, - IFormRendererRegistry, - IFormRenderer, + IFormRendererRegistry } from '@jupyterlab/ui-components'; import { find } from '@lumino/algorithm'; @@ -44,10 +43,11 @@ import { Widget } from '@lumino/widgets'; const METADATA_EDITOR_ID = 'elyra-metadata-editor'; const METADATA_WIDGET_ID = 'elyra-metadata'; +const METADATA_EXTENSION_ID = '@elyra/metadata-extension'; const commandIDs = { openMetadata: 'elyra-metadata:open', - closeTabCommand: 'elyra-metadata:close', + closeTabCommand: 'elyra-metadata:close' }; /** @@ -61,7 +61,7 @@ const extension: JupyterFrontEndPlugin = { IEditorServices, ILabStatus, IFormRendererRegistry, - ITranslator, + ITranslator ], activate: async ( app: JupyterFrontEnd, @@ -69,26 +69,30 @@ const extension: JupyterFrontEndPlugin = { editorServices: IEditorServices, status: ILabStatus, componentRegistry: IFormRendererRegistry, - translator: ITranslator, + translator: ITranslator ) => { console.log('Elyra - metadata extension is activated!'); - componentRegistry.addRenderer( - 'code', - CodeBlock as unknown as IFormRenderer, - ); - componentRegistry.addRenderer( - 'tags', - TagsField as unknown as IFormRenderer, - ); - componentRegistry.addRenderer( - 'dropdown', - DropDown as unknown as IFormRenderer, - ); - componentRegistry.addRenderer( - 'password', - PasswordField as unknown as IFormRenderer, - ); + componentRegistry.addRenderer(`${METADATA_EXTENSION_ID}:plugin.code`, { + fieldRenderer: (props) => { + return CodeBlock(props); + } + }); + componentRegistry.addRenderer(`${METADATA_EXTENSION_ID}:plugin.tags`, { + fieldRenderer: (props) => { + return TagsField(props); + } + }); + componentRegistry.addRenderer(`${METADATA_EXTENSION_ID}:plugin.dropdown`, { + fieldRenderer: (props) => { + return DropDown(props); + } + }); + componentRegistry.addRenderer(`${METADATA_EXTENSION_ID}:plugin.password`, { + fieldRenderer: (props) => { + return PasswordField(props); + } + }); const openMetadataEditor = (args: { schema: string; @@ -110,7 +114,7 @@ const extension: JupyterFrontEndPlugin = { app.shell.widgets('main'), (widget: Widget, index: number) => { return widget.id === widgetId; - }, + } ); if (openWidget) { app.shell.activateById(widgetId); @@ -123,7 +127,7 @@ const extension: JupyterFrontEndPlugin = { editorServices, status, translator: translator.load('jupyterlab'), - componentRegistry, + componentRegistry }); const main = new MainAreaWidget({ content: metadataEditorWidget }); main.title.label = widgetLabel; @@ -142,13 +146,14 @@ const extension: JupyterFrontEndPlugin = { }, execute: (args: any) => { openMetadataEditor(args); - }, + } }); const openMetadataWidget = (args: { display_name: string; schemaspace: string; icon: string; + addLabel?: string; }): void => { const labIcon = LabIcon.resolve({ icon: args.icon }); const widgetId = `${METADATA_WIDGET_ID}:${args.schemaspace}`; @@ -157,6 +162,7 @@ const extension: JupyterFrontEndPlugin = { display_name: args.display_name, schemaspace: args.schemaspace, icon: labIcon, + addLabel: args.addLabel }); metadataWidget.id = widgetId; metadataWidget.title.icon = labIcon; @@ -178,7 +184,7 @@ const extension: JupyterFrontEndPlugin = { // Rank has been chosen somewhat arbitrarily to give priority // to the running sessions widget in the sidebar. openMetadataWidget(args); - }, + } }); // Add command to close metadata tab @@ -187,7 +193,7 @@ const extension: JupyterFrontEndPlugin = { label: 'Close Tab', execute: (args) => { const contextNode: HTMLElement | undefined = app.contextMenuHitTest( - (node) => !!node.dataset.id, + (node) => !!node.dataset.id ); if (contextNode) { const id = contextNode.dataset['id']!; @@ -195,18 +201,18 @@ const extension: JupyterFrontEndPlugin = { app.shell.widgets('left'), (widget: Widget, index: number) => { return widget.id === id; - }, + } ); if (widget) { widget.dispose(); } } - }, + } }); app.contextMenu.addItem({ selector: '[data-id^="elyra-metadata:"]:not([data-id$="code-snippets"]):not([data-id$="runtimes"])', - command: closeTabCommand, + command: closeTabCommand }); try { @@ -228,15 +234,15 @@ const extension: JupyterFrontEndPlugin = { label: `Manage ${title}`, display_name: schema.uihints.title, schemaspace: schema.schemaspace, - icon: icon, + icon: icon }, - category: 'Elyra', + category: 'Elyra' }); } } catch (error) { RequestErrors.serverError(error); } - }, + } }; export default extension; diff --git a/packages/pipeline-editor/install.json b/packages/pipeline-editor/install.json new file mode 100644 index 000000000..7a6828db9 --- /dev/null +++ b/packages/pipeline-editor/install.json @@ -0,0 +1,5 @@ +{ + "packageManager": "python", + "packageName": "elyra_pipeline_editor_extension", + "uninstallInstructions": "Use your Python package manager (pip, conda, etc.) to uninstall the package elyra_pipeline_editor_extension" +} diff --git a/packages/pipeline-editor/package.json b/packages/pipeline-editor/package.json index 772eb8b4a..36242ab67 100644 --- a/packages/pipeline-editor/package.json +++ b/packages/pipeline-editor/package.json @@ -22,27 +22,35 @@ "files": [ "lib/**/*.{d.ts,eot,gif,html,jpg,js,js.map,json,png,svg,woff2,ttf}", "src/**/*.{ts,tsx}", - "style/**/*.{css,eot,gif,html,jpg,json,png,svg,woff2,ttf}" + "style/**/*.{css,eot,gif,html,jpg,json,png,svg,woff2,ttf}", + "schema/*.json" ], "scripts": { - "build:test": "tsc --build tsconfig.test.json", - "build": "jlpm run build:lib && jlpm run build:labextension:dev", - "build:prod": "jlpm run build:lib && jlpm run build:labextension", - "build:lib": "tsc", + "build": "jlpm build:lib && jlpm build:labextension:dev", + "build:prod": "jlpm clean && jlpm build:lib:prod && jlpm build:labextension", "build:labextension": "jupyter labextension build .", "build:labextension:dev": "jupyter labextension build --development True .", - "clean": "rimraf lib tsconfig.tsbuildinfo ../../build/labextensions/@elyra/metadata-extension", - "lab:dev": "jupyter labextension develop --overwrite ../../build/labextensions/@elyra/metadata-extension", - "dist": "npm pack .", - "prepare": "npm run build ", - "test": "FORCE_COLOR=true jest", + "build:lib": "tsc --sourceMap", + "build:lib:prod": "tsc", + "clean": "jlpm clean:lib", + "clean:lib": "rimraf lib tsconfig.tsbuildinfo", + "clean:lintcache": "rimraf .eslintcache .stylelintcache", + "clean:labextension": "rimraf ../../../../labextensions/elyra_pipeline_editor_extension/labextension ../../../../labextensions/elyra_pipeline_editor_extension/_version.py", + "clean:all": "jlpm clean:lib && jlpm clean:labextension && jlpm clean:lintcache", + "eslint": "jlpm eslint:check --fix", + "eslint:check": "eslint . --cache --ext .ts,.tsx", + "install:extension": "jlpm build", + "lint": "jlpm stylelint && jlpm prettier && jlpm eslint", + "lint:check": "jlpm stylelint:check && jlpm prettier:check && jlpm eslint:check", + "prettier": "jlpm prettier:base --write --list-different", + "prettier:base": "prettier \"**/*{.ts,.tsx,.js,.jsx,.css,.json,.md}\"", + "prettier:check": "jlpm prettier:base --check", + "stylelint": "jlpm stylelint:check --fix", + "stylelint:check": "stylelint --cache \"style/**/*.css\"", + "test": "jest --coverage --passWithNoTests", "watch": "run-p watch:src watch:labextension", - "watch:src": "tsc -w", - "watch:labextension": "jupyter labextension watch .", - "lab:install": "jupyter labextension install --no-build", - "lab:uninstall": "jupyter labextension uninstall --no-build", - "link:dev": "yarn link @jupyterlab/builder", - "unlink:dev": "yarn unlink @jupyterlab/builder" + "watch:src": "tsc -w --sourceMap", + "watch:labextension": "jupyter labextension watch ." }, "dependencies": { "@elyra/metadata-common": "4.0.0-dev", @@ -50,20 +58,20 @@ "@elyra/pipeline-services": "1.12.1", "@elyra/services": "4.0.0-dev", "@elyra/ui-components": "4.0.0-dev", - "@jupyterlab/application": "^4.0.6", - "@jupyterlab/apputils": "^4.1.6", - "@jupyterlab/builder": "^4.0.6", + "@jupyterlab/application": "^4.2.5", + "@jupyterlab/apputils": "^4.2.5", + "@jupyterlab/builder": "^4.2.5", "@jupyterlab/coreutils": "^6.0.6", - "@jupyterlab/docregistry": "^4.0.6", - "@jupyterlab/filebrowser": "^4.0.6", - "@jupyterlab/filebrowser-extension": "^4.0.6", - "@jupyterlab/fileeditor": "^4.0.6", - "@jupyterlab/launcher": "^4.0.6", - "@jupyterlab/mainmenu": "^4.0.6", - "@jupyterlab/notebook": "^4.0.6", - "@jupyterlab/outputarea": "^4.0.6", + "@jupyterlab/docregistry": "^4.2.5", + "@jupyterlab/filebrowser": "^4.2.5", + "@jupyterlab/filebrowser-extension": "^4.2.5", + "@jupyterlab/fileeditor": "^4.2.5", + "@jupyterlab/launcher": "^4.2.5", + "@jupyterlab/mainmenu": "^4.2.5", + "@jupyterlab/notebook": "^4.2.5", + "@jupyterlab/outputarea": "^4.2.5", "@jupyterlab/services": "^7.0.6", - "@jupyterlab/ui-components": "^4.0.6", + "@jupyterlab/ui-components": "^4.2.5", "@lumino/algorithm": "*", "@lumino/coreutils": "^2.1.2", "@lumino/disposable": "^2.1.2", @@ -84,17 +92,17 @@ "uuid": "^3.4.0" }, "devDependencies": { - "@types/jest": "^23.3.11", + "@types/jest": "^29.2.0", "@types/node": "^12.0.10", "@types/react": "18.0.26", "@types/react-dom": "18.0.9", "@types/uuid": "^3.4.7", "cheerio": "^1.0.0-rc.3", - "jest": "^24.7.1", + "jest": "^29.2.0", "jest-raw-loader": "^1.0.1", "rimraf": "~5.0.5", "source-map-loader": "^0.2.4", - "ts-jest": "^24.0.2", + "ts-jest": "^29.2.5", "ts-loader": "^6.2.1", "typescript": "~5.1.6" }, @@ -107,6 +115,6 @@ "jupyterlab": { "extension": true, "schemaDir": "schema", - "outputDir": "../../build/labextensions/@elyra/pipeline-editor-extension" + "outputDir": "../../labextensions/elyra_pipeline_editor_extension/labextension" } } diff --git a/packages/pipeline-editor/schema/src/ComponentCatalogsWidget.tsx b/packages/pipeline-editor/schema/src/ComponentCatalogsWidget.tsx deleted file mode 100644 index c934203d4..000000000 --- a/packages/pipeline-editor/schema/src/ComponentCatalogsWidget.tsx +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - MetadataWidget, - IMetadataWidgetProps, - IMetadata, - MetadataDisplay, - IMetadataDisplayProps, - //IMetadataDisplayState, - IMetadataActionButton, -} from '@elyra/metadata-common'; -import { IDictionary, MetadataService } from '@elyra/services'; -import { RequestErrors } from '@elyra/ui-components'; -import { JupyterFrontEnd } from '@jupyterlab/application'; -import { LabIcon, refreshIcon } from '@jupyterlab/ui-components'; - -import React from 'react'; - -import { IRuntimeType, PipelineService } from './PipelineService'; - -export const COMPONENT_CATALOGS_SCHEMASPACE = 'component-catalogs'; - -const COMPONENT_CATALOGS_CLASS = 'elyra-metadata-component-catalogs'; - -const handleError = (error: any): void => { - // silently eat a 409, the server will log in in the console - if (error.status !== 409) { - RequestErrors.serverError(error); - } -}; - -/** - * A React Component for displaying the component catalogs list. - */ -class ComponentCatalogsDisplay extends MetadataDisplay { - actionButtons(metadata: IMetadata): IMetadataActionButton[] { - return [ - { - title: 'Reload components from catalog', - icon: refreshIcon, - onClick: (): void => { - PipelineService.refreshComponentsCache(metadata.name) - .then((response: any): void => { - this.props.updateMetadata(); - }) - .catch((error) => - console.error( - 'An error occurred while refreshing components from catalog:', - error, - ), - ); - }, - }, - ...super.actionButtons(metadata), - ]; - } - - //render catalog entries - renderExpandableContent(metadata: IDictionary): JSX.Element { - let category_output =
  • No category
    • ; - if (metadata.metadata.categories) { - category_output = metadata.metadata.categories.map((category: string) => ( -
  • {category}
    • - )); - } - - return ( -
    -
    Runtime Type
    - {metadata.metadata.runtime_type} -
    -
    -
    Description
    - {metadata.metadata.description ?? 'No description'} -
    -
    -
    Categories
    -
      {category_output}
    -
    - ); - } - - // Allow for filtering by display_name, name, and description - matchesSearch(searchValue: string, metadata: IMetadata): boolean { - searchValue = searchValue.toLowerCase(); - // True if search string is in name or display_name, - // or if the search string is empty - const description = (metadata.metadata.description || '').toLowerCase(); - return ( - metadata.name.toLowerCase().includes(searchValue) || - metadata.display_name.toLowerCase().includes(searchValue) || - description.includes(searchValue) - ); - } -} - -/** - * ComponentCatalogsWidget props. - */ -export interface IComponentCatalogsWidgetProps extends IMetadataWidgetProps { - app: JupyterFrontEnd; - display_name: string; - schemaspace: string; - icon: LabIcon; - titleContext?: string; - appendToTitle?: boolean; - refreshCallback?: () => void; -} - -/** - * A widget for displaying component catalogs. - */ -export class ComponentCatalogsWidget extends MetadataWidget { - refreshButtonTooltip: string; - refreshCallback?: () => void; - runtimeTypes: IRuntimeType[] = []; - - constructor(props: IComponentCatalogsWidgetProps) { - super(props); - this.refreshCallback = props.refreshCallback; - this.refreshButtonTooltip = - 'Refresh list and reload components from all catalogs'; - } - - async getSchemas(): Promise { - try { - const schemas = await MetadataService.getSchema(this.props.schemaspace); - this.runtimeTypes = await PipelineService.getRuntimeTypes(); - const sortedSchema = schemas.sort((a: any, b: any) => - a.title.localeCompare(b.title), - ); - this.schemas = sortedSchema.filter((schema: any) => { - return !!this.runtimeTypes.find( - (r) => - schema.properties?.metadata?.properties?.runtime_type?.enum?.includes( - r.id, - ) && r.runtime_enabled, - ); - }); - if (this.schemas?.length ?? 0 > 1) { - for (const schema of this.schemas ?? []) { - this.props.app.contextMenu.addItem({ - selector: `#${this.props.schemaspace} .elyra-metadataHeader-addButton`, - command: 'elyra-metadata-editor:open', - args: { - onSave: this.updateMetadata, - schemaspace: this.props.schemaspace, - schema: schema.name, - title: schema.title, - titleContext: this.props.titleContext, - appendToTitle: this.props.appendToTitle, - } as any, - }); - } - } - this.update(); - } catch (error) { - RequestErrors.serverError(error); - } - } - - // wrapper function that refreshes the palette after calling updateMetadata - updateMetadataAndRefresh = (): void => { - super.updateMetadata(); - if (this.refreshCallback) { - this.refreshCallback(); - } - }; - - refreshMetadata(): void { - PipelineService.refreshComponentsCache() - .then((response: any): void => { - this.updateMetadataAndRefresh(); - }) - .catch((error) => handleError(error)); - } - - renderDisplay(metadata: IMetadata[]): React.ReactElement { - if (Array.isArray(metadata) && !metadata.length) { - // Empty metadata - return ( -
    -
    -
    - Click the + button to add {this.props.display_name.toLowerCase()} -
    -
    - ); - } - - const filteredMetadata = metadata.filter((m) => { - return !!this.runtimeTypes.find((r) => m.metadata?.runtime_type === r.id); - }); - - return ( - - ); - } -} diff --git a/packages/pipeline-editor/schema/src/EmptyPipelineContent.tsx b/packages/pipeline-editor/schema/src/EmptyPipelineContent.tsx deleted file mode 100644 index 3af93d71d..000000000 --- a/packages/pipeline-editor/schema/src/EmptyPipelineContent.tsx +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { componentCatalogIcon, dragDropIcon } from '@elyra/ui-components'; -import { settingsIcon } from '@jupyterlab/ui-components'; - -import React from 'react'; - -const HEADER_CLASS = 'empty-pipeline-header'; -const BUTTON_CLASS = 'empty-pipeline-button'; -const ICON_CLASS = 'empty-pipeline-icon'; - -export interface IEmptyGenericPipelineProps { - onOpenSettings: () => void; -} - -export const EmptyGenericPipeline: React.FC = ({ - onOpenSettings, -}) => { - return ( -
    - -

    - Start your new pipeline by dragging files from the file browser pane -

    -
    -
    -

    - Click{' '} - {' '} - to configure the pipeline editor. -

    -
    - ); -}; - -export interface IEmptyPlatformSpecificPipelineProps { - onOpenCatalog: () => void; - onOpenSettings: () => void; -} - -export const EmptyPlatformSpecificPipeline: React.FC< - IEmptyPlatformSpecificPipelineProps -> = ({ onOpenCatalog, onOpenSettings }) => { - // Note: the URL is rewritten by the release script by replacing `latest` with a - // specific version number, e.g. https://.../en/v3.6.0/user_guide/pi... - const customComponentsHelpTopicURL = - 'https://elyra.readthedocs.io/en/latest/user_guide/pipeline-components.html'; - - return ( -
    - -

    - Start your new pipeline by dragging files from the file browser pane or - add custom components by clicking the{' '} - {' '} - button. -

    -

    - Refer to the - - {' '} - 'pipeline components' help topic{' '} - - for details. -

    -
    -
    -

    - Click{' '} - {' '} - to configure the pipeline editor. -

    -
    - ); -}; diff --git a/packages/pipeline-editor/schema/src/FileSubmissionDialog.tsx b/packages/pipeline-editor/schema/src/FileSubmissionDialog.tsx deleted file mode 100644 index 978af1112..000000000 --- a/packages/pipeline-editor/schema/src/FileSubmissionDialog.tsx +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { IDictionary } from '@elyra/services'; -import * as React from 'react'; - -import { IRuntimeData } from './runtime-utils'; -import RuntimeConfigSelect from './RuntimeConfigSelect'; -import Utils from './utils'; - -interface IProps { - env: string[]; - dependencyFileExtension: string; - images: IDictionary; - runtimeData: IRuntimeData; -} - -const EnvForm: React.FC<{ env: string[] }> = ({ env }) => { - if (env.length > 0) { - return ( - <> -
    -
    -
    Environment Variables:
    -
    - {Utils.chunkArray(env, 4).map((col, i) => ( -
    - {col.map((envVar) => ( -
    - -
    - -
    - ))} -
    - ))} - - ); - } - return null; -}; - -export const FileSubmissionDialog: React.FC = ({ - env, - images, - dependencyFileExtension, - runtimeData, -}) => { - const [includeDependency, setIncludeDependency] = React.useState(true); - - const handleToggle = (): void => { - setIncludeDependency((prev) => !prev); - }; - - return ( -
    - - -
    - -
    -
    -
    - -
    - For CPU-intensive workloads, you can choose more than 1 CPU (e.g. - 1.5). -
    - -
    -
    - -
    - For GPU-intensive workloads, you can choose more than 1 GPU. Must be - an integer. -
    - -
    -
    - -
    - The total amount of RAM specified. -
    - -
    -
    -
    - - -
    - {includeDependency && ( -
    -
    - -
    - )} - - - ); -}; diff --git a/packages/pipeline-editor/schema/src/ParameterInputForm.tsx b/packages/pipeline-editor/schema/src/ParameterInputForm.tsx deleted file mode 100644 index 45306a1f4..000000000 --- a/packages/pipeline-editor/schema/src/ParameterInputForm.tsx +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import React from 'react'; - -const DIALOG_WIDTH = 27; - -export interface IParameterProps { - parameters?: { - name: string; - default_value?: { - type: 'String' | 'Integer' | 'Float' | 'Bool'; - value: any; - }; - type?: string; - required?: boolean; - description?: string; - }[]; -} - -export const ParameterInputForm: React.FC = ({ - parameters, -}) => { - return parameters && parameters.length > 0 ? ( -
    - - {parameters.map((param) => { - if (!param.name) { - return undefined; - } - const required = - param.required === true && param.default_value?.value === '' - ? true - : undefined; - let type = 'text'; - switch (param.default_value?.type) { - case 'Bool': - type = 'checkbox'; - break; - case 'Float': - case 'Integer': - type = 'number'; - break; - } - if (type === 'checkbox') { - return ( -
    - - -
    -
    -
    - ); - } - return ( -
    -
    - - {param.description && ( -
    -
    ?
    -

    - {param.description} -

    -
    - )} -
    - -
    -
    -
    - ); - })} -
    - ) : ( -
    - ); -}; diff --git a/packages/pipeline-editor/schema/src/PipelineEditorWidget.tsx b/packages/pipeline-editor/schema/src/PipelineEditorWidget.tsx deleted file mode 100644 index 4e8a37b47..000000000 --- a/packages/pipeline-editor/schema/src/PipelineEditorWidget.tsx +++ /dev/null @@ -1,1215 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - PipelineEditor, - PipelineOutOfDateError, - ThemeProvider, -} from '@elyra/pipeline-editor'; -import { - migrate, - validate, - ComponentNotFoundError, -} from '@elyra/pipeline-services'; -import { ContentParser } from '@elyra/services'; -import { - IconUtil, - clearPipelineIcon, - exportPipelineIcon, - pipelineIcon, - savePipelineIcon, - showBrowseFileDialog, - runtimesIcon, - containerIcon, - Dropzone, - RequestErrors, - showFormDialog, - componentCatalogIcon, -} from '@elyra/ui-components'; -import { ILabShell } from '@jupyterlab/application'; -import { Dialog, ReactWidget, showDialog } from '@jupyterlab/apputils'; -import { PathExt } from '@jupyterlab/coreutils'; -import { - DocumentRegistry, - ABCWidgetFactory, - DocumentWidget, - Context, -} from '@jupyterlab/docregistry'; -import { IFileBrowserFactory } from '@jupyterlab/filebrowser'; -import { ISettingRegistry } from '@jupyterlab/settingregistry'; - -import 'carbon-components/css/carbon-components.min.css'; - -import { toArray } from '@lumino/algorithm'; -import { IDragEvent } from '@lumino/dragdrop'; -import { Signal } from '@lumino/signaling'; - -import React, { useCallback, useEffect, useRef, useState } from 'react'; -import { ToastContainer, toast } from 'react-toastify'; -import 'react-toastify/dist/ReactToastify.css'; - -import { - EmptyGenericPipeline, - EmptyPlatformSpecificPipeline, -} from './EmptyPipelineContent'; -import { formDialogWidget } from './formDialogWidget'; -import { - usePalette, - useRuntimeImages, - useRuntimesSchema, -} from './pipeline-hooks'; -import { PipelineExportDialog } from './PipelineExportDialog'; -import { - PipelineService, - RUNTIMES_SCHEMASPACE, - RUNTIME_IMAGES_SCHEMASPACE, - COMPONENT_CATALOGS_SCHEMASPACE, -} from './PipelineService'; -import { PipelineSubmissionDialog } from './PipelineSubmissionDialog'; -import { - createRuntimeData, - getConfigDetails, - IRuntimeData, -} from './runtime-utils'; -import { theme } from './theme'; - -const PIPELINE_CLASS = 'elyra-PipelineEditor'; - -export const commandIDs = { - openPipelineEditor: 'pipeline-editor:open', - openMetadata: 'elyra-metadata:open', - openDocManager: 'docmanager:open', - newDocManager: 'docmanager:new-untitled', - saveDocManager: 'docmanager:save', - submitScript: 'script-editor:submit', - submitNotebook: 'notebook:submit', - addFileToPipeline: 'pipeline-editor:add-node', - refreshPalette: 'pipeline-editor:refresh-palette', - openViewer: 'elyra-code-viewer:open', -}; - -const getAllPaletteNodes = (palette: any): any[] => { - if (palette.categories === undefined) { - return []; - } - - const nodes = []; - for (const c of palette.categories) { - if (c.node_types) { - nodes.push(...c.node_types); - } - } - - return nodes; -}; - -const isRuntimeTypeAvailable = (data: IRuntimeData, type?: string): boolean => { - for (const p of data.platforms) { - if (type === undefined || p.id === type) { - if (p.configs.length > 0) { - return true; - } - } - } - return false; -}; - -const getDisplayName = ( - runtimesSchema: any, - type?: string, -): string | undefined => { - if (!type) { - return undefined; - } - const schema = runtimesSchema?.find((s: any) => s.runtime_type === type); - return schema?.title; -}; - -class PipelineEditorWidget extends ReactWidget { - browserFactory: IFileBrowserFactory; - shell: ILabShell; - commands: any; - addFileToPipelineSignal: Signal; - refreshPaletteSignal: Signal; - context: Context; - settings: ISettingRegistry.ISettings; - - constructor(options: any) { - super(options); - this.browserFactory = options.browserFactory; - this.shell = options.shell; - this.commands = options.commands; - this.addFileToPipelineSignal = options.addFileToPipelineSignal; - this.refreshPaletteSignal = options.refreshPaletteSignal; - this.context = options.context; - this.settings = options.settings; - let nullPipeline = this.context.model.toJSON() === null; - this.context.model.contentChanged.connect(() => { - if (nullPipeline) { - nullPipeline = false; - this.update(); - } - }); - } - - render(): any { - if (this.context.model.toJSON() === null) { - return
    ; - } - return ( - - ); - } -} - -interface IProps { - context: DocumentRegistry.Context; - browserFactory: IFileBrowserFactory; - shell: ILabShell; - commands: any; - addFileToPipelineSignal: Signal; - refreshPaletteSignal: Signal; - settings?: ISettingRegistry.ISettings; - widgetId?: string; -} - -const PipelineWrapper: React.FC = ({ - context, - browserFactory, - shell, - commands, - addFileToPipelineSignal, - refreshPaletteSignal, - settings, - widgetId, -}) => { - const ref = useRef(null); - const [loading, setLoading] = useState(true); - const [pipeline, setPipeline] = useState(context.model.toJSON()); - const [panelOpen, setPanelOpen] = React.useState(false); - - const type: string | undefined = - pipeline?.pipelines?.[0]?.app_data?.runtime_type; - - const { data: runtimesSchema, error: runtimesSchemaError } = - useRuntimesSchema(); - - const doubleClickToOpenProperties = - settings?.composite['doubleClickToOpenProperties'] ?? true; - - const runtimeDisplayName = getDisplayName(runtimesSchema, type) ?? 'Generic'; - - const { - data: palette, - error: paletteError, - mutate: mutatePalette, - } = usePalette(type); - - useEffect(() => { - const handleMutateSignal = (): void => { - mutatePalette(); - }; - refreshPaletteSignal.connect(handleMutateSignal); - return (): void => { - refreshPaletteSignal.disconnect(handleMutateSignal); - }; - }, [refreshPaletteSignal, mutatePalette]); - - const { data: runtimeImages, error: runtimeImagesError } = useRuntimeImages(); - - useEffect(() => { - if (runtimeImages?.length === 0) { - RequestErrors.noMetadataError('runtime image'); - } - }, [runtimeImages?.length]); - - useEffect(() => { - if (paletteError) { - RequestErrors.serverError(paletteError); - shell.currentWidget?.close(); - } - }, [paletteError, shell.currentWidget]); - - useEffect(() => { - if (runtimeImagesError) { - RequestErrors.serverError(runtimeImagesError); - shell.currentWidget?.close(); - } - }, [runtimeImagesError, shell.currentWidget]); - - useEffect(() => { - if (runtimesSchemaError) { - RequestErrors.serverError(runtimesSchemaError); - shell.currentWidget?.close(); - } - }, [runtimesSchemaError, shell.currentWidget]); - - const contextRef = useRef(context); - useEffect(() => { - const currentContext = contextRef.current; - - const changeHandler = (): void => { - const pipelineJson: any = currentContext.model.toJSON(); - - // map IDs to display names - const nodes = pipelineJson?.pipelines?.[0]?.nodes; - if (nodes?.length > 0) { - for (const node of nodes) { - if (node?.app_data?.component_parameters) { - for (const [key, val] of Object.entries( - node?.app_data?.component_parameters, - )) { - if (val === null) { - node.app_data.component_parameters[key] = undefined; - } - } - } - } - } - // TODO: don't persist this, but this will break things right now - if (pipelineJson?.pipelines?.[0]?.app_data) { - if (!pipelineJson.pipelines[0].app_data.properties) { - pipelineJson.pipelines[0].app_data.properties = {}; - } - const pipeline_path = contextRef.current.path; - const pipeline_name = PathExt.basename( - pipeline_path, - PathExt.extname(pipeline_path), - ); - pipelineJson.pipelines[0].app_data.properties.name = pipeline_name; - pipelineJson.pipelines[0].app_data.properties.runtime = - runtimeDisplayName; - } - setPipeline(pipelineJson); - setLoading(false); - }; - - currentContext.ready.then(changeHandler); - currentContext.model.contentChanged.connect(changeHandler); - - return (): void => { - currentContext.model.contentChanged.disconnect(changeHandler); - }; - }, [runtimeDisplayName]); - - const onChange = useCallback((pipelineJson: any): void => { - const removeNullValues = (data: any, removeEmptyString?: boolean): void => { - for (const key in data) { - if ( - data[key] === null || - data[key] === undefined || - (removeEmptyString && data[key] === '') - ) { - delete data[key]; - } else if (Array.isArray(data[key])) { - const newArray = []; - for (const i in data[key]) { - if (typeof data[key][i] === 'object') { - removeNullValues(data[key][i], true); - if (Object.keys(data[key][i]).length > 0) { - newArray.push(data[key][i]); - } - } else if (data[key][i] !== null && data[key][i] !== '') { - newArray.push(data[key][i]); - } - } - data[key] = newArray; - } else if (typeof data[key] === 'object') { - removeNullValues(data[key]); - } - } - }; - - // Remove all null values from the pipeline - for (const node of pipelineJson?.pipelines?.[0]?.nodes ?? []) { - removeNullValues(node.app_data ?? {}); - } - removeNullValues( - pipelineJson?.pipelines?.[0]?.app_data?.properties?.pipeline_defaults ?? - {}, - ); - if (contextRef.current.isReady) { - contextRef.current.model.fromString( - JSON.stringify(pipelineJson, null, 2), - ); - } - }, []); - - const isDialogAlreadyShowing = useRef(false); - const onError = useCallback( - (error?: Error): void => { - if (isDialogAlreadyShowing.current) { - return; // bail, we are already showing a dialog. - } - isDialogAlreadyShowing.current = true; - if (error instanceof PipelineOutOfDateError) { - showDialog({ - title: 'Migrate pipeline?', - body: ( -

    - This pipeline corresponds to an older version of Elyra and needs - to be migrated. -
    - Although the pipeline can be further edited and/or submitted after - its update, -
    - the migration will not be completed until the pipeline has been - saved within the editor. -
    -
    - Proceed with migration? -

    - ), - buttons: [Dialog.cancelButton(), Dialog.okButton()], - }).then(async (result) => { - isDialogAlreadyShowing.current = false; - if (result.button.accept) { - // proceed with migration - console.log('migrating pipeline'); - const pipelineJSON: any = contextRef.current.model.toJSON(); - try { - const migratedPipeline = migrate(pipelineJSON, (pipeline) => { - // function for updating to relative paths in v2 - // uses location of filename as expected in v1 - for (const node of pipeline.nodes) { - node.app_data.filename = - PipelineService.getPipelineRelativeNodePath( - contextRef.current.path, - node.app_data.filename, - ); - } - return pipeline; - }); - contextRef.current.model.fromString( - JSON.stringify(migratedPipeline, null, 2), - ); - } catch (migrationError) { - if (migrationError instanceof ComponentNotFoundError) { - showDialog({ - title: 'Pipeline migration aborted!', - body: ( -

    - {' '} - The pipeline you are trying to migrate uses example - components, which are not
    - enabled in your environment. Complete the setup - instructions in{' '} - - Example Custom Components - {' '} - and try again. -

    - ), - buttons: [Dialog.okButton({ label: 'Close' })], - }).then(() => { - shell.currentWidget?.close(); - }); - } else { - showDialog({ - title: 'Pipeline migration failed!', - body:

    {migrationError?.message || ''}

    , - buttons: [Dialog.okButton()], - }).then(() => { - shell.currentWidget?.close(); - }); - } - } - } else { - shell.currentWidget?.close(); - } - }); - } else { - showDialog({ - title: 'Load pipeline failed!', - body:

    {error?.message || ''}

    , - buttons: [Dialog.okButton()], - }).then(() => { - isDialogAlreadyShowing.current = false; - shell.currentWidget?.close(); - }); - } - }, - [shell.currentWidget], - ); - - const onFileRequested = async (args: any): Promise => { - const filename = PipelineService.getWorkspaceRelativeNodePath( - contextRef.current.path, - args.filename ?? '', - ); - if (args.propertyID.includes('dependencies')) { - const res = await showBrowseFileDialog( - browserFactory.defaultBrowser.model.manager, - { - multiselect: true, - includeDir: true, - rootPath: PathExt.dirname(filename), - filter: (model: any): boolean => { - return model.path !== filename; - }, - }, - ); - - if (res.button.accept && res.value.length) { - return res.value.map((v: any) => v.path); - } - } else { - const res = await showBrowseFileDialog( - browserFactory.defaultBrowser.model.manager, - { - startPath: PathExt.dirname(filename), - filter: (model: any): boolean => { - if (args.filters?.File === undefined) { - return true; - } - - const ext = PathExt.extname(model.path); - return args.filters.File.includes(ext); - }, - }, - ); - - if (res.button.accept && res.value.length) { - const file = PipelineService.getPipelineRelativeNodePath( - contextRef.current.path, - res.value[0].path, - ); - return [file]; - } - } - - return undefined; - }; - - const onPropertiesUpdateRequested = async (args: any): Promise => { - if (!contextRef.current.path) { - return args; - } - const path = PipelineService.getWorkspaceRelativeNodePath( - contextRef.current.path, - args.component_parameters.filename, - ); - const new_env_vars = await ContentParser.getEnvVars(path).then( - (response: any) => - response.map((str: string) => { - return { env_var: str }; - }), - ); - - const env_vars = args.component_parameters?.env_vars ?? []; - const merged_env_vars = [ - ...env_vars, - ...new_env_vars.filter( - (new_var: any) => - !env_vars.some((old_var: any) => { - return old_var.env_var === new_var.env_var; - }), - ), - ]; - - return { - ...args, - component_parameters: { - ...args.component_parameters, - env_vars: merged_env_vars.filter(Boolean), - }, - }; - }; - - const handleOpenComponentDef = useCallback( - (componentId: string, componentSource: string) => { - // Show error dialog if the component does not exist - if (!componentId) { - const dialogBody = []; - try { - const componentSourceJson = JSON.parse(componentSource); - dialogBody.push(`catalog_type: ${componentSourceJson.catalog_type}`); - for (const [key, value] of Object.entries( - componentSourceJson.component_ref, - )) { - dialogBody.push(`${key}: ${value}`); - } - } catch { - dialogBody.push(componentSource); - } - return showDialog({ - title: 'Component not found', - body: ( -

    - This node uses a component that is not stored in your component - registry. - {dialogBody.map((line, i) => ( - -
    - {line} -     - ))} -
    -
    - - Learn more... - -

    - ), - buttons: [Dialog.okButton()], - }); - } - return PipelineService.getComponentDef(type, componentId) - .then((res) => { - const nodeDef = getAllPaletteNodes(palette).find( - (n) => n.id === componentId, - ); - commands.execute(commandIDs.openViewer, { - content: res.content, - mimeType: res.mimeType, - label: nodeDef?.label ?? componentId, - }); - }) - .catch((e) => RequestErrors.serverError(e)); - }, - [commands, palette, type], - ); - - const onDoubleClick = (data: any): void => { - for (let i = 0; i < data.selectedObjectIds.length; i++) { - const node = pipeline.pipelines[0].nodes.find( - (node: any) => node.id === data.selectedObjectIds[i], - ); - const nodeDef = getAllPaletteNodes(palette).find( - (n) => n.op === node?.op, - ); - if (node?.app_data?.component_parameters?.filename) { - commands.execute(commandIDs.openDocManager, { - path: PipelineService.getWorkspaceRelativeNodePath( - contextRef.current.path, - node.app_data.component_parameters.filename, - ), - }); - } else if (!nodeDef?.app_data?.parameter_refs?.['filehandler']) { - handleOpenComponentDef(nodeDef?.id, node?.app_data?.component_source); - } - } - }; - - const handleSubmission = useCallback( - async (actionType: 'run' | 'export'): Promise => { - const pipelineJson: any = context.model.toJSON(); - // Check that all nodes are valid - const errorMessages = validate( - JSON.stringify(pipelineJson), - getAllPaletteNodes(palette), - palette.properties, - ); - if (errorMessages && errorMessages.length > 0) { - let errorMessage = ''; - for (const error of errorMessages) { - errorMessage += (errorMessage ? '\n' : '') + error.message; - } - toast.error(`Failed ${actionType}: ${errorMessage}`); - return; - } - - if (contextRef.current.model.dirty) { - const dialogResult = await showDialog({ - title: - 'This pipeline contains unsaved changes. To submit the pipeline the changes need to be saved.', - buttons: [ - Dialog.cancelButton(), - Dialog.okButton({ label: 'Save and Submit' }), - ], - }); - if (dialogResult.button && dialogResult.button.accept === true) { - await contextRef.current.save(); - } else { - // Don't proceed if cancel button pressed - return; - } - } - - const pipelineName = PathExt.basename( - contextRef.current.path, - PathExt.extname(contextRef.current.path), - ); - - // TODO: Parallelize this - const runtimeTypes = await PipelineService.getRuntimeTypes(); - const runtimes = await PipelineService.getRuntimes() - .then((runtimeList) => { - return runtimeList.filter((runtime: any) => { - return ( - !runtime.metadata.runtime_enabled && - !!runtimeTypes.find( - (r: any) => runtime.metadata.runtime_type === r.id, - ) - ); - }); - }) - .catch((error) => RequestErrors.serverError(error)); - const schema = await PipelineService.getRuntimesSchema().catch((error) => - RequestErrors.serverError(error), - ); - - const runtimeData = createRuntimeData({ - schema, - runtimes, - allowLocal: actionType === 'run', - }); - - let title = - type !== undefined - ? `${actionType} pipeline for ${runtimeDisplayName}` - : `${actionType} pipeline`; - - if (actionType === 'export' || type !== undefined) { - if (!isRuntimeTypeAvailable(runtimeData, type)) { - const res = await RequestErrors.noMetadataError( - 'runtime', - `${actionType} pipeline.`, - type !== undefined ? runtimeDisplayName : undefined, - ); - - if (res.button.label.includes(RUNTIMES_SCHEMASPACE)) { - // Open the runtimes widget - shell.activateById(`elyra-metadata:${RUNTIMES_SCHEMASPACE}`); - } - return; - } - } - // Capitalize - title = title.charAt(0).toUpperCase() + title.slice(1); - - let dialogOptions: Partial>; - - pipelineJson.pipelines[0].app_data.properties.pipeline_parameters = - pipelineJson.pipelines[0].app_data.properties.pipeline_parameters?.filter( - (param: any) => { - return !!pipelineJson.pipelines[0].nodes.find((node: any) => { - return ( - param.name !== '' && - (node.app_data.component_parameters?.pipeline_parameters?.includes( - param.name, - ) || - Object.values(node.app_data.component_parameters ?? {}).find( - (property: any) => - property.widget === 'parameter' && - property.value === param.name, - )) - ); - }); - }, - ); - - const parameters = - pipelineJson?.pipelines[0].app_data.properties.pipeline_parameters; - - switch (actionType) { - case 'run': - dialogOptions = { - title, - body: formDialogWidget( - , - ), - buttons: [Dialog.cancelButton(), Dialog.okButton()], - defaultButton: 1, - focusNodeSelector: '#pipeline_name', - }; - break; - case 'export': - dialogOptions = { - title, - body: formDialogWidget( - , - ), - buttons: [Dialog.cancelButton(), Dialog.okButton()], - defaultButton: 1, - focusNodeSelector: '#runtime_config', - }; - break; - } - - const dialogResult = await showFormDialog(dialogOptions); - - if (dialogResult.value === null) { - // When Cancel is clicked on the dialog, just return - return; - } - - // Clean null properties - for (const node of pipelineJson.pipelines[0].nodes) { - if (node.app_data.component_parameters.cpu === null) { - delete node.app_data.component_parameters.cpu; - } - if (node.app_data.component_parameters.memory === null) { - delete node.app_data.component_parameters.memory; - } - if (node.app_data.component_parameters.gpu === null) { - delete node.app_data.component_parameters.gpu; - } - } - - const configDetails = getConfigDetails( - runtimeData, - dialogResult.value.runtime_config, - ); - - PipelineService.setNodePathsRelativeToWorkspace( - pipelineJson.pipelines[0], - getAllPaletteNodes(palette), - contextRef.current.path, - ); - - // Metadata - pipelineJson.pipelines[0].app_data.name = - dialogResult.value.pipeline_name ?? pipelineName; - pipelineJson.pipelines[0].app_data.source = PathExt.basename( - contextRef.current.path, - ); - - // Pipeline parameter overrides - for (const paramIndex in parameters ?? []) { - const param = parameters[paramIndex]; - if (param.name) { - let paramOverride = dialogResult.value[`${param.name}-paramInput`]; - if ( - (param.default_value?.type === 'Integer' || - param.default_value?.type === 'Float') && - paramOverride !== '' - ) { - paramOverride = Number(paramOverride); - } - pipelineJson.pipelines[0].app_data.properties.pipeline_parameters[ - paramIndex - ].value = - paramOverride === '' ? param.default_value?.value : paramOverride; - } - } - - // Pipeline name - pipelineJson.pipelines[0].app_data.name = - dialogResult.value.pipeline_name ?? pipelineName; - - // Runtime info - pipelineJson.pipelines[0].app_data.runtime_config = - configDetails?.id ?? null; - - // Export info - const pipeline_dir = PathExt.dirname(contextRef.current.path); - const basePath = pipeline_dir ? `${pipeline_dir}/` : ''; - const exportType = dialogResult.value.pipeline_filetype; - const exportName = dialogResult.value.export_name; - const exportPath = `${basePath}${exportName}.${exportType}`; - - switch (actionType) { - case 'run': - PipelineService.submitPipeline( - pipelineJson, - configDetails?.platform.displayName ?? '', - ).catch((error) => RequestErrors.serverError(error)); - break; - case 'export': - PipelineService.exportPipeline( - pipelineJson, - exportType, - exportPath, - dialogResult.value.overwrite, - ).catch((error) => RequestErrors.serverError(error)); - break; - } - }, - [context.model, palette, runtimeDisplayName, type, shell], - ); - - const handleClearPipeline = useCallback(async (data: any): Promise => { - return showDialog({ - title: 'Clear Pipeline', - body: 'Are you sure you want to clear the pipeline?', - buttons: [ - Dialog.cancelButton(), - Dialog.okButton({ label: 'Clear All' }), - Dialog.okButton({ label: 'Clear Canvas' }), - ], - }).then((result) => { - if (result.button.accept) { - const newPipeline: any = contextRef.current.model.toJSON(); - if (newPipeline?.pipelines?.[0]?.nodes?.length > 0) { - newPipeline.pipelines[0].nodes = []; - } - // remove supernode pipelines - newPipeline.pipelines = [newPipeline.pipelines[0]]; - // only clear pipeline properties when "Clear All" is selected - if (result.button.label === 'Clear All') { - const pipelineProperties = - newPipeline?.pipelines?.[0]?.app_data?.properties; - if (pipelineProperties) { - // Remove all fields of pipeline properties except for the name/runtime (readonly) - newPipeline.pipelines[0].app_data.properties = { - name: pipelineProperties.name, - runtime: pipelineProperties.runtime, - }; - } - } - contextRef.current.model.fromJSON(newPipeline); - } - }); - }, []); - - const onAction = useCallback( - (args: { type: string; payload?: any }) => { - switch (args.type) { - case 'save': - contextRef.current.save(); - break; - case 'run': - case 'export': - handleSubmission(args.type); - break; - case 'clear': - handleClearPipeline(args.payload); - break; - case 'toggleOpenPanel': - setPanelOpen(!panelOpen); - break; - case 'properties': - setPanelOpen(true); - break; - case 'openRuntimes': - shell.activateById(`elyra-metadata:${RUNTIMES_SCHEMASPACE}`); - break; - case 'openRuntimeImages': - shell.activateById(`elyra-metadata:${RUNTIME_IMAGES_SCHEMASPACE}`); - break; - case 'openComponentCatalogs': - shell.activateById( - `elyra-metadata:${COMPONENT_CATALOGS_SCHEMASPACE}`, - ); - break; - case 'openFile': - commands.execute(commandIDs.openDocManager, { - path: PipelineService.getWorkspaceRelativeNodePath( - contextRef.current.path, - args.payload, - ), - }); - break; - case 'openComponentDef': - handleOpenComponentDef( - args.payload.componentId, - args.payload.componentSource, - ); - break; - default: - break; - } - }, - [ - handleSubmission, - handleClearPipeline, - panelOpen, - shell, - commands, - handleOpenComponentDef, - ], - ); - - const toolbar = { - leftBar: [ - { - action: 'run', - label: 'Run Pipeline', - enable: true, - }, - { - action: 'save', - label: 'Save Pipeline', - enable: true, - iconEnabled: IconUtil.encode(savePipelineIcon), - iconDisabled: IconUtil.encode(savePipelineIcon), - }, - { - action: 'export', - label: 'Export Pipeline', - enable: true, - iconEnabled: IconUtil.encode(exportPipelineIcon), - iconDisabled: IconUtil.encode(exportPipelineIcon), - }, - { - action: 'clear', - label: 'Clear Pipeline', - enable: true, - iconEnabled: IconUtil.encode(clearPipelineIcon), - iconDisabled: IconUtil.encode(clearPipelineIcon), - }, - { - action: 'openRuntimes', - label: 'Open Runtimes', - enable: true, - iconEnabled: IconUtil.encode(runtimesIcon), - iconDisabled: IconUtil.encode(runtimesIcon), - }, - { - action: 'openRuntimeImages', - label: 'Open Runtime Images', - enable: true, - iconEnabled: IconUtil.encode(containerIcon), - iconDisabled: IconUtil.encode(containerIcon), - }, - { - action: 'openComponentCatalogs', - label: 'Open Component Catalogs', - enable: true, - iconEnabled: IconUtil.encode(componentCatalogIcon), - iconDisabled: IconUtil.encode(componentCatalogIcon), - }, - { action: 'undo', label: 'Undo' }, - { action: 'redo', label: 'Redo' }, - { action: 'cut', label: 'Cut' }, - { action: 'copy', label: 'Copy' }, - { action: 'paste', label: 'Paste' }, - { action: 'createAutoComment', label: 'Add Comment', enable: true }, - { action: 'deleteSelectedObjects', label: 'Delete' }, - { - action: 'arrangeHorizontally', - label: 'Arrange Horizontally', - enable: true, - }, - { - action: 'arrangeVertically', - label: 'Arrange Vertically', - enable: true, - }, - ], - rightBar: [ - { - action: '', - label: `Runtime: ${runtimeDisplayName}`, - incLabelWithIcon: 'before', - enable: false, - kind: 'tertiary', - // TODO: re-add icon - // iconEnabled: IconUtil.encode(ICON_MAP[type ?? ''] ?? pipelineIcon) - }, - { - action: 'toggleOpenPanel', - label: panelOpen ? 'Close Panel' : 'Open Panel', - enable: true, - iconTypeOverride: panelOpen ? 'paletteOpen' : 'paletteClose', - }, - ], - }; - - const [defaultPosition, setDefaultPosition] = useState(10); - - const handleAddFileToPipeline = useCallback( - (location?: { x: number; y: number }) => { - const fileBrowser = browserFactory.defaultBrowser; - // Only add file to pipeline if it is currently in focus - if (shell.currentWidget?.id !== widgetId) { - return; - } - - let failedAdd = 0; - let position = 0; - const missingXY = !location; - - // if either x or y is undefined use the default coordinates - if (missingXY) { - position = defaultPosition; - location = { - x: 75, - y: 85, - }; - } - - toArray(fileBrowser.selectedItems()).map((item: any): void => { - if (PipelineService.isSupportedNode(item)) { - item.op = PipelineService.getNodeType(item.path); - item.path = PipelineService.getPipelineRelativeNodePath( - contextRef.current.path, - item.path, - ); - item.x = (location?.x ?? 0) + position; - item.y = (location?.y ?? 0) + position; - - const success = ref.current?.addFile({ - nodeTemplate: { - op: item.op, - }, - offsetX: item.x, - offsetY: item.y, - path: item.path, - }); - - if (success) { - position += 20; - } else { - // handle error - } - } else { - failedAdd++; - } - }); - // update position if the default coordinates were used - if (missingXY) { - setDefaultPosition(position); - } - - if (failedAdd) { - return showDialog({ - title: 'Unsupported File(s)', - body: 'Only supported files (Notebooks, Python scripts, and R scripts) can be added to a pipeline.', - buttons: [Dialog.okButton()], - }); - } - - return; - }, - [browserFactory.defaultBrowser, defaultPosition, shell, widgetId], - ); - - const handleDrop = async (e: IDragEvent): Promise => { - handleAddFileToPipeline({ x: e.offsetX, y: e.offsetY }); - }; - - useEffect(() => { - const handleSignal = (): void => { - handleAddFileToPipeline(); - }; - addFileToPipelineSignal.connect(handleSignal); - return (): void => { - addFileToPipelineSignal.disconnect(handleSignal); - }; - }, [addFileToPipelineSignal, handleAddFileToPipeline]); - - if (loading || palette === undefined) { - return
    ; - } - - const handleOpenCatalog = (): void => { - shell.activateById(`elyra-metadata:${COMPONENT_CATALOGS_SCHEMASPACE}`); - }; - - const handleOpenSettings = (): void => { - commands.execute('settingeditor:open', { query: 'Pipeline Editor' }); - }; - - return ( - - - - - {type === undefined ? ( - - ) : ( - - )} - - - - ); -}; - -export class PipelineEditorFactory extends ABCWidgetFactory { - browserFactory: IFileBrowserFactory; - shell: ILabShell; - commands: any; - addFileToPipelineSignal: Signal; - refreshPaletteSignal: Signal; - settings: ISettingRegistry.ISettings; - - constructor(options: any) { - super(options); - this.browserFactory = options.browserFactory; - this.shell = options.shell; - this.commands = options.commands; - this.addFileToPipelineSignal = new Signal(this); - this.refreshPaletteSignal = new Signal(this); - this.settings = options.settings; - } - - protected createNewWidget(context: DocumentRegistry.Context): DocumentWidget { - // Creates a blank widget with a DocumentWidget wrapper - const props = { - shell: this.shell, - commands: this.commands, - browserFactory: this.browserFactory, - context: context, - addFileToPipelineSignal: this.addFileToPipelineSignal, - refreshPaletteSignal: this.refreshPaletteSignal, - settings: this.settings, - }; - const content = new PipelineEditorWidget(props); - - const widget = new DocumentWidget({ content, context }); - widget.addClass(PIPELINE_CLASS); - widget.title.icon = pipelineIcon; - return widget; - } -} diff --git a/packages/pipeline-editor/schema/src/PipelineExportDialog.tsx b/packages/pipeline-editor/schema/src/PipelineExportDialog.tsx deleted file mode 100644 index ca5ccddf4..000000000 --- a/packages/pipeline-editor/schema/src/PipelineExportDialog.tsx +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as React from 'react'; - -import { IParameterProps, ParameterInputForm } from './ParameterInputForm'; - -import { IRuntimeType } from './PipelineService'; -import { IRuntimeData } from './runtime-utils'; -import RuntimeConfigSelect from './RuntimeConfigSelect'; - -interface IFileTypeSelectProps { - fileTypes: { display_name: string; id: string }[]; -} - -const FileTypeSelect: React.FC = ({ fileTypes }) => { - return ( - <> - -
    - - - ); -}; - -interface IProps extends IParameterProps { - runtimeData: IRuntimeData; - runtimeTypeInfo: IRuntimeType[]; - pipelineType?: string; - exportName: string; -} - -export const PipelineExportDialog: React.FC = ({ - runtimeData, - runtimeTypeInfo, - pipelineType, - exportName, - parameters, -}) => { - return ( -
    - - {(platform): JSX.Element => { - const info = runtimeTypeInfo.find((i) => i.id === platform); - return ; - }} - - -
    - -
    -
    - - -
    -
    - - - ); -}; diff --git a/packages/pipeline-editor/schema/src/PipelineService.tsx b/packages/pipeline-editor/schema/src/PipelineService.tsx deleted file mode 100644 index be220e548..000000000 --- a/packages/pipeline-editor/schema/src/PipelineService.tsx +++ /dev/null @@ -1,359 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { MetadataService, IDictionary, RequestHandler } from '@elyra/services'; -import { RequestErrors } from '@elyra/ui-components'; - -import { showDialog, Dialog } from '@jupyterlab/apputils'; -import { PathExt } from '@jupyterlab/coreutils'; - -import * as React from 'react'; - -export const KFP_SCHEMA = 'kfp'; -export const RUNTIMES_SCHEMASPACE = 'runtimes'; -export const RUNTIME_IMAGES_SCHEMASPACE = 'runtime-images'; -export const COMPONENT_CATALOGS_SCHEMASPACE = 'component-catalogs'; - -export interface IRuntime { - name: string; - display_name: string; - schema_name: string; - metadata: { - runtime_type: string; - }; -} - -export interface ISchema { - name: string; - title: string; - runtime_type: string; -} - -interface IComponentDef { - content: string; - mimeType: string; -} - -enum ContentType { - notebook = 'execute-notebook-node', - python = 'execute-python-node', - r = 'execute-r-node', - other = 'other', -} - -const CONTENT_TYPE_MAPPER: Map = new Map([ - ['.py', ContentType.python], - ['.ipynb', ContentType.notebook], - ['.r', ContentType.r], -]); - -export interface IRuntimeType { - runtime_enabled: boolean; - id: string; - display_name: string; - icon: string; - export_file_types: { id: string; display_name: string }[]; -} - -export class PipelineService { - /** - * Returns a list of resources corresponding to each active runtime-type. - */ - static async getRuntimeTypes(): Promise { - const res = await RequestHandler.makeGetRequest( - 'elyra/pipeline/runtimes/types', - ); - return res.runtime_types.sort((a: any, b: any) => a.id.localeCompare(b.id)); - } - - /** - * Returns a list of external runtime configurations available as - * `runtimes metadata`. This is used to submit the pipeline to be - * executed on these runtimes. - */ - static async getRuntimes(): Promise { - return MetadataService.getMetadata(RUNTIMES_SCHEMASPACE); - } - - /** - * Returns a list of runtime schema - */ - static async getRuntimesSchema(showError = true): Promise { - return MetadataService.getSchema(RUNTIMES_SCHEMASPACE).then((schema) => { - if (showError && Object.keys(schema).length === 0) { - return RequestErrors.noMetadataError('schema'); - } - - return schema; - }); - } - - /** - * Return a list of configured container images that are used as runtimes environments - * to run the pipeline nodes. - */ - static async getRuntimeImages(): Promise { - try { - let runtimeImages = await MetadataService.getMetadata('runtime-images'); - - runtimeImages = runtimeImages.sort( - (a: any, b: any) => 0 - (a.name > b.name ? -1 : 1), - ); - - if (Object.keys(runtimeImages).length === 0) { - return RequestErrors.noMetadataError('runtime image'); - } - - const images: IDictionary = {}; - for (const image in runtimeImages) { - const imageName: string = - runtimeImages[image]['metadata']['image_name']; - images[imageName] = runtimeImages[image]['display_name']; - } - return images; - } catch (error) { - Promise.reject(error); - } - } - - static async getComponentDef( - type = 'local', - componentID: string, - ): Promise { - return await RequestHandler.makeGetRequest( - `elyra/pipeline/components/${type}/${componentID}`, - ); - } - - /** - * Submit a request to refresh the component cache. If catalogName is given - * only refreshes the given catalog - * - * @param catalogName - */ - static async refreshComponentsCache(catalogName?: string): Promise { - return await RequestHandler.makePutRequest( - `elyra/pipeline/components/cache${catalogName ? '/' + catalogName : ''}`, - JSON.stringify({ action: 'refresh' }), - ); - } - - /** - * Creates a Dialog for passing to makeServerRequest - */ - static getWaitDialog( - title = 'Making server request...', - body = 'This may take some time', - ): Dialog { - return new Dialog({ - title: title, - body: body, - buttons: [Dialog.okButton()], - }); - } - - /** - * Submit the pipeline to be executed on an external runtime (e.g. Kbeflow Pipelines) - * - * @param pipeline - * @param runtimeName - */ - static async submitPipeline( - pipeline: any, - runtimeName: string, - ): Promise { - return RequestHandler.makePostRequest( - 'elyra/pipeline/schedule', - JSON.stringify(pipeline), - this.getWaitDialog('Packaging and submitting pipeline ...'), - ).then((response) => { - let dialogTitle; - let dialogBody; - if (response['run_url']) { - // pipeline executed remotely in a runtime of choice - dialogTitle = 'Job submission to ' + runtimeName + ' succeeded'; - dialogBody = ( -

    - {response['platform'] === 'APACHE_AIRFLOW' ? ( -

    - Apache Airflow DAG has been pushed to the{' '} - - Git repository. - -
    -

    - ) : null} - Check the status of your job at{' '} - - Run Details. - - {response['object_storage_path'] !== null ? ( -

    - The results and outputs are in the{' '} - {response['object_storage_path']} working directory in{' '} - - object storage - - . -

    - ) : null} -
    -

    - ); - } else { - // pipeline executed in-place locally - dialogTitle = 'Job execution succeeded'; - dialogBody = ( -

    Your job has been executed in-place in your local environment.

    - ); - } - - return showDialog({ - title: dialogTitle, - body: dialogBody, - buttons: [Dialog.okButton()], - }); - }); - } - - /** - * Export a pipeline to different formats (e.g. DSL, YAML, etc). These formats - * are understood by a given runtime. - * - * @param pipeline - * @param pipeline_export_format - * @param pipeline_export_path - * @param overwrite - */ - static async exportPipeline( - pipeline: any, - pipeline_export_format: string, - pipeline_export_path: string, - overwrite: boolean, - ): Promise { - console.log( - 'Exporting pipeline to [' + pipeline_export_format + '] format', - ); - - console.log('Overwriting existing file: ' + overwrite); - - const body = { - pipeline: pipeline, - export_format: pipeline_export_format, - export_path: pipeline_export_path, - overwrite: overwrite, - }; - - return RequestHandler.makePostRequest( - 'elyra/pipeline/export', - JSON.stringify(body), - this.getWaitDialog('Generating pipeline artifacts ...'), - ).then((response) => { - return showDialog({ - title: 'Pipeline export succeeded', - body:

    Exported file: {response['export_path']}

    , - buttons: [Dialog.okButton()], - }); - }); - } - - static getNodeType(filepath: string): string { - const extension: string = PathExt.extname(filepath); - const type: string = CONTENT_TYPE_MAPPER.get(extension)!; - - // TODO: throw error when file extension is not supported? - return type; - } - - /** - * Check if a given file is allowed to be added to the pipeline - * @param item - */ - static isSupportedNode(file: any): boolean { - if (PipelineService.getNodeType(file.path)) { - return true; - } else { - return false; - } - } - - static getPipelineRelativeNodePath( - pipelinePath: string, - nodePath: string, - ): string { - const relativePath: string = PathExt.relative( - PathExt.dirname(pipelinePath), - nodePath, - ); - return relativePath; - } - - static getWorkspaceRelativeNodePath( - pipelinePath: string, - nodePath: string, - ): string { - // since resolve returns an "absolute" path we need to strip off the leading '/' - const workspacePath: string = PathExt.resolve( - PathExt.dirname(pipelinePath), - nodePath, - ); - return workspacePath; - } - - static setNodePathsRelativeToWorkspace( - pipeline: any, - paletteNodes: any[], - pipelinePath: string, - ): any { - for (const node of pipeline.nodes) { - const nodeDef = paletteNodes.find((n) => { - return n.op === node.op; - }); - const parameters = - nodeDef.app_data.properties.properties.component_parameters.properties; - for (const param in parameters) { - if (parameters[param].uihints?.['ui:widget'] === 'file') { - node.app_data.component_parameters[param] = - this.getWorkspaceRelativeNodePath( - pipelinePath, - node.app_data.component_parameters[param], - ); - } else if ( - node.app_data.component_parameters[param]?.widget === 'file' - ) { - node.app_data.component_parameters[param].value = - this.getWorkspaceRelativeNodePath( - pipelinePath, - node.app_data.component_parameters[param].value, - ); - } - } - } - return pipeline; - } -} diff --git a/packages/pipeline-editor/schema/src/PipelineSubmissionDialog.tsx b/packages/pipeline-editor/schema/src/PipelineSubmissionDialog.tsx deleted file mode 100644 index 8ff85a02e..000000000 --- a/packages/pipeline-editor/schema/src/PipelineSubmissionDialog.tsx +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as React from 'react'; - -import { IParameterProps, ParameterInputForm } from './ParameterInputForm'; - -import { IRuntimeData } from './runtime-utils'; -import RuntimeConfigSelect from './RuntimeConfigSelect'; - -interface IProps extends IParameterProps { - name: string; - runtimeData: IRuntimeData; - pipelineType?: string; -} - -export const PipelineSubmissionDialog: React.FC = ({ - name, - runtimeData, - pipelineType, - parameters, -}) => { - return ( -
    - -
    - -
    -
    - - - - ); -}; diff --git a/packages/pipeline-editor/schema/src/RuntimeConfigSelect.tsx b/packages/pipeline-editor/schema/src/RuntimeConfigSelect.tsx deleted file mode 100644 index eec55127f..000000000 --- a/packages/pipeline-editor/schema/src/RuntimeConfigSelect.tsx +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import * as React from 'react'; - -import { IRuntimeData } from './runtime-utils'; - -const RUN_LOCALLY_ID = '__elyra_local__'; - -interface IProps { - runtimeData: IRuntimeData; - pipelineType?: string; - children?(platform: string): JSX.Element; -} - -const RuntimeConfigSelect: React.FC = ({ - runtimeData: { platforms, allowLocal }, - pipelineType, - children, -}) => { - const filteredPlatforms = platforms.filter((p) => p.configs.length > 0); - if (allowLocal) { - filteredPlatforms.unshift({ - id: RUN_LOCALLY_ID, - displayName: 'Run in-place locally', - configs: [], - }); - } - - // NOTE: platform is only selectable if pipelineType is undefined - const [platform, setPlatform] = React.useState( - pipelineType ?? filteredPlatforms[0]?.id, - ); - - const handleChange = (e: React.ChangeEvent): void => { - setPlatform(e.target.value); - }; - - const configs = - filteredPlatforms.find((p) => p.id === platform)?.configs ?? []; - configs.sort((a, b) => a.displayName.localeCompare(b.displayName)); - - return ( - <> - {!pipelineType && ( -
    - -
    - -
    - )} - - {/* must be present in dom at initial render */} -
    - -
    - -
    - {children?.(platform)} - - ); -}; - -export default RuntimeConfigSelect; diff --git a/packages/pipeline-editor/schema/src/RuntimeImagesWidget.tsx b/packages/pipeline-editor/schema/src/RuntimeImagesWidget.tsx deleted file mode 100644 index f77b3f2fd..000000000 --- a/packages/pipeline-editor/schema/src/RuntimeImagesWidget.tsx +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { - MetadataWidget, - IMetadataWidgetProps, - IMetadata, - MetadataDisplay, - IMetadataDisplayProps, - IMetadataDisplayState, -} from '@elyra/metadata-common'; -import { IDictionary } from '@elyra/services'; - -import React from 'react'; - -export const RUNTIME_IMAGES_SCHEMASPACE = 'runtime-images'; - -const RUNTIME_IMAGES_CLASS = 'elyra-metadata-runtime-images'; - -const getLinkFromImageName = (imageName: string): string => { - let hostname = ''; - const fqinParts = imageName.split('/'); - - if ( - fqinParts[0].includes('.') || - fqinParts[0].includes(':') || - fqinParts[0].includes('localhost') - ) { - hostname = fqinParts[0]; - imageName = fqinParts.slice(1).join('/'); - } - - if (!hostname || hostname.includes('docker.io')) { - hostname = 'hub.docker.com/r'; - } - - const imageRepo = imageName.split(':')[0]; - - return `https://${hostname}/${imageRepo}`; -}; - -/** - * A React Component for displaying the runtime images list. - */ -class RuntimeImagesDisplay extends MetadataDisplay< - IMetadataDisplayProps, - IMetadataDisplayState -> { - renderExpandableContent(metadata: IDictionary): JSX.Element { - return ( - - ); - } -} - -/** - * A widget for displaying runtime images. - */ -export class RuntimeImagesWidget extends MetadataWidget { - constructor(props: IMetadataWidgetProps) { - super(props); - } - - renderDisplay(metadata: IMetadata[]): React.ReactElement { - if (Array.isArray(metadata) && !metadata.length) { - // Empty metadata - return ( -
    -
    -
    - Click the + button to add {this.props.display_name.toLowerCase()} -
    -
    - ); - } - return ( - { - return 'runtime image'; - }} - /> - ); - } -} diff --git a/packages/pipeline-editor/schema/src/RuntimesWidget.tsx b/packages/pipeline-editor/schema/src/RuntimesWidget.tsx deleted file mode 100644 index 2842428b8..000000000 --- a/packages/pipeline-editor/schema/src/RuntimesWidget.tsx +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { - MetadataWidget, - IMetadataWidgetProps, - IMetadata, - MetadataDisplay, - IMetadataDisplayProps, - IMetadataDisplayState, -} from '@elyra/metadata-common'; -import { IDictionary, MetadataService } from '@elyra/services'; -import { RequestErrors } from '@elyra/ui-components'; -import React from 'react'; - -import { - IRuntimeType, - PipelineService, - RUNTIMES_SCHEMASPACE, -} from './PipelineService'; - -const RUNTIMES_METADATA_CLASS = 'elyra-metadata-runtimes'; - -const addTrailingSlash = (url: string): string => { - return url.endsWith('/') ? url : url + '/'; -}; - -const getGithubURLFromAPI = (apiEndpoint: string): string => { - // For Enterprise Server the api is located at /api/ - let baseURL = new URL(apiEndpoint).origin; - - // For Github.com and Github AE the api is located at api. - baseURL = baseURL.replace('api.', ''); - - return addTrailingSlash(baseURL); -}; - -export interface IRuntimesDisplayProps extends IMetadataDisplayProps { - metadata: IMetadata[]; - openMetadataEditor: (args: any) => void; - updateMetadata: () => void; - schemaspace: string; - sortMetadata: boolean; - className: string; - schemas?: IDictionary[]; - titleContext?: string; - appendToTitle?: boolean; -} - -/** - * A React Component for displaying the runtimes list. - */ -class RuntimesDisplay extends MetadataDisplay< - IRuntimesDisplayProps, - IMetadataDisplayState -> { - renderExpandableContent(metadata: IDictionary): JSX.Element { - let apiEndpoint = addTrailingSlash(metadata.metadata.api_endpoint); - let cosEndpoint = addTrailingSlash(metadata.metadata.cos_endpoint); - - let githubRepoElement = null; - let metadata_props = null; - - for (const schema of this.props.schemas ?? []) { - if (schema.name === metadata.schema_name) { - metadata_props = schema.properties.metadata.properties; - } - } - - if (metadata.schema_name === 'airflow' && metadata_props) { - const githubRepoUrl = - getGithubURLFromAPI(metadata.metadata.github_api_endpoint) + - metadata.metadata.github_repo + - '/tree/' + - metadata.metadata.github_branch + - '/'; - githubRepoElement = ( - -
    {metadata_props.github_repo.title}
    - - {githubRepoUrl} - -
    -
    -     - ); - } - if (metadata.schema_name === 'kfp') { - if (metadata.metadata.public_api_endpoint) { - // user specified a public API endpoint. use it instead of the API endpoint - apiEndpoint = addTrailingSlash(metadata.metadata.public_api_endpoint); - } - } - - if (metadata.metadata.public_cos_endpoint) { - // user specified a public COS endpoint. use it instead of the API endpoint - cosEndpoint = addTrailingSlash(metadata.metadata.public_cos_endpoint); - } - - return ( -
    -
    - {metadata_props ? metadata_props.api_endpoint.title : 'API Endpoint'} -
    - - {apiEndpoint} - -
    -
    - {githubRepoElement} -
    - {metadata_props - ? metadata_props.cos_endpoint.title - : 'Cloud Object Storage'} -
    - - {cosEndpoint} - -
    - ); - } -} - -/** - * A widget for displaying runtimes. - */ -export class RuntimesWidget extends MetadataWidget { - runtimeTypes: IRuntimeType[] = []; - - constructor(props: IMetadataWidgetProps) { - super(props); - } - - async fetchMetadata(): Promise { - return await PipelineService.getRuntimes().catch((error) => - RequestErrors.serverError(error), - ); - } - - async getSchemas(): Promise { - try { - const schemas = await MetadataService.getSchema(this.props.schemaspace); - this.runtimeTypes = await PipelineService.getRuntimeTypes(); - const sortedSchema = schemas.sort((a: any, b: any) => - a.title.localeCompare(b.title), - ); - this.schemas = sortedSchema.filter((schema: any) => { - return !!this.runtimeTypes.find( - (r) => r.id === schema.runtime_type && r.runtime_enabled, - ); - }); - if (this.schemas?.length ?? 0 > 1) { - for (const schema of this.schemas ?? []) { - this.props.app.contextMenu.addItem({ - selector: `#${this.props.schemaspace} .elyra-metadataHeader-addButton`, - command: 'elyra-metadata-editor:open', - args: { - onSave: this.updateMetadata, - schemaspace: this.props.schemaspace, - schema: schema.name, - title: schema.title, - titleContext: this.props.titleContext, - appendToTitle: this.props.appendToTitle, - } as any, - }); - } - } - this.update(); - } catch (error) { - RequestErrors.serverError(error); - } - } - - private getSchemaTitle = (metadata: IMetadata): string => { - if (this.schemas) { - for (const schema of this.schemas) { - if (schema.name === metadata.schema_name) { - return schema.title; - } - } - } - - return 'runtime configuration'; - }; - - addMetadata(schema: string, titleContext?: string): void { - this.openMetadataEditor({ - onSave: this.updateMetadata, - schemaspace: this.props.schemaspace, - schema: schema, - titleContext: titleContext, - }); - } - - renderDisplay(metadata: IMetadata[]): React.ReactElement { - if (Array.isArray(metadata) && !metadata.length) { - // Empty metadata - return ( -
    -
    -
    - Click the + button to add {this.props.display_name.toLowerCase()} -
    -
    - ); - } - - const filteredMetadata = metadata.filter((m) => { - return !!this.runtimeTypes.find((r) => m.metadata?.runtime_type === r.id); - }); - - return ( - - ); - } -} diff --git a/packages/pipeline-editor/schema/src/SubmitFileButtonExtension.tsx b/packages/pipeline-editor/schema/src/SubmitFileButtonExtension.tsx deleted file mode 100644 index 03f480738..000000000 --- a/packages/pipeline-editor/schema/src/SubmitFileButtonExtension.tsx +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { ContentParser } from '@elyra/services'; -import { RequestErrors, showFormDialog } from '@elyra/ui-components'; -import { Dialog, showDialog, ToolbarButton } from '@jupyterlab/apputils'; -import { PathExt } from '@jupyterlab/coreutils'; -import { DocumentRegistry, DocumentWidget } from '@jupyterlab/docregistry'; -import { IDisposable } from '@lumino/disposable'; - -import * as React from 'react'; - -import { FileSubmissionDialog } from './FileSubmissionDialog'; -import { formDialogWidget } from './formDialogWidget'; -import { PipelineService, RUNTIMES_SCHEMASPACE } from './PipelineService'; -import { createRuntimeData, getConfigDetails } from './runtime-utils'; -import Utils from './utils'; - -/** - * Submit file button extension - * - Attach button to editor toolbar and launch a dialog requesting - * information about the remote location to where submit the file - * for execution - */ - -export class SubmitFileButtonExtension< - T extends DocumentWidget, - U extends DocumentRegistry.IModel, -> implements DocumentRegistry.IWidgetExtension -{ - showWidget = async (document: T): Promise => { - const { context } = document; - if (context.model.dirty) { - const dialogResult = await showDialog({ - title: - 'This file contains unsaved changes. To run the file as pipeline the changes need to be saved.', - buttons: [ - Dialog.cancelButton(), - Dialog.okButton({ label: 'Save and Submit' }), - ], - }); - if (dialogResult.button.accept === false) { - return; - } - await context.save(); - } - - const env = await ContentParser.getEnvVars(context.path).catch((error) => - RequestErrors.serverError(error), - ); - const runtimeTypes: any = await PipelineService.getRuntimeTypes().catch( - (error) => RequestErrors.serverError(error), - ); - const runtimes = await PipelineService.getRuntimes() - .then((runtimeList) => { - return runtimeList.filter((runtime: any) => { - return ( - !runtime.metadata.runtime_enabled && - !!runtimeTypes.find( - (r: any) => runtime.metadata.runtime_type === r.id, - ) - ); - }); - }) - .catch((error) => RequestErrors.serverError(error)); - const images = await PipelineService.getRuntimeImages().catch((error) => - RequestErrors.serverError(error), - ); - const schema = await PipelineService.getRuntimesSchema().catch((error) => - RequestErrors.serverError(error), - ); - - const runtimeData = createRuntimeData({ schema, runtimes }); - - if (!runtimeData.platforms.find((p) => p.configs.length > 0)) { - const res = await RequestErrors.noMetadataError( - 'runtime', - `run file as pipeline.`, - ); - - if (res.button.label.includes(RUNTIMES_SCHEMASPACE)) { - // Open the runtimes widget - Utils.getLabShell(document).activateById( - `elyra-metadata:${RUNTIMES_SCHEMASPACE}`, - ); - } - return; - } - - let dependencyFileExtension = PathExt.extname(context.path); - if (dependencyFileExtension === '.ipynb') { - dependencyFileExtension = '.py'; - } - - const dialogOptions = { - title: 'Run file as pipeline', - body: formDialogWidget( - , - ), - buttons: [Dialog.cancelButton(), Dialog.okButton()], - }; - - const dialogResult = await showFormDialog(dialogOptions); - - if (dialogResult.value === null) { - // When Cancel is clicked on the dialog, just return - return; - } - - const { - runtime_config, - framework, - cpu, - gpu, - memory, - dependency_include, - dependencies, - ...envObject - } = dialogResult.value; - - const configDetails = getConfigDetails(runtimeData, runtime_config); - - // prepare file submission details - const pipeline = Utils.generateSingleFilePipeline( - context.path, - configDetails, - framework, - dependency_include ? dependencies.split(',') : undefined, - envObject, - cpu, - gpu, - memory, - ); - - PipelineService.submitPipeline( - pipeline, - configDetails?.platform.displayName ?? '', - ).catch((error) => RequestErrors.serverError(error)); - }; - - createNew(editor: T): IDisposable { - // Create the toolbar button - const submitFileButton = new ToolbarButton({ - label: 'Run as Pipeline', - onClick: (): any => this.showWidget(editor), - tooltip: 'Run file as batch', - }); - - // Add the toolbar button to the editor - editor.toolbar.insertItem(10, 'submitFile', submitFileButton); - - // The ToolbarButton class implements `IDisposable`, so the - // button *is* the extension for the purposes of this method. - return submitFileButton; - } -} diff --git a/packages/pipeline-editor/schema/src/dialogs.tsx b/packages/pipeline-editor/schema/src/dialogs.tsx deleted file mode 100644 index b6a026bb7..000000000 --- a/packages/pipeline-editor/schema/src/dialogs.tsx +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Dialog } from '@jupyterlab/apputils'; -import React from 'react'; - -export const unknownError = (message: string): any => ({ - title: 'Load pipeline failed!', - body: message, - buttons: [Dialog.okButton()], -}); - -export const elyraOutOfDate = { - title: 'Load pipeline failed!', - body: `This pipeline corresponds to a more recent version of Elyra and cannot be used until Elyra has been upgraded.`, - buttons: [Dialog.okButton()], -}; - -export const unsupportedVersion = { - title: 'Load pipeline failed!', - body: 'This pipeline has an unrecognizable version.', - buttons: [Dialog.okButton()], -}; - -export const pipelineOutOfDate = { - title: 'Migrate pipeline?', - body: ( -

    - This pipeline corresponds to an older version of Elyra and needs to be - migrated. -
    - Although the pipeline can be further edited and/or submitted after its - update, -
    - the migration will not be completed until the pipeline has been saved - within the editor. -
    -
    - Proceed with migration? -

    - ), - buttons: [Dialog.cancelButton(), Dialog.okButton()], -}; - -export const unsupportedFile = { - title: 'Unsupported File(s)', - body: 'Only supported files have been added to the pipeline.', - buttons: [Dialog.okButton()], -}; - -export const clearPipeline = { - title: 'Clear Pipeline', - body: 'Are you sure you want to clear the pipeline?', - buttons: [Dialog.cancelButton(), Dialog.okButton({ label: 'Clear' })], -}; diff --git a/packages/pipeline-editor/schema/src/formDialogWidget.ts b/packages/pipeline-editor/schema/src/formDialogWidget.ts deleted file mode 100644 index ef762f027..000000000 --- a/packages/pipeline-editor/schema/src/formDialogWidget.ts +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { ReactWidget, Dialog } from '@jupyterlab/apputils'; -import { MessageLoop } from '@lumino/messaging'; -import { Widget } from '@lumino/widgets'; - -export const formDialogWidget = ( - dialogComponent: JSX.Element, -): Dialog.IBodyWidget => { - const widget = ReactWidget.create(dialogComponent) as Dialog.IBodyWidget; - - // Immediately update the body even though it has not yet attached in - // order to trigger a render of the DOM nodes from the React element. - MessageLoop.sendMessage(widget, Widget.Msg.UpdateRequest); - - widget.getValue = (): any => { - const form = widget.node.querySelector('form'); - const formValues: { [key: string]: any } = {}; - for (const element of Object.values( - form?.elements ?? [], - ) as HTMLInputElement[]) { - switch (element.type) { - case 'checkbox': - formValues[element.name] = element.checked; - break; - default: - formValues[element.name] = element.value; - break; - } - } - return formValues; - }; - - return widget; -}; diff --git a/packages/pipeline-editor/schema/src/index.ts b/packages/pipeline-editor/schema/src/index.ts deleted file mode 100644 index 92a33f2c0..000000000 --- a/packages/pipeline-editor/schema/src/index.ts +++ /dev/null @@ -1,395 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { PIPELINE_CURRENT_VERSION } from '@elyra/pipeline-editor'; -import { RequestHandler } from '@elyra/services'; -import { - containerIcon, - pipelineIcon, - RequestErrors, - runtimesIcon, - componentCatalogIcon, -} from '@elyra/ui-components'; - -import { - JupyterFrontEnd, - JupyterFrontEndPlugin, - ILayoutRestorer, -} from '@jupyterlab/application'; -import { ICommandPalette, WidgetTracker } from '@jupyterlab/apputils'; -import { DocumentWidget } from '@jupyterlab/docregistry'; -import { IFileBrowserFactory } from '@jupyterlab/filebrowser'; -import { ILauncher } from '@jupyterlab/launcher'; -import { IMainMenu } from '@jupyterlab/mainmenu'; -import { ISettingRegistry } from '@jupyterlab/settingregistry'; -import { - addIcon, - IRankedMenu, - LabIcon, - refreshIcon, -} from '@jupyterlab/ui-components'; - -import { - COMPONENT_CATALOGS_SCHEMASPACE, - ComponentCatalogsWidget, -} from './ComponentCatalogsWidget'; -import { PipelineEditorFactory, commandIDs } from './PipelineEditorWidget'; -import { PipelineService, RUNTIMES_SCHEMASPACE } from './PipelineService'; -import { - RUNTIME_IMAGES_SCHEMASPACE, - RuntimeImagesWidget, -} from './RuntimeImagesWidget'; -import { RuntimesWidget } from './RuntimesWidget'; -import { SubmitFileButtonExtension } from './SubmitFileButtonExtension'; - -import '../style/index.css'; - -const PIPELINE_EDITOR = 'Pipeline Editor'; -const PIPELINE = 'pipeline'; -const PIPELINE_EDITOR_NAMESPACE = 'elyra-pipeline-editor-extension'; -const PLUGIN_ID = '@elyra/pipeline-editor-extension:plugin'; - -const createRemoteIcon = async ({ - name, - url, -}: { - name: string; - url: string; -}): Promise => { - const svgstr = await RequestHandler.makeServerRequest(url, { - method: 'GET', - type: 'text', - }); - return new LabIcon({ name, svgstr }); -}; - -/** - * Initialization data for the pipeline-editor-extension extension. - */ -const extension: JupyterFrontEndPlugin = { - id: PIPELINE, - autoStart: true, - requires: [ - ICommandPalette, - ILauncher, - IFileBrowserFactory, - ILayoutRestorer, - IMainMenu, - ISettingRegistry, - ], - activate: async ( - app: JupyterFrontEnd, - palette: ICommandPalette, - launcher: ILauncher, - browserFactory: IFileBrowserFactory, - restorer: ILayoutRestorer, - menu: IMainMenu, - registry: ISettingRegistry, - ) => { - console.log('Elyra - pipeline-editor extension is activated!'); - - // Fetch the initial state of the settings. - const settings = await registry - .load(PLUGIN_ID) - .catch((error) => console.log(error)); - - // Set up new widget Factory for .pipeline files - const pipelineEditorFactory = new PipelineEditorFactory({ - name: PIPELINE_EDITOR, - fileTypes: [PIPELINE], - defaultFor: [PIPELINE], - shell: app.shell, - commands: app.commands, - browserFactory: browserFactory, - serviceManager: app.serviceManager, - settings: settings, - }); - - // Add the default behavior of opening the widget for .pipeline files - app.docRegistry.addFileType( - { - name: PIPELINE, - displayName: 'Pipeline', - extensions: ['.pipeline'], - icon: pipelineIcon, - }, - ['JSON'], - ); - app.docRegistry.addWidgetFactory(pipelineEditorFactory); - - const tracker = new WidgetTracker({ - namespace: PIPELINE_EDITOR_NAMESPACE, - }); - - pipelineEditorFactory.widgetCreated.connect((sender, widget) => { - void tracker.add(widget); - - // Notify the widget tracker if restore data needs to update - widget.context.pathChanged.connect(() => { - void tracker.save(widget); - }); - }); - - // Handle state restoration - void restorer.restore(tracker, { - command: commandIDs.openDocManager, - args: (widget) => ({ - path: widget.context.path, - factory: PIPELINE_EDITOR, - }), - name: (widget) => widget.context.path, - }); - - // Add command to add file to pipeline - const addFileToPipelineCommand: string = commandIDs.addFileToPipeline; - app.commands.addCommand(addFileToPipelineCommand, { - label: 'Add File to Pipeline', - icon: addIcon, - execute: (args) => { - pipelineEditorFactory.addFileToPipelineSignal.emit(args); - }, - }); - const refreshPaletteCommand: string = commandIDs.refreshPalette; - app.commands.addCommand(refreshPaletteCommand, { - label: 'Refresh Pipeline Palette', - icon: refreshIcon, - execute: (args) => { - pipelineEditorFactory.refreshPaletteSignal.emit(args); - }, - }); - app.contextMenu.addItem({ - selector: '[data-file-type="notebook"]', - command: addFileToPipelineCommand, - }); - app.contextMenu.addItem({ - selector: '[data-file-type="python"]', - command: addFileToPipelineCommand, - }); - app.contextMenu.addItem({ - selector: '[data-file-type="r"]', - command: addFileToPipelineCommand, - }); - - // Add an application command - const openPipelineEditorCommand: string = commandIDs.openPipelineEditor; - app.commands.addCommand(openPipelineEditorCommand, { - label: (args: any) => { - if (args.isPalette) { - return `New ${PIPELINE_EDITOR}`; - } - if (args.runtimeType?.id === 'LOCAL') { - return `Generic ${PIPELINE_EDITOR}`; - } - if (args.isMenu) { - return `${args.runtimeType?.display_name} ${PIPELINE_EDITOR}`; - } - return PIPELINE_EDITOR; - }, - caption: (args: any) => { - if (args.runtimeType?.id === 'LOCAL') { - return `Generic ${PIPELINE_EDITOR}`; - } - return `${args.runtimeType?.display_name} ${PIPELINE_EDITOR}`; - }, - iconLabel: (args: any) => { - if (args.isPalette) { - return ''; - } - if (args.runtimeType?.id === 'LOCAL') { - return `Generic ${PIPELINE_EDITOR}`; - } - return `${args.runtimeType?.display_name} ${PIPELINE_EDITOR}`; - }, - icon: (args: any) => { - if (args.isPalette) { - return undefined; - } - return args.runtimeType?.icon; - }, - execute: (args: any) => { - // Creates blank file, then opens it in a new window - app.commands - .execute(commandIDs.newDocManager, { - type: 'file', - path: browserFactory.defaultBrowser.model.path, - ext: '.pipeline', - }) - .then(async (model) => { - const platformId = args.runtimeType?.id; - const runtime_type = - platformId === 'LOCAL' ? undefined : platformId; - - const pipelineJson = { - doc_type: 'pipeline', - version: '3.0', - json_schema: - 'http://api.dataplatform.ibm.com/schemas/common-pipeline/pipeline-flow/pipeline-flow-v3-schema.json', - id: 'elyra-auto-generated-pipeline', - primary_pipeline: 'primary', - pipelines: [ - { - id: 'primary', - nodes: [], - app_data: { - ui_data: { - comments: [], - }, - version: PIPELINE_CURRENT_VERSION, - runtime_type, - }, - runtime_ref: '', - }, - ], - schemas: [], - }; - const newWidget = await app.commands.execute( - commandIDs.openDocManager, - { - path: model.path, - factory: PIPELINE_EDITOR, - }, - ); - newWidget.context.ready.then(() => { - newWidget.context.model.fromJSON(pipelineJson); - app.commands.execute(commandIDs.saveDocManager, { - path: model.path, - }); - }); - }); - }, - }); - // Add the command to the palette. - palette.addItem({ - command: openPipelineEditorCommand, - args: { isPalette: true }, - category: 'Elyra', - }); - - PipelineService.getRuntimeTypes() - .then(async (types) => { - const filteredTypes = types.filter((t) => t.runtime_enabled); - const promises = filteredTypes.map(async (t) => { - return { - ...t, - icon: await createRemoteIcon({ - name: `elyra:platform:${t.id}`, - url: t.icon, - }), - }; - }); - - const resolvedTypes = await Promise.all(promises); - - // Add the command to the launcher - if (launcher) { - const fileMenuItems: IRankedMenu.IItemOptions[] = []; - - for (const t of resolvedTypes as any) { - launcher.add({ - command: openPipelineEditorCommand, - category: 'Elyra', - args: { runtimeType: t }, - rank: t.id === 'LOCAL' ? 1 : 2, - }); - - fileMenuItems.push({ - command: openPipelineEditorCommand, - args: { runtimeType: t, isMenu: true }, - rank: t.id === 'LOCAL' ? 90 : 91, - }); - } - - menu.fileMenu.newMenu.addGroup(fileMenuItems); - } - }) - .catch((error) => RequestErrors.serverError(error)); - - // SubmitNotebookButtonExtension initialization code - const notebookButtonExtension = new SubmitFileButtonExtension(); - app.docRegistry.addWidgetExtension('Notebook', notebookButtonExtension); - app.contextMenu.addItem({ - selector: '.jp-Notebook', - command: commandIDs.submitNotebook, - rank: -0.5, - }); - - // SubmitScriptButtonExtension initialization code - const scriptButtonExtension = new SubmitFileButtonExtension(); - app.docRegistry.addWidgetExtension('Python Editor', scriptButtonExtension); - app.contextMenu.addItem({ - selector: '.elyra-ScriptEditor', - command: commandIDs.submitScript, - rank: -0.5, - }); - - app.docRegistry.addWidgetExtension('R Editor', scriptButtonExtension); - app.contextMenu.addItem({ - selector: '.elyra-ScriptEditor', - command: commandIDs.submitScript, - rank: -0.5, - }); - - const runtimesWidget = new RuntimesWidget({ - app, - display_name: 'Runtimes', - schemaspace: RUNTIMES_SCHEMASPACE, - icon: runtimesIcon, - titleContext: 'runtime configuration', - appendToTitle: true, - }); - const runtimesWidgetID = `elyra-metadata:${RUNTIMES_SCHEMASPACE}`; - runtimesWidget.id = runtimesWidgetID; - runtimesWidget.title.icon = runtimesIcon; - runtimesWidget.title.caption = 'Runtimes'; - - restorer.add(runtimesWidget, runtimesWidgetID); - app.shell.add(runtimesWidget, 'left', { rank: 950 }); - - const runtimeImagesWidget = new RuntimeImagesWidget({ - app, - display_name: 'Runtime Images', - schemaspace: RUNTIME_IMAGES_SCHEMASPACE, - icon: containerIcon, - titleContext: 'runtime image', - }); - const runtimeImagesWidgetID = `elyra-metadata:${RUNTIME_IMAGES_SCHEMASPACE}`; - runtimeImagesWidget.id = runtimeImagesWidgetID; - runtimeImagesWidget.title.icon = containerIcon; - runtimeImagesWidget.title.caption = 'Runtime Images'; - - restorer.add(runtimeImagesWidget, runtimeImagesWidgetID); - app.shell.add(runtimeImagesWidget, 'left', { rank: 951 }); - - const componentCatalogWidget = new ComponentCatalogsWidget({ - app, - display_name: 'Component Catalogs', // TODO: This info should come from the server for all schemaspaces - schemaspace: COMPONENT_CATALOGS_SCHEMASPACE, - icon: componentCatalogIcon, - titleContext: 'component catalog', - refreshCallback: (): void => { - app.commands.execute(commandIDs.refreshPalette); - }, - }); - const componentCatalogWidgetID = `elyra-metadata:${COMPONENT_CATALOGS_SCHEMASPACE}`; - componentCatalogWidget.id = componentCatalogWidgetID; - componentCatalogWidget.title.icon = componentCatalogIcon; - componentCatalogWidget.title.caption = 'Component Catalogs'; - - restorer.add(componentCatalogWidget, componentCatalogWidgetID); - app.shell.add(componentCatalogWidget, 'left', { rank: 961 }); - }, -}; -export default extension; diff --git a/packages/pipeline-editor/schema/src/pipeline-hooks.ts b/packages/pipeline-editor/schema/src/pipeline-hooks.ts deleted file mode 100644 index 163d577fa..000000000 --- a/packages/pipeline-editor/schema/src/pipeline-hooks.ts +++ /dev/null @@ -1,297 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { MetadataService, RequestHandler } from '@elyra/services'; -import { URLExt } from '@jupyterlab/coreutils'; -import { ServerConnection } from '@jupyterlab/services'; -import produce from 'immer'; -import useSWR from 'swr'; - -import { PipelineService } from './PipelineService'; - -export const GENERIC_CATEGORY_ID = 'Elyra'; - -interface IReturn { - data?: T | undefined; - error?: any; - mutate?: any; -} - -type IRuntimeImagesResponse = IRuntimeImage[]; - -interface IRuntimeImage { - name: string; - display_name: string; - metadata: { - image_name: string; - }; -} - -const metadataFetcher = async (key: string): Promise => { - return await MetadataService.getMetadata(key); -}; - -export const useRuntimeImages = (): IReturn => { - const { data, error } = useSWR( - 'runtime-images', - metadataFetcher, - ); - - data?.sort((a, b) => 0 - (a.name > b.name ? -1 : 1)); - - return { data, error }; -}; - -const schemaFetcher = async (key: string): Promise => { - return await MetadataService.getSchema(key); -}; - -// TODO: type this -export const useRuntimesSchema = (): IReturn => { - const { data, error } = useSWR('runtimes', schemaFetcher); - - return { data, error }; -}; - -interface IRuntimeComponentsResponse { - version: string; - categories: IRuntimeComponent[]; - properties: IComponentPropertiesResponse; - parameters: IComponentPropertiesResponse; -} - -export interface IRuntimeComponent { - label: string; - image: string; - id: string; - description: string; - runtime?: string; - node_types: { - op: string; - id: string; - label: string; - image: string; - runtime_type?: string; - type: 'execution_node'; - inputs: { app_data: any }[]; - outputs: { app_data: any }[]; - app_data: any; - }[]; - extensions?: string[]; -} - -interface IComponentPropertiesResponse { - current_parameters: { [key: string]: any }; - parameters: { id: string }[]; - uihints: { - parameter_info: { - parameter_ref: string; - control: 'custom'; - custom_control_id: string; - label: { default: string }; - description: { - default: string; - placement: 'on_panel'; - }; - data: any; - }[]; - }; - group_info: { - group_info: { - id: string; - parameter_refs: string[]; - }[]; - }[]; -} - -/** - * Sort palette in place. Takes a list of categories each containing a list of - * components. - * - Categories: alphabetically by "label" (exception: "generic" always first) - * - Components: alphabetically by "op" (where is component label stored?) - */ -export const sortPalette = (palette: { - categories: IRuntimeComponent[]; -}): void => { - palette.categories.sort((a, b) => { - if (a.id === GENERIC_CATEGORY_ID) { - return -1; - } - if (b.id === GENERIC_CATEGORY_ID) { - return 1; - } - return a.label.localeCompare(b.label, undefined, { numeric: true }); - }); - - for (const components of palette.categories) { - components.node_types.sort((a, b) => - a.label.localeCompare(b.label, undefined, { - numeric: true, - }), - ); - } -}; - -// TODO: This should be enabled through `extensions` -const NodeIcons: Map = new Map([ - ['execute-notebook-node', 'static/elyra/notebook.svg'], - ['execute-python-node', 'static/elyra/python.svg'], - ['execute-r-node', 'static/elyra/r-logo.svg'], -]); - -// TODO: We should decouple components and properties to support lazy loading. -// TODO: type this -export const componentFetcher = async (type: string): Promise => { - const palettePromise = - RequestHandler.makeGetRequest( - `elyra/pipeline/components/${type}`, - ); - - const pipelinePropertiesPromise = - RequestHandler.makeGetRequest( - `elyra/pipeline/${type}/properties`, - ); - - const pipelineParametersPromise = - RequestHandler.makeGetRequest( - `elyra/pipeline/${type}/parameters`, - ); - - const typesPromise = PipelineService.getRuntimeTypes(); - - const [palette, pipelineProperties, pipelineParameters, types] = - await Promise.all([ - palettePromise, - pipelinePropertiesPromise, - pipelineParametersPromise, - typesPromise, - ]); - - palette.properties = pipelineProperties; - palette.parameters = pipelineParameters; - - // Gather list of component IDs to fetch properties for. - const componentList: string[] = []; - for (const category of palette.categories) { - for (const node of category.node_types) { - componentList.push(node.id); - } - } - - const propertiesPromises = componentList.map(async (componentID) => { - const res = - await RequestHandler.makeGetRequest( - `elyra/pipeline/components/${type}/${componentID}/properties`, - ); - return { - id: componentID, - properties: res, - }; - }); - - // load all of the properties in parallel instead of serially - const properties = await Promise.all(propertiesPromises); - - // inject properties - for (const category of palette.categories) { - // Use the runtime_type from the first node of the category to determine category - // icon. - // TODO: Ideally, this would be included in the category. - const category_runtime_type = - category.node_types?.[0]?.runtime_type ?? 'LOCAL'; - - const type = types.find((t: any) => t.id === category_runtime_type); - const baseUrl = ServerConnection.makeSettings().baseUrl; - const defaultIcon = URLExt.parse( - URLExt.join(baseUrl, type?.icon || ''), - ).pathname; - - category.image = defaultIcon; - - for (const node of category.node_types) { - // update icon - const genericNodeIcon = NodeIcons.get(node.op); - const nodeIcon = genericNodeIcon - ? URLExt.parse(URLExt.join(baseUrl, genericNodeIcon)).pathname - : defaultIcon; - - // Not sure which is needed... - node.image = nodeIcon; - node.app_data.image = nodeIcon; - node.app_data.ui_data.image = nodeIcon; - - const prop = properties.find((p) => p.id === node.id); - node.app_data.properties = prop?.properties; - } - } - - sortPalette(palette); - - return palette; -}; - -const updateRuntimeImages = ( - properties: any, - runtimeImages: IRuntimeImage[] | undefined, -): void => { - const runtimeImageProperties = - properties?.properties?.component_parameters?.properties?.runtime_image ?? - properties?.properties?.pipeline_defaults?.properties?.runtime_image; - - const imageNames = (runtimeImages ?? []).map((i) => i.metadata.image_name); - - const displayNames: { [key: string]: string } = {}; - - (runtimeImages ?? []).forEach((i: IRuntimeImage) => { - displayNames[i.metadata.image_name] = i.display_name; - }); - - if (runtimeImageProperties) { - runtimeImageProperties.enumNames = (runtimeImages ?? []).map( - (i) => i.display_name, - ); - runtimeImageProperties.enum = imageNames; - } -}; - -export const usePalette = (type = 'local'): IReturn => { - const { data: runtimeImages, error: runtimeError } = useRuntimeImages(); - - const { - data: palette, - error: paletteError, - mutate: mutate, - } = useSWR(type, componentFetcher); - - let updatedPalette; - if (palette !== undefined) { - updatedPalette = produce(palette, (draft: any) => { - for (const category of draft.categories) { - for (const node of category.node_types) { - // update runtime images - updateRuntimeImages(node.app_data.properties, runtimeImages); - } - } - updateRuntimeImages(draft.properties, runtimeImages); - }); - } - - return { - data: updatedPalette, - error: runtimeError ?? paletteError, - mutate: mutate, - }; -}; diff --git a/packages/pipeline-editor/schema/src/runtime-utils.ts b/packages/pipeline-editor/schema/src/runtime-utils.ts deleted file mode 100644 index c4cf0bffc..000000000 --- a/packages/pipeline-editor/schema/src/runtime-utils.ts +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { IRuntime, ISchema } from './PipelineService'; - -export interface IRuntimeData { - platforms: { - id: string; - displayName: string; - configs: { - id: string; - displayName: string; - processor: { - id: string; - }; - }[]; - }[]; - allowLocal: boolean; -} - -export const createRuntimeData = ({ - runtimes, - schema, - allowLocal, -}: { - runtimes: IRuntime[]; - schema: ISchema[]; - allowLocal?: boolean; -}): IRuntimeData => { - const platforms: IRuntimeData['platforms'] = []; - for (const s of schema) { - const found = platforms.find((p) => p.id === s.runtime_type); - if (found) { - continue; - } - platforms.push({ - id: s.runtime_type, - displayName: s.title, - configs: runtimes - .filter((r) => r.metadata.runtime_type === s.runtime_type) - .map((r) => ({ - id: r.name, - displayName: r.display_name, - processor: { - id: r.schema_name, - }, - })), - }); - } - return { platforms, allowLocal: !!allowLocal }; -}; - -export interface IConfigDetails { - id: string; - displayName: string; - platform: { - id: string; - displayName: string; - }; - processor: { - id: string; - }; -} - -export const getConfigDetails = ( - runtimeData: IRuntimeData, - configId: string, -): IConfigDetails | undefined => { - for (const platform of runtimeData.platforms) { - for (const config of platform.configs) { - if (config.id === configId) { - return { - id: config.id, - displayName: config.displayName, - platform: { - id: platform.id, - displayName: platform.displayName, - }, - processor: { - id: config.processor.id, - }, - }; - } - } - } - return undefined; -}; diff --git a/packages/pipeline-editor/schema/src/test/pipeline-hooks.spec.ts b/packages/pipeline-editor/schema/src/test/pipeline-hooks.spec.ts deleted file mode 100644 index 9319babbe..000000000 --- a/packages/pipeline-editor/schema/src/test/pipeline-hooks.spec.ts +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { - IRuntimeComponent, - GENERIC_CATEGORY_ID, - sortPalette, -} from '../pipeline-hooks'; - -const GENERIC_CATEGORY = { - label: 'ZZZZ should not matter', - image: 'string', - id: GENERIC_CATEGORY_ID, - description: 'string', - node_types: [], -}; - -type Category = IRuntimeComponent; -type Component = Category['node_types'][0]; - -const createMockCategory = ( - name: string, - components: Component[] = [], -): Category => { - return { - label: name, - image: 'string', - id: 'string', - description: 'string', - node_types: components, - }; -}; - -const createMockComponent = (name: string): Component => { - return { - op: 'string', - label: name, - id: 'string', - image: 'string', - type: 'execution_node', - inputs: [], - outputs: [], - app_data: {}, - }; -}; - -describe('sortPalette', () => { - it('should function with no categories', () => { - const palette = { categories: [] }; - const expected = { categories: [] }; - sortPalette(palette); - expect(palette).toStrictEqual(expected); - }); - - it('should sort categories alphabetically', () => { - const palette = { - categories: [ - createMockCategory('a'), - createMockCategory('c'), - createMockCategory('b'), - ], - }; - const expected = { - categories: [ - createMockCategory('a'), - createMockCategory('b'), - createMockCategory('c'), - ], - }; - sortPalette(palette); - expect(palette).toStrictEqual(expected); - }); - - it('should sort components alphabetically', () => { - const palette = { - categories: [ - createMockCategory('a', [ - createMockComponent('c'), - createMockComponent('a'), - createMockComponent('b'), - ]), - ], - }; - const expected = { - categories: [ - createMockCategory('a', [ - createMockComponent('a'), - createMockComponent('b'), - createMockComponent('c'), - ]), - ], - }; - sortPalette(palette); - expect(palette).toStrictEqual(expected); - }); - - it('should sort categories numerically', () => { - const palette = { - categories: [ - createMockCategory('c200'), - createMockCategory('c2'), - createMockCategory('c20'), - createMockCategory('c100'), - createMockCategory('c1'), - createMockCategory('c10'), - ], - }; - const expected = { - categories: [ - createMockCategory('c1'), - createMockCategory('c2'), - createMockCategory('c10'), - createMockCategory('c20'), - createMockCategory('c100'), - createMockCategory('c200'), - ], - }; - sortPalette(palette); - expect(palette).toStrictEqual(expected); - }); - - it('should sort components numerically', () => { - const palette = { - categories: [ - createMockCategory('a', [ - createMockComponent('c200'), - createMockComponent('c2'), - createMockComponent('c20'), - createMockComponent('c100'), - createMockComponent('c1'), - createMockComponent('c10'), - ]), - ], - }; - const expected = { - categories: [ - createMockCategory('a', [ - createMockComponent('c1'), - createMockComponent('c2'), - createMockComponent('c10'), - createMockComponent('c20'), - createMockComponent('c100'), - createMockComponent('c200'), - ]), - ], - }; - sortPalette(palette); - expect(palette).toStrictEqual(expected); - }); - - it('should sort generic category first', () => { - const palette = { - categories: [ - GENERIC_CATEGORY, - createMockCategory('a'), - createMockCategory('c'), - createMockCategory('b'), - ], - }; - const expected = { - categories: [ - GENERIC_CATEGORY, - createMockCategory('a'), - createMockCategory('b'), - createMockCategory('c'), - ], - }; - sortPalette(palette); - expect(palette).toStrictEqual(expected); - }); -}); diff --git a/packages/pipeline-editor/schema/src/test/pipeline-service.spec.ts b/packages/pipeline-editor/schema/src/test/pipeline-service.spec.ts deleted file mode 100644 index a9dd6baf6..000000000 --- a/packages/pipeline-editor/schema/src/test/pipeline-service.spec.ts +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { JupyterServer } from '@jupyterlab/testutils'; - -import { PipelineService } from '../PipelineService'; -jest.setTimeout(3 * 60 * 1000); - -const server = new JupyterServer(); - -beforeAll(async () => { - await server.start(); -}); - -afterAll(async () => { - await server.shutdown(); -}); - -describe('@elyra/pipeline-editor', () => { - describe('PipelineService', () => { - describe('#getRuntimeTypes', () => { - it('should get runtimes ordered by id', async () => { - const runtime_types = await PipelineService.getRuntimeTypes(); - const expected_runtime_ids = [ - 'APACHE_AIRFLOW', - 'KUBEFLOW_PIPELINES', - 'LOCAL', - ]; - expect( - runtime_types.map((runtime_type) => runtime_type.id), - ).toStrictEqual(expected_runtime_ids); - }); - }); - }); -}); diff --git a/packages/pipeline-editor/schema/src/theme.tsx b/packages/pipeline-editor/schema/src/theme.tsx deleted file mode 100644 index 57c974548..000000000 --- a/packages/pipeline-editor/schema/src/theme.tsx +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { trashIcon } from '@elyra/ui-components'; -import { - closeIcon, - caretDownEmptyIcon, - editIcon, - folderIcon, - LabIcon, - paletteIcon, -} from '@jupyterlab/ui-components'; -import * as React from 'react'; - -const SvgIcon: React.FC = ({ children }): any => { - return ( - - {children} - - ); -}; - -const theme: any = { - palette: { - focus: 'var(--jp-brand-color0)', - border: 'var(--jp-border-color0)', - divider: 'var(--jp-border-color0)', - hover: 'var(--jp-border-color1)', - active: 'rgba(255, 255, 255, 0.18)', - inputBorder: 'var(--jp-border-color0)', - primary: { - main: 'var(--jp-inverse-layout-color4)', - hover: 'transparent', - contrastText: 'var(--jp-layout-color1)', - }, - secondary: { - main: 'transparent', - contrastText: 'var(--jp-content-font-color1)', - }, - error: { - main: 'var(--jp-error-color0)', - contrastText: 'var(--jp-icon-contrast-color3)', - }, - errorMessage: { - main: 'var(--jp-error-color1)', - contrastText: 'rgba(255, 255, 255, 0.9)', - errorBorder: 'var(--jp-error-color0)', - }, - icon: { - primary: 'var(--jp-ui-font-color0)', - secondary: 'var(--jp-ui-font-color0)', - }, - text: { - primary: 'var(--jp-content-font-color0)', - secondary: 'var(--jp-ui-font-color1)', - bold: 'var(--jp-inverse-layout-color2)', - inactive: 'var(--jp-inverse-layout-color4)', - disabled: 'var(--jp-content-font-color3)', - link: 'var(--jp-content-link-color)', - error: 'var(--jp-error-color0)', - icon: 'var(--jp-inverse-layout-color2)', - }, - background: { - default: 'var(--jp-layout-color1)', - secondary: 'var(--jp-border-color2)', - input: 'transparent', - }, - highlight: { - border: 'transparent', - hover: 'var(--jp-content-font-color0)', - focus: 'transparent', - }, - }, - shape: { - borderRadius: '4px', - }, - typography: { - fontFamily: 'var(--jp-ui-font-family)', - fontWeight: 'normal', - fontSize: 'var(--jp-content-font-size1)', - }, - overrides: { - deleteIcon: LabIcon.resolveReact({ icon: trashIcon }), - editIcon: LabIcon.resolveReact({ icon: editIcon }), - folderIcon: LabIcon.resolveReact({ icon: folderIcon }), - closeIcon: LabIcon.resolveReact({ icon: closeIcon }), - propertiesIcon: ( - - - - ), - paletteIcon: LabIcon.resolveReact({ icon: paletteIcon }), - checkIcon: ( - - - - ), - chevronDownIcon: LabIcon.resolveReact({ icon: caretDownEmptyIcon }), - }, -}; - -export { theme }; diff --git a/packages/pipeline-editor/schema/src/utils.ts b/packages/pipeline-editor/schema/src/utils.ts deleted file mode 100644 index 24e438601..000000000 --- a/packages/pipeline-editor/schema/src/utils.ts +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright 2018-2023 Elyra Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { PIPELINE_CURRENT_VERSION } from '@elyra/pipeline-editor'; - -import { LabShell } from '@jupyterlab/application'; -import { PathExt } from '@jupyterlab/coreutils'; - -import uuid4 from 'uuid/v4'; - -import { IConfigDetails } from './runtime-utils'; - -/** - * A utilities class for static functions. - */ -export default class Utils { - /** - * Utility to create a one node pipeline to submit a single file as a pipeline - */ - static generateSingleFilePipeline( - filename: string, - configDetails: IConfigDetails | undefined, - runtimeImage: string, - dependencies: string[] | undefined, - envObject: { [key: string]: string }, - cpu?: number, - gpu?: number, - memory?: number, - ): any { - const generated_uuid = uuid4(); - - const artifactName = PathExt.basename(filename, PathExt.extname(filename)); - - const envVars = Object.entries(envObject).map( - ([key, val]) => `${key}=${val}`, - ); - - return { - doc_type: 'pipeline', - version: '3.0', - json_schema: - 'http://api.dataplatform.ibm.com/schemas/common-pipeline/pipeline-flow/pipeline-flow-v3-schema.json', - id: generated_uuid, - primary_pipeline: generated_uuid, - pipelines: [ - { - id: generated_uuid, - nodes: [ - { - id: generated_uuid, - type: 'execution_node', - op: 'execute-notebook-node', - app_data: { - component_parameters: { - filename, - runtime_image: runtimeImage, - outputs: [], - env_vars: envVars, - dependencies, - cpu, - gpu, - memory, - include_subdirectories: false, - }, - ui_data: { - label: PathExt.basename(filename), - }, - }, - }, - ], - app_data: { - name: artifactName, - runtime_config: configDetails?.id, - version: PIPELINE_CURRENT_VERSION, - source: PathExt.basename(filename), - properties: { - name: 'generic', - }, - ui_data: { - comments: [], - }, - }, - }, - ], - schemas: [], - }; - } - - /** - * Break an array into an array of "chunks", each "chunk" having "n" elements. - * The final "chuck" may have less than "n" elements. - * Example: - * chunkArray(['a', 'b', 'c', 'd', 'e', 'f', 'g'], 4) - * -> [['a', 'b', 'c', 'd'], ['e', 'f', 'g']] - */ - static chunkArray(arr: T[], n: number): T[][] { - return Array.from(Array(Math.ceil(arr.length / n)), (_, i) => - arr.slice(i * n, i * n + n), - ); - } - - /** - * From a given widget, find the application shell and return it - */ - static getLabShell = (widget: any): LabShell => { - while (widget !== null && !(widget instanceof LabShell)) { - widget = widget.parent; - } - - return widget; - }; -} diff --git a/packages/pipeline-editor/setup.py b/packages/pipeline-editor/setup.py new file mode 100644 index 000000000..aefdf20db --- /dev/null +++ b/packages/pipeline-editor/setup.py @@ -0,0 +1 @@ +__import__("setuptools").setup() diff --git a/packages/pipeline-editor/src/ComponentCatalogsWidget.tsx b/packages/pipeline-editor/src/ComponentCatalogsWidget.tsx index c934203d4..47e503c7e 100644 --- a/packages/pipeline-editor/src/ComponentCatalogsWidget.tsx +++ b/packages/pipeline-editor/src/ComponentCatalogsWidget.tsx @@ -21,7 +21,7 @@ import { MetadataDisplay, IMetadataDisplayProps, //IMetadataDisplayState, - IMetadataActionButton, + IMetadataActionButton } from '@elyra/metadata-common'; import { IDictionary, MetadataService } from '@elyra/services'; import { RequestErrors } from '@elyra/ui-components'; @@ -60,12 +60,12 @@ class ComponentCatalogsDisplay extends MetadataDisplay { .catch((error) => console.error( 'An error occurred while refreshing components from catalog:', - error, - ), + error + ) ); - }, + } }, - ...super.actionButtons(metadata), + ...super.actionButtons(metadata) ]; } @@ -141,14 +141,14 @@ export class ComponentCatalogsWidget extends MetadataWidget { const schemas = await MetadataService.getSchema(this.props.schemaspace); this.runtimeTypes = await PipelineService.getRuntimeTypes(); const sortedSchema = schemas.sort((a: any, b: any) => - a.title.localeCompare(b.title), + a.title.localeCompare(b.title) ); this.schemas = sortedSchema.filter((schema: any) => { return !!this.runtimeTypes.find( (r) => schema.properties?.metadata?.properties?.runtime_type?.enum?.includes( - r.id, - ) && r.runtime_enabled, + r.id + ) && r.runtime_enabled ); }); if (this.schemas?.length ?? 0 > 1) { @@ -162,8 +162,8 @@ export class ComponentCatalogsWidget extends MetadataWidget { schema: schema.name, title: schema.title, titleContext: this.props.titleContext, - appendToTitle: this.props.appendToTitle, - } as any, + appendToTitle: this.props.appendToTitle + } as any }); } } diff --git a/packages/pipeline-editor/src/EmptyPipelineContent.tsx b/packages/pipeline-editor/src/EmptyPipelineContent.tsx index 3af93d71d..a8c24cd19 100644 --- a/packages/pipeline-editor/src/EmptyPipelineContent.tsx +++ b/packages/pipeline-editor/src/EmptyPipelineContent.tsx @@ -28,7 +28,7 @@ export interface IEmptyGenericPipelineProps { } export const EmptyGenericPipeline: React.FC = ({ - onOpenSettings, + onOpenSettings }) => { return (
    diff --git a/packages/pipeline-editor/src/FileSubmissionDialog.tsx b/packages/pipeline-editor/src/FileSubmissionDialog.tsx index 978af1112..7405e9889 100644 --- a/packages/pipeline-editor/src/FileSubmissionDialog.tsx +++ b/packages/pipeline-editor/src/FileSubmissionDialog.tsx @@ -63,7 +63,7 @@ export const FileSubmissionDialog: React.FC = ({ env, images, dependencyFileExtension, - runtimeData, + runtimeData }) => { const [includeDependency, setIncludeDependency] = React.useState(true); diff --git a/packages/pipeline-editor/src/ParameterInputForm.tsx b/packages/pipeline-editor/src/ParameterInputForm.tsx index 45306a1f4..0f800f871 100644 --- a/packages/pipeline-editor/src/ParameterInputForm.tsx +++ b/packages/pipeline-editor/src/ParameterInputForm.tsx @@ -31,14 +31,14 @@ export interface IParameterProps { } export const ParameterInputForm: React.FC = ({ - parameters, + parameters }) => { return parameters && parameters.length > 0 ? (