From b789b85ef94846d337f622b9d56f094b775e2585 Mon Sep 17 00:00:00 2001 From: Junwei Dai Date: Fri, 6 Sep 2024 08:10:06 -0700 Subject: [PATCH 01/11] Add some 4xx error test Signed-off-by: Junwei Dai --- spec/namespaces/flow_framework.yaml | 79 +++++++++++++------ spec/schemas/flow_framework.common.yaml | 19 +++++ spec/schemas/flow_framework.errors.yaml | 2 +- tests/default/flow_framework/workflow.yaml | 90 +++++++++++++++++++++- 4 files changed, 165 insertions(+), 25 deletions(-) diff --git a/spec/namespaces/flow_framework.yaml b/spec/namespaces/flow_framework.yaml index eff3c923a..1e73e693a 100644 --- a/spec/namespaces/flow_framework.yaml +++ b/spec/namespaces/flow_framework.yaml @@ -15,6 +15,7 @@ paths: parameters: - $ref: '#/components/parameters/flow_framework.create::query.provision' - $ref: '#/components/parameters/flow_framework.create::query.reprovision' + - $ref: '#/components/parameters/flow_framework.create::query.update_fields' - $ref: '#/components/parameters/flow_framework.create::query.use_case' - $ref: '#/components/parameters/flow_framework.create::query.validation' requestBody: @@ -50,7 +51,11 @@ paths: url: https://opensearch.org/docs/latest/automating-configurations/api/create-workflow/ parameters: - $ref: '#/components/parameters/flow_framework.update::path.workflow_id' + - $ref: '#/components/parameters/flow_framework.update::query.provision' + - $ref: '#/components/parameters/flow_framework.update::query.reprovision' - $ref: '#/components/parameters/flow_framework.update::query.update_fields' + - $ref: '#/components/parameters/flow_framework.update::query.use_case' + - $ref: '#/components/parameters/flow_framework.update::query.validation' requestBody: $ref: '#/components/requestBodies/flow_framework.update' responses: @@ -59,6 +64,8 @@ paths: description: It returns the workflow_id '404': $ref: '#/components/responses/flow_framework.update@404' + '400': + $ref: '#/components/responses/flow_framework.update@400' delete: operationId: flow_framework.delete.0 x-operation-group: flow_framework.delete @@ -76,8 +83,6 @@ paths: $ref: '#/components/responses/flow_framework.delete@400' '403': $ref: '#/components/responses/flow_framework.delete@403' - '404': - $ref: '#/components/responses/flow_framework.delete@404' components: parameters: flow_framework.get::path.workflow_id: @@ -85,33 +90,41 @@ components: in: path required: true schema: - type: string + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' flow_framework.update::path.workflow_id: name: workflow_id in: path required: true schema: - type: string + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' flow_framework.delete::path.workflow_id: name: workflow_id in: path required: true schema: - type: string + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' flow_framework.create::query.provision: in: query name: provision schema: - description: Provision the workflow as part of the request - type: boolean - default: false + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/Provision' + flow_framework.update::query.provision: + in: query + name: provision + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/Provision' flow_framework.create::query.reprovision: x-version-added: '2.17' in: query name: reprovision schema: - type: boolean - default: false + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/Reprovision' + flow_framework.update::query.reprovision: + x-version-added: '2.17' + in: query + name: reprovision + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/Reprovision' flow_framework.create::query.use_case: description: To use a workflow template, specify it in the use_case query parameter when creating a workflow. externalDocs: @@ -119,21 +132,35 @@ components: in: query name: use_case schema: - type: string + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/UseCase' + flow_framework.update::query.use_case: + description: To use a workflow template, specify it in the use_case query parameter when creating a workflow. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/workflow-templates/#supported-workflow-templates + in: query + name: use_case + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/UseCase' flow_framework.create::query.validation: in: query name: validation schema: - description: Validate the workflow. Valid values are all (validate the template) and none (do not validate the template). Default is all. - type: string - default: all + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/Validation' + flow_framework.update::query.validation: + in: query + name: validation + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/Validation' flow_framework.update::query.update_fields: in: query name: update_fields schema: - description: Update only the fields included in the request body. - type: boolean - default: false + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/UpdateFields' + flow_framework.create::query.update_fields: + in: query + name: update_fields + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/UpdateFields' flow_framework.delete::query.clear_status: in: query name: clear_status @@ -173,7 +200,18 @@ components: content: application/json: oneOf: - - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/BadRequestError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/InvalidParameterError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/ConflictError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/MissingParameterError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/ParameterConflictError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/MaxWorkflowsLimitError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkflowSaveError' + flow_framework.update@400: + description: Bad Request - Multiple possible reasons + content: + application/json: + oneOf: + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/InvalidParameterError' - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/ConflictError' - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/MissingParameterError' - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/ParameterConflictError' @@ -213,11 +251,6 @@ components: content: application/json: $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' - flow_framework.delete@404: - description: Template Not Found Error - content: - application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' flow_framework.get@404: description: Template Not Found Error content: diff --git a/spec/schemas/flow_framework.common.yaml b/spec/schemas/flow_framework.common.yaml index 6ff3cda19..79cb98c12 100644 --- a/spec/schemas/flow_framework.common.yaml +++ b/spec/schemas/flow_framework.common.yaml @@ -6,6 +6,25 @@ info: paths: { } components: schemas: + WorkflowID: + type: string + Provision: + description: Provision the workflow as part of the request. + type: boolean + default: false + Reprovision: + type: boolean + default: false + UseCase: + type: string + Validation: + description: Validate the workflow. Valid values are all (validate the template) and none (do not validate the template). Default is all. + type: string + default: all + UpdateFields: + description: Update only the fields included in the request body. + type: boolean + default: false FlowFrameworkCreate: type: object properties: diff --git a/spec/schemas/flow_framework.errors.yaml b/spec/schemas/flow_framework.errors.yaml index 0224b1cba..d0d4b1844 100644 --- a/spec/schemas/flow_framework.errors.yaml +++ b/spec/schemas/flow_framework.errors.yaml @@ -16,7 +16,7 @@ components: example: This API is disabled. To enable it, set [flow_framework.enabled] to true. status: type: integer - BadRequestError: + InvalidParameterError: content: application/json: type: object diff --git a/tests/default/flow_framework/workflow.yaml b/tests/default/flow_framework/workflow.yaml index 53da81c66..66390bb75 100644 --- a/tests/default/flow_framework/workflow.yaml +++ b/tests/default/flow_framework/workflow.yaml @@ -19,6 +19,84 @@ chapters: status: 201 output: test_workflow_id: payload.workflow_id + - synopsis: Create workflow With Provision and updateFields set true. + id: create_flow_framework + path: /_plugins/_flow_framework/workflow + method: POST + parameters: + provision: true + update_fields: true + request: + payload: + name: test_create_work_flow + response: + status: 400 + - synopsis: Update workflow With Provision and updateFields set true. + id: create_flow_framework + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: PUT + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + provision: true + update_fields: true + request: + payload: + name: test_create_work_flow + response: + status: 400 + - synopsis: Update workflow With Reprovision and updateFields set true. + version: '>= 2.17' + id: create_flow_framework + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: PUT + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + reprovision: true + update_fields: true + request: + payload: + name: test_create_work_flow + response: + status: 400 + - synopsis: Create workflow With Reprovision set true. + version: '>= 2.17' + id: create_flow_framework + path: /_plugins/_flow_framework/workflow + method: PUT + parameters: + reprovision: true + request: + payload: + name: test_create_work_flow + response: + status: 400 + - synopsis: Create workflow With Reprovision set true and UseCase Not null. + version: '>= 2.17' + id: create_flow_framework + path: /_plugins/_flow_framework/workflow + method: PUT + parameters: + reprovision: true + use_case: test_use_case + request: + payload: + name: test_create_work_flow + response: + status: 400 + - synopsis: Update workflow With Reprovision set true and UseCase Not null. + version: '>= 2.17' + id: create_flow_framework + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: POST + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + reprovision: true + use_case: test_use_case + request: + payload: + name: test_create_work_flow + response: + status: 400 - synopsis: Get workflow. id: get_flow_framework path: /_plugins/_flow_framework/workflow/{workflow_id} @@ -56,4 +134,14 @@ chapters: parameters: workflow_id: ${create_flow_framework.test_workflow_id} response: - status: 200 \ No newline at end of file + status: 200 + - synopsis: Delete workflow With Wrong ID. + id: delete_flow_framework + path: /_plugins/_flow_framework/workflow/{workflow_id} + method: DELETE + parameters: + workflow_id: test + response: + status: 200 + payload: + result: not_found \ No newline at end of file From 4a4503a80a9156ec6ac140b3be2455f17fb32d07 Mon Sep 17 00:00:00 2001 From: Junwei Dai Date: Mon, 9 Sep 2024 13:21:27 -0700 Subject: [PATCH 02/11] 1. Adding API spec for GET /workflow/_steps 2. Adding Test for this API Signed-off-by: Junwei Dai --- spec/namespaces/flow_framework.yaml | 51 +++++++++++++++++++++- spec/schemas/flow_framework.common.yaml | 22 +++++++++- spec/schemas/flow_framework.errors.yaml | 10 +++++ tests/default/flow_framework/workflow.yaml | 24 +++++----- 4 files changed, 94 insertions(+), 13 deletions(-) diff --git a/spec/namespaces/flow_framework.yaml b/spec/namespaces/flow_framework.yaml index 1e73e693a..9db1806d9 100644 --- a/spec/namespaces/flow_framework.yaml +++ b/spec/namespaces/flow_framework.yaml @@ -83,6 +83,35 @@ paths: $ref: '#/components/responses/flow_framework.delete@400' '403': $ref: '#/components/responses/flow_framework.delete@403' + /_plugins/_flow_framework/workflow/_steps: + get: + operationId: flow_framework.get_steps.0 + x-operation-group: flow_framework.get_steps + x-version-added: '2.12' + description: Get a list of workflow steps. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/api/get-workflow-steps/ + parameters: + - $ref: '#/components/parameters/flow_framework.get_steps::query.workflow_step' + responses: + '200': + $ref: '#/components/responses/flow_framework.get_steps@200' + '403': + $ref: '#/components/responses/flow_framework.get_steps@403' + '400': + $ref: '#/components/responses/flow_framework.get_steps@400' +# /_plugins/_flow_framework/workflow/_search: +# post: { } +# get: { } +# /_plugins/_flow_framework/workflow/state/_search: +# post: { } +# get: { } +# /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision: +# post: { } +# /_plugins/_flow_framework/workflow/{workflow_id}/_provision: +# post: { } +# /_plugins/_flow_framework/workflow/{workflow_id}/_status: +# get: {} components: parameters: flow_framework.get::path.workflow_id: @@ -168,6 +197,13 @@ components: description: Delete the workflow state (without deprovisioning resources) after deleting the template. OpenSearch deletes the workflow state only if the provisioning status is not IN_PROGRESS. Default is false. type: boolean default: false + flow_framework.get_steps::query.workflow_step: + in: query + name: workflow_step + required: false + schema: + description: The name of the step to retrieve. + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowStepName' requestBodies: flow_framework.create: content: @@ -255,4 +291,17 @@ components: description: Template Not Found Error content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' \ No newline at end of file + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' + flow_framework.get_steps@200: + content: + application/json: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowStep' + flow_framework.get_steps@403: + description: Flow Framework API Disabled Error + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + flow_framework.get_steps@400: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkflowStepsRetrieveError' diff --git a/spec/schemas/flow_framework.common.yaml b/spec/schemas/flow_framework.common.yaml index 79cb98c12..a86a37d92 100644 --- a/spec/schemas/flow_framework.common.yaml +++ b/spec/schemas/flow_framework.common.yaml @@ -8,6 +8,8 @@ components: schemas: WorkflowID: type: string + WorkflowStepName: + type: string Provision: description: Provision the workflow as part of the request. type: boolean @@ -143,4 +145,22 @@ components: type: string user_requested_tenant: type: string - nullable: true \ No newline at end of file + nullable: true + WorkflowStep: + type: object + properties: + inputs: + type: array + items: + type: string + description: The list of required inputs for the workflow step + outputs: + type: array + items: + type: string + description: The list of outputs generated by the workflow step + required_plugins: + type: array + items: + type: string + description: The list of plugins required for the workflow step \ No newline at end of file diff --git a/spec/schemas/flow_framework.errors.yaml b/spec/schemas/flow_framework.errors.yaml index d0d4b1844..39bd27057 100644 --- a/spec/schemas/flow_framework.errors.yaml +++ b/spec/schemas/flow_framework.errors.yaml @@ -94,5 +94,15 @@ components: message: type: string example: Failed to retrieve template (12345) from global context. + code: + type: integer + WorkflowStepsRetrieveError: + content: + application/json: + type: object + properties: + message: + type: string + example: Failed to retrieve workflow step json. code: type: integer \ No newline at end of file diff --git a/tests/default/flow_framework/workflow.yaml b/tests/default/flow_framework/workflow.yaml index 66390bb75..405ffbe03 100644 --- a/tests/default/flow_framework/workflow.yaml +++ b/tests/default/flow_framework/workflow.yaml @@ -20,7 +20,6 @@ chapters: output: test_workflow_id: payload.workflow_id - synopsis: Create workflow With Provision and updateFields set true. - id: create_flow_framework path: /_plugins/_flow_framework/workflow method: POST parameters: @@ -32,7 +31,6 @@ chapters: response: status: 400 - synopsis: Update workflow With Provision and updateFields set true. - id: create_flow_framework path: /_plugins/_flow_framework/workflow/{workflow_id} method: PUT parameters: @@ -46,7 +44,6 @@ chapters: status: 400 - synopsis: Update workflow With Reprovision and updateFields set true. version: '>= 2.17' - id: create_flow_framework path: /_plugins/_flow_framework/workflow/{workflow_id} method: PUT parameters: @@ -60,7 +57,6 @@ chapters: status: 400 - synopsis: Create workflow With Reprovision set true. version: '>= 2.17' - id: create_flow_framework path: /_plugins/_flow_framework/workflow method: PUT parameters: @@ -72,7 +68,6 @@ chapters: status: 400 - synopsis: Create workflow With Reprovision set true and UseCase Not null. version: '>= 2.17' - id: create_flow_framework path: /_plugins/_flow_framework/workflow method: PUT parameters: @@ -98,7 +93,6 @@ chapters: response: status: 400 - synopsis: Get workflow. - id: get_flow_framework path: /_plugins/_flow_framework/workflow/{workflow_id} method: GET parameters: @@ -106,7 +100,6 @@ chapters: response: status: 200 - synopsis: Update workflow. - id: get_flow_framework path: /_plugins/_flow_framework/workflow/{workflow_id} method: PUT parameters: @@ -117,7 +110,6 @@ chapters: response: status: 201 - synopsis: Update workflow with a non-existent workflow ID. - id: get_flow_framework path: /_plugins/_flow_framework/workflow/{workflow_id} method: PUT parameters: @@ -127,8 +119,19 @@ chapters: name: test_update_work_flow response: status: 404 + - synopsis: Get workflow steps. + path: /_plugins/_flow_framework/workflow/_steps + method: GET + response: + status: 200 + - synopsis: Get workflow steps with bad request. + path: /_plugins/_flow_framework/workflow/_steps + method: GET + parameters: + workflow_step: wrong_query + response: + status: 400 - synopsis: Delete workflow. - id: delete_flow_framework path: /_plugins/_flow_framework/workflow/{workflow_id} method: DELETE parameters: @@ -136,7 +139,6 @@ chapters: response: status: 200 - synopsis: Delete workflow With Wrong ID. - id: delete_flow_framework path: /_plugins/_flow_framework/workflow/{workflow_id} method: DELETE parameters: @@ -144,4 +146,4 @@ chapters: response: status: 200 payload: - result: not_found \ No newline at end of file + result: not_found From b957fdb23f2177920c368a55b0e5cb1182b1cc36 Mon Sep 17 00:00:00 2001 From: Junwei Dai Date: Mon, 9 Sep 2024 14:55:27 -0700 Subject: [PATCH 03/11] 1. Adding API spec for GET /workflow/_status 2. Adding Test for this API Signed-off-by: Junwei Dai --- spec/namespaces/flow_framework.yaml | 48 ++++++++++++++++- spec/schemas/flow_framework.common.yaml | 61 +++++++++++++++++++++- tests/default/flow_framework/workflow.yaml | 14 +++++ 3 files changed, 120 insertions(+), 3 deletions(-) diff --git a/spec/namespaces/flow_framework.yaml b/spec/namespaces/flow_framework.yaml index 9db1806d9..8dd44cbea 100644 --- a/spec/namespaces/flow_framework.yaml +++ b/spec/namespaces/flow_framework.yaml @@ -100,6 +100,24 @@ paths: $ref: '#/components/responses/flow_framework.get_steps@403' '400': $ref: '#/components/responses/flow_framework.get_steps@400' + /_plugins/_flow_framework/workflow/{workflow_id}/_status: + get: + operationId: flow_framework.get_status.0 + x-operation-group: flow_framework.get_status + x-version-added: '2.12' + description: Get the provisioning deployment status until it is complete. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/api/get-workflow-status/ + parameters: + - $ref: '#/components/parameters/flow_framework.get_status::path.workflow_id' + - $ref: '#/components/parameters/flow_framework.get_status::query.all' + responses: + '200': + $ref: '#/components/responses/flow_framework.get_status@200' + '403': + $ref: '#/components/responses/flow_framework.get_status@403' + '404': + $ref: '#/components/responses/flow_framework.get_status@404' # /_plugins/_flow_framework/workflow/_search: # post: { } # get: { } @@ -110,8 +128,7 @@ paths: # post: { } # /_plugins/_flow_framework/workflow/{workflow_id}/_provision: # post: { } -# /_plugins/_flow_framework/workflow/{workflow_id}/_status: -# get: {} + components: parameters: flow_framework.get::path.workflow_id: @@ -120,6 +137,12 @@ components: required: true schema: $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' + flow_framework.get_status::path.workflow_id: + name: workflow_id + in: path + required: true + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' flow_framework.update::path.workflow_id: name: workflow_id in: path @@ -162,6 +185,12 @@ components: name: use_case schema: $ref: '../schemas/flow_framework.common.yaml#/components/schemas/UseCase' + flow_framework.get_status::query.all: + description: The all parameter specifies whether the response should return all fields. + in: query + name: all + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/All' flow_framework.update::query.use_case: description: To use a workflow template, specify it in the use_case query parameter when creating a workflow. externalDocs: @@ -305,3 +334,18 @@ components: content: application/json: $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkflowStepsRetrieveError' + flow_framework.get_status@403: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + flow_framework.get_status@404: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' + flow_framework.get_status@200: + content: + application/json: + oneOf: + - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkFlowStatusFullResponse' + - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkFlowStatusDefaultResponse' + diff --git a/spec/schemas/flow_framework.common.yaml b/spec/schemas/flow_framework.common.yaml index a86a37d92..8a0fe16d2 100644 --- a/spec/schemas/flow_framework.common.yaml +++ b/spec/schemas/flow_framework.common.yaml @@ -27,6 +27,10 @@ components: description: Update only the fields included in the request body. type: boolean default: false + All: + description: The all parameter specifies whether the response should return all fields. + type: boolean + default: false FlowFrameworkCreate: type: object properties: @@ -163,4 +167,59 @@ components: type: array items: type: string - description: The list of plugins required for the workflow step \ No newline at end of file + description: The list of plugins required for the workflow step + WorkFlowStatusDefaultResponse: + type: object + properties: + workflow_id: + type: string + description: The ID of the workflow + error: + type: string + description: Any error state associated with the workflow + nullable: true + state: + type: string + description: The current state of the workflow + resources_created: + type: array + items: + type: string + description: A list of resources created by the workflow + WorkFlowStatusFullResponse: + type: object + properties: + workflow_id: + type: string + description: The ID of the workflow + error: + type: string + description: Any error state associated with the workflow + nullable: true + state: + type: string + description: The current state of the workflow + resources_created: + type: array + items: + type: string + description: A list of resources created by the workflow + provisioning_progress: + type: string + description: The progress of the provisioning process + nullable: true + provision_start_time: + type: string + format: date-time + description: The start time of the provisioning process + provision_end_time: + type: string + format: date-time + description: The end time of the provisioning process + user: + $ref: '#/components/schemas/user' + user_outputs: + type: array + items: + type: string + description: Outputs generated by the user \ No newline at end of file diff --git a/tests/default/flow_framework/workflow.yaml b/tests/default/flow_framework/workflow.yaml index 405ffbe03..9515e03d8 100644 --- a/tests/default/flow_framework/workflow.yaml +++ b/tests/default/flow_framework/workflow.yaml @@ -131,6 +131,20 @@ chapters: workflow_step: wrong_query response: status: 400 + - synopsis: Get workflow status. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_status + method: GET + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + response: + status: 200 + - synopsis: Get workflow status with wrong workflow id. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_status + method: GET + parameters: + workflow_id: test + response: + status: 404 - synopsis: Delete workflow. path: /_plugins/_flow_framework/workflow/{workflow_id} method: DELETE From 0ea367e5e9a103c0be169056a1f0fd47f5077741 Mon Sep 17 00:00:00 2001 From: Junwei Dai Date: Mon, 9 Sep 2024 15:42:42 -0700 Subject: [PATCH 04/11] 1. Adding API spec for POST /workflow/_deprovision 2. Adding Test for this API Signed-off-by: Junwei Dai --- .cspell | 3 +- spec/namespaces/flow_framework.yaml | 57 +++++++++++++++++++--- spec/schemas/flow_framework.common.yaml | 3 ++ spec/schemas/flow_framework.errors.yaml | 20 +++++++- tests/default/flow_framework/workflow.yaml | 16 +++++- 5 files changed, 90 insertions(+), 9 deletions(-) diff --git a/.cspell b/.cspell index 518d11f61..df8953cd6 100644 --- a/.cspell +++ b/.cspell @@ -192,4 +192,5 @@ vectory whoamiprotected wordnet Yrtsd -reprovision \ No newline at end of file +reprovision +deprovision \ No newline at end of file diff --git a/spec/namespaces/flow_framework.yaml b/spec/namespaces/flow_framework.yaml index 8dd44cbea..51d8d774a 100644 --- a/spec/namespaces/flow_framework.yaml +++ b/spec/namespaces/flow_framework.yaml @@ -118,17 +118,34 @@ paths: $ref: '#/components/responses/flow_framework.get_status@403' '404': $ref: '#/components/responses/flow_framework.get_status@404' + /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision: + post: + operationId: flow_framework.deprovision.0 + x-operation-group: flow_framework.deprovision + x-version-added: '2.12' + description: Deprovision workflow's resources when you no longer need it. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/api/deprovision-workflow/ + parameters: + - $ref: '#/components/parameters/flow_framework.deprovision::path.workflow_id' + - $ref: '#/components/parameters/flow_framework.deprovision::query.allow_delete' + responses: + '200': + $ref: '#/components/responses/flow_framework.deprovision@200' + '202': + $ref: '#/components/responses/flow_framework.deprovision@202' + '403': + $ref: '#/components/responses/flow_framework.deprovision@403' + '404': + $ref: '#/components/responses/flow_framework.deprovision@404' +# /_plugins/_flow_framework/workflow/{workflow_id}/_provision: +# post: { } # /_plugins/_flow_framework/workflow/_search: # post: { } # get: { } # /_plugins/_flow_framework/workflow/state/_search: # post: { } # get: { } -# /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision: -# post: { } -# /_plugins/_flow_framework/workflow/{workflow_id}/_provision: -# post: { } - components: parameters: flow_framework.get::path.workflow_id: @@ -143,6 +160,12 @@ components: required: true schema: $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' + flow_framework.deprovision::path.workflow_id: + name: workflow_id + in: path + required: true + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' flow_framework.update::path.workflow_id: name: workflow_id in: path @@ -171,6 +194,11 @@ components: name: reprovision schema: $ref: '../schemas/flow_framework.common.yaml#/components/schemas/Reprovision' + flow_framework.deprovision::query.allow_delete: + in: query + name: allow_delete + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/AllowDelete' flow_framework.update::query.reprovision: x-version-added: '2.17' in: query @@ -348,4 +376,21 @@ components: oneOf: - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkFlowStatusFullResponse' - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkFlowStatusDefaultResponse' - + flow_framework.deprovision@200: + content: + application/json: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' + flow_framework.deprovision@202: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/DeprovisioningError' + flow_framework.deprovision@403: + content: + application/json: + oneOf: + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/DeprovisioningForbiddenError' + flow_framework.deprovision@404: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' \ No newline at end of file diff --git a/spec/schemas/flow_framework.common.yaml b/spec/schemas/flow_framework.common.yaml index 8a0fe16d2..1d90f80bf 100644 --- a/spec/schemas/flow_framework.common.yaml +++ b/spec/schemas/flow_framework.common.yaml @@ -6,6 +6,9 @@ info: paths: { } components: schemas: + AllowDelete: + description: A comma-separated list of resource IDs to be deprovisioned. Required if deleting resources of type index_name or pipeline_id. + type: string WorkflowID: type: string WorkflowStepName: diff --git a/spec/schemas/flow_framework.errors.yaml b/spec/schemas/flow_framework.errors.yaml index 39bd27057..3f937e2a4 100644 --- a/spec/schemas/flow_framework.errors.yaml +++ b/spec/schemas/flow_framework.errors.yaml @@ -105,4 +105,22 @@ components: type: string example: Failed to retrieve workflow step json. code: - type: integer \ No newline at end of file + type: integer + DeprovisioningError: + type: object + properties: + error: + type: string + description: Describes the deprovisioning error and identifies resources that were not deprovisioned + example: Failed to deprovision some resources [connector_id Lw7PX4wBfVtHp98y06wV]. + required: + - error + DeprovisioningForbiddenError: + type: object + properties: + error: + type: string + description: Describes the resources that require the allow_delete parameter for deprovisioning + example: 'These resources require the allow_delete parameter to deprovision: [index_name my-index].' + required: + - error \ No newline at end of file diff --git a/tests/default/flow_framework/workflow.yaml b/tests/default/flow_framework/workflow.yaml index 9515e03d8..2dc4d1f6d 100644 --- a/tests/default/flow_framework/workflow.yaml +++ b/tests/default/flow_framework/workflow.yaml @@ -138,13 +138,27 @@ chapters: workflow_id: ${create_flow_framework.test_workflow_id} response: status: 200 - - synopsis: Get workflow status with wrong workflow id. + - synopsis: Get workflow status with wrong workflow ID. path: /_plugins/_flow_framework/workflow/{workflow_id}/_status method: GET parameters: workflow_id: test response: status: 404 + - synopsis: Deprovision workflow. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision + method: POST + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + response: + status: 200 + - synopsis: Deprovision workflow with wrong workflow ID. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision + method: POST + parameters: + workflow_id: test + response: + status: 404 - synopsis: Delete workflow. path: /_plugins/_flow_framework/workflow/{workflow_id} method: DELETE From 21327942b962060911521053ec0c91c44be5a0ad Mon Sep 17 00:00:00 2001 From: Junwei Dai Date: Mon, 9 Sep 2024 16:50:32 -0700 Subject: [PATCH 05/11] 1. Adding API spec for POST /workflow/_provision 2. Adding Test for this API Signed-off-by: Junwei Dai --- spec/namespaces/flow_framework.yaml | 58 +++++++++++++++++++++++-- spec/schemas/flow_framework.common.yaml | 7 ++- spec/schemas/flow_framework.errors.yaml | 29 +++++++++++++ 3 files changed, 90 insertions(+), 4 deletions(-) diff --git a/spec/namespaces/flow_framework.yaml b/spec/namespaces/flow_framework.yaml index 51d8d774a..916e11329 100644 --- a/spec/namespaces/flow_framework.yaml +++ b/spec/namespaces/flow_framework.yaml @@ -138,8 +138,31 @@ paths: $ref: '#/components/responses/flow_framework.deprovision@403' '404': $ref: '#/components/responses/flow_framework.deprovision@404' -# /_plugins/_flow_framework/workflow/{workflow_id}/_provision: -# post: { } + /_plugins/_flow_framework/workflow/{workflow_id}/_provision: + post: + operationId: flow_framework.provision.0 + x-operation-group: flow_framework.provision + x-version-added: '2.12' + description: Provisioning a workflow. This API is also executed when the Create or Update Workflow API is called with the provision parameter set to true. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/api/provision-workflow/ + parameters: + - $ref: '#/components/parameters/flow_framework.provision::path.workflow_id' + requestBody: + description: |- + Pass dynamic substitution expressions as key-value pairs, either as query parameters or in the request body. + For example, if you specified a credential field in the template as `openai_key: '$'`, + you can pass `openai_key` as a query parameter or in the request body. + $ref: '#/components/requestBodies/flow_framework.provision' + responses: + '200': + $ref: '#/components/responses/flow_framework.provision@200' + '403': + $ref: '#/components/responses/flow_framework.provision@403' + '400': + $ref: '#/components/responses/flow_framework.provision@400' + + # /_plugins/_flow_framework/workflow/_search: # post: { } # get: { } @@ -166,6 +189,12 @@ components: required: true schema: $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' + flow_framework.provision::path.workflow_id: + name: workflow_id + in: path + required: true + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' flow_framework.update::path.workflow_id: name: workflow_id in: path @@ -272,6 +301,12 @@ components: application/json: schema: $ref: '../schemas/flow_framework.common.yaml#/components/schemas/FlowFrameworkUpdate' + flow_framework.provision: + required: false + content: + application/json: + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/UserProvidedSubstitutionExpressions' responses: flow_framework.create@201: content: @@ -380,6 +415,10 @@ components: content: application/json: $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' + flow_framework.provision@200: + content: + application/json: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowID' flow_framework.deprovision@202: content: application/json: @@ -393,4 +432,17 @@ components: flow_framework.deprovision@404: content: application/json: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' \ No newline at end of file + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/TemplateNotFoundError' + flow_framework.provision@403: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + flow_framework.provision@400: + content: + application/json: + oneOf: + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/WorkFlowIdNullError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/RequestBodyParsingFailedError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/InvalidRequestBodyFieldError' + - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/DuplicateKeyError' + diff --git a/spec/schemas/flow_framework.common.yaml b/spec/schemas/flow_framework.common.yaml index 1d90f80bf..19ac77659 100644 --- a/spec/schemas/flow_framework.common.yaml +++ b/spec/schemas/flow_framework.common.yaml @@ -225,4 +225,9 @@ components: type: array items: type: string - description: Outputs generated by the user \ No newline at end of file + description: Outputs generated by the user + UserProvidedSubstitutionExpressions: + type: object + additionalProperties: + type: string + description: The value for the user-provided key. Keys represent substitution expressions. \ No newline at end of file diff --git a/spec/schemas/flow_framework.errors.yaml b/spec/schemas/flow_framework.errors.yaml index 3f937e2a4..a044a885b 100644 --- a/spec/schemas/flow_framework.errors.yaml +++ b/spec/schemas/flow_framework.errors.yaml @@ -66,6 +66,35 @@ components: example: Workflow ID can not be null status: type: integer + DuplicateKeyError: + type: object + properties: + error: + type: string + description: Error message when a duplicate key is found in the request. + status: + type: integer + description: HTTP status code for the error. + example: 400 + InvalidRequestBodyFieldError: + type: object + properties: + error: + type: string + description: Error message when a request body field is not a string. + status: + type: integer + description: HTTP status code for the error. + RequestBodyParsingFailedError: + type: object + properties: + error: + type: string + description: Error message when request body parsing fails. + status: + type: integer + description: HTTP status code for the error. + example: 400 WorkflowSaveError: content: application/json: From 06e14c0be082054fc3019e84079f0ec4cebdd0ad Mon Sep 17 00:00:00 2001 From: Junwei Dai Date: Mon, 9 Sep 2024 18:26:03 -0700 Subject: [PATCH 06/11] 1. Adding API spec for POST /workflow/_search 2. Adding Test for this API Signed-off-by: Junwei Dai --- spec/namespaces/flow_framework.yaml | 65 ++++++++++++++++-- spec/schemas/flow_framework.common.yaml | 76 +++++++++++++++++++++- spec/schemas/flow_framework.errors.yaml | 25 ++++++- tests/default/flow_framework/workflow.yaml | 10 +++ 4 files changed, 166 insertions(+), 10 deletions(-) diff --git a/spec/namespaces/flow_framework.yaml b/spec/namespaces/flow_framework.yaml index 916e11329..64dc905a4 100644 --- a/spec/namespaces/flow_framework.yaml +++ b/spec/namespaces/flow_framework.yaml @@ -161,11 +161,43 @@ paths: $ref: '#/components/responses/flow_framework.provision@403' '400': $ref: '#/components/responses/flow_framework.provision@400' - - -# /_plugins/_flow_framework/workflow/_search: -# post: { } -# get: { } + /_plugins/_flow_framework/workflow/_search: + get: + operationId: flow_framework.search.1 + x-operation-group: flow_framework.search + x-version-added: '2.12' + description: Search for workflows by using a query matching a field. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/api/provision-workflow/ + requestBody: + $ref: '#/components/requestBodies/flow_framework.search' + responses: + '200': + $ref: '#/components/responses/flow_framework.search@200' + '403': + $ref: '#/components/responses/flow_framework.search@403' + '408': + $ref: '#/components/responses/flow_framework.search@408' + '400': + $ref: '#/components/responses/flow_framework.search@400' + post: + operationId: flow_framework.search.0 + x-operation-group: flow_framework.search + x-version-added: '2.12' + description: Search for workflows by using a query matching a field. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/api/provision-workflow/ + requestBody: + $ref: '#/components/requestBodies/flow_framework.search' + responses: + '200': + $ref: '#/components/responses/flow_framework.search@200' + '403': + $ref: '#/components/responses/flow_framework.search@403' + '408': + $ref: '#/components/responses/flow_framework.search@408' + '400': + $ref: '#/components/responses/flow_framework.search@400' # /_plugins/_flow_framework/workflow/state/_search: # post: { } # get: { } @@ -307,6 +339,12 @@ components: application/json: schema: $ref: '../schemas/flow_framework.common.yaml#/components/schemas/UserProvidedSubstitutionExpressions' + flow_framework.search: + required: true + content: + application/json: + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/SearchWorkflowRequest' responses: flow_framework.create@201: content: @@ -445,4 +483,19 @@ components: - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/RequestBodyParsingFailedError' - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/InvalidRequestBodyFieldError' - $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/DuplicateKeyError' - + flow_framework.search@200: + content: + application/json: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowSearchResponse' + flow_framework.search@403: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + flow_framework.search@408: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/RequestTimeoutError' + flow_framework.search@400: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/BadRequestError' \ No newline at end of file diff --git a/spec/schemas/flow_framework.common.yaml b/spec/schemas/flow_framework.common.yaml index 19ac77659..5582797a5 100644 --- a/spec/schemas/flow_framework.common.yaml +++ b/spec/schemas/flow_framework.common.yaml @@ -83,7 +83,7 @@ components: result: type: string description: The result of the deletion operation. - enum: [deleted, not_found] + enum: [ deleted, not_found ] _shards: $ref: '#/components/schemas/shards' _seq_no: @@ -133,6 +133,47 @@ components: failed: type: integer description: The number of shards where the deletion failed. + query: + type: object + description: The search query to match workflows. Use `match_all` to retrieve all workflows, or `match` to search by specific fields like `use_case`. + properties: + match: + type: object + match_all: + type: string + total: + type: object + properties: + value: + type: integer + description: Total number of matching documents. + relation: + type: string + description: The relation of the total hits. + itemsObject: + type: object + properties: + _index: + type: string + description: The index name where the document resides. + _id: + type: string + description: The document ID. + _version: + type: integer + description: The version of the document. + _seq_no: + type: integer + description: The sequence number of the document. + _primary_term: + type: integer + description: The primary term of the document. + _score: + type: number + format: float + description: The score of the search hit. + _source: + $ref: '#/components/schemas/FlowFrameworkGetResponse' user: type: object properties: @@ -153,6 +194,19 @@ components: user_requested_tenant: type: string nullable: true + hits: + type: object + properties: + total: + $ref: '#/components/schemas/total' + max_score: + type: number + format: float + description: The maximum score of the search hits. + hits: + type: array + items: + $ref: '#/components/schemas/itemsObject' WorkflowStep: type: object properties: @@ -230,4 +284,22 @@ components: type: object additionalProperties: type: string - description: The value for the user-provided key. Keys represent substitution expressions. \ No newline at end of file + description: The value for the user-provided key. Keys represent substitution expressions. + SearchWorkflowRequest: + type: object + properties: + query: + $ref: '#/components/schemas/query' + WorkflowSearchResponse: + type: object + properties: + took: + type: integer + description: Time in milliseconds that the request took to complete. + timed_out: + type: boolean + description: Indicates if the request timed out. + _shards: + $ref: '#/components/schemas/shards' + hits: + $ref: '#/components/schemas/hits' diff --git a/spec/schemas/flow_framework.errors.yaml b/spec/schemas/flow_framework.errors.yaml index a044a885b..6f46f2435 100644 --- a/spec/schemas/flow_framework.errors.yaml +++ b/spec/schemas/flow_framework.errors.yaml @@ -150,6 +150,27 @@ components: error: type: string description: Describes the resources that require the allow_delete parameter for deprovisioning - example: 'These resources require the allow_delete parameter to deprovision: [index_name my-index].' + example: These resources require the allow_delete parameter to deprovision [index_name my-index]. required: - - error \ No newline at end of file + - error + RequestTimeoutError: + type: object + properties: + error: + type: string + description: Error message when the request times out. + status: + type: integer + description: HTTP status code for the error. + example: 408 + BadRequestError: + type: object + properties: + error: + type: string + description: Error message when the request body or parameters are invalid. + example: Invalid request body or query parameters. + status: + type: integer + description: HTTP status code for the error. + example: 400 \ No newline at end of file diff --git a/tests/default/flow_framework/workflow.yaml b/tests/default/flow_framework/workflow.yaml index 2dc4d1f6d..415a0eada 100644 --- a/tests/default/flow_framework/workflow.yaml +++ b/tests/default/flow_framework/workflow.yaml @@ -152,6 +152,16 @@ chapters: workflow_id: ${create_flow_framework.test_workflow_id} response: status: 200 + - synopsis: Search workflow. + path: /_plugins/_flow_framework/workflow/_search + method: POST + request: + payload: + query: + match: + name: test + response: + status: 200 - synopsis: Deprovision workflow with wrong workflow ID. path: /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision method: POST From 312b4714023950e3f1d5f5a240ac7aa1ff43f1eb Mon Sep 17 00:00:00 2001 From: Junwei Dai Date: Mon, 9 Sep 2024 20:09:39 -0700 Subject: [PATCH 07/11] 1. Adding API spec for POST/GET /workflow/state/_search 2. Adding Test for this API Signed-off-by: Junwei Dai --- spec/namespaces/flow_framework.yaml | 64 ++++++++++++++++++++-- spec/schemas/flow_framework.common.yaml | 4 +- tests/default/flow_framework/workflow.yaml | 18 ++++-- 3 files changed, 76 insertions(+), 10 deletions(-) diff --git a/spec/namespaces/flow_framework.yaml b/spec/namespaces/flow_framework.yaml index 64dc905a4..0d8c48797 100644 --- a/spec/namespaces/flow_framework.yaml +++ b/spec/namespaces/flow_framework.yaml @@ -198,9 +198,43 @@ paths: $ref: '#/components/responses/flow_framework.search@408' '400': $ref: '#/components/responses/flow_framework.search@400' -# /_plugins/_flow_framework/workflow/state/_search: -# post: { } -# get: { } + /_plugins/_flow_framework/workflow/state/_search: + get: + operationId: flow_framework.search_state.1 + x-operation-group: flow_framework.search_state + x-version-added: '2.12' + description: Search for workflows by using a query matching a field. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/api/search-workflow-state/ + requestBody: + $ref: '#/components/requestBodies/flow_framework.search_state' + responses: + '200': + $ref: '#/components/responses/flow_framework.search_state@200' + '403': + $ref: '#/components/responses/flow_framework.search_state@403' + '408': + $ref: '#/components/responses/flow_framework.search_state@408' + '400': + $ref: '#/components/responses/flow_framework.search_state@400' + post: + operationId: flow_framework.search_state.0 + x-operation-group: flow_framework.search_state + x-version-added: '2.12' + description: Search for workflows by using a query matching a field. + externalDocs: + url: https://opensearch.org/docs/latest/automating-configurations/api/search-workflow-state/ + requestBody: + $ref: '#/components/requestBodies/flow_framework.search_state' + responses: + '200': + $ref: '#/components/responses/flow_framework.search_state@200' + '403': + $ref: '#/components/responses/flow_framework.search_state@403' + '408': + $ref: '#/components/responses/flow_framework.search_state@408' + '400': + $ref: '#/components/responses/flow_framework.search_state@400' components: parameters: flow_framework.get::path.workflow_id: @@ -345,6 +379,12 @@ components: application/json: schema: $ref: '../schemas/flow_framework.common.yaml#/components/schemas/SearchWorkflowRequest' + flow_framework.search_state: + required: true + content: + application/json: + schema: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/SearchWorkflowRequest' responses: flow_framework.create@201: content: @@ -486,7 +526,7 @@ components: flow_framework.search@200: content: application/json: - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowSearchResponse' + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowSearchResponse' flow_framework.search@403: content: application/json: @@ -496,6 +536,22 @@ components: application/json: $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/RequestTimeoutError' flow_framework.search@400: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/BadRequestError' + flow_framework.search_state@200: + content: + application/json: + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowSearchResponse' + flow_framework.search_state@403: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/FlowFrameworkAPIDisabledError' + flow_framework.search_state@408: + content: + application/json: + $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/RequestTimeoutError' + flow_framework.search_state@400: content: application/json: $ref: '../schemas/flow_framework.errors.yaml#/components/schemas/BadRequestError' \ No newline at end of file diff --git a/spec/schemas/flow_framework.common.yaml b/spec/schemas/flow_framework.common.yaml index 5582797a5..bad3bdfc8 100644 --- a/spec/schemas/flow_framework.common.yaml +++ b/spec/schemas/flow_framework.common.yaml @@ -83,7 +83,7 @@ components: result: type: string description: The result of the deletion operation. - enum: [ deleted, not_found ] + enum: [deleted, not_found] _shards: $ref: '#/components/schemas/shards' _seq_no: @@ -302,4 +302,4 @@ components: _shards: $ref: '#/components/schemas/shards' hits: - $ref: '#/components/schemas/hits' + $ref: '#/components/schemas/hits' \ No newline at end of file diff --git a/tests/default/flow_framework/workflow.yaml b/tests/default/flow_framework/workflow.yaml index 415a0eada..7d17448c5 100644 --- a/tests/default/flow_framework/workflow.yaml +++ b/tests/default/flow_framework/workflow.yaml @@ -156,12 +156,22 @@ chapters: path: /_plugins/_flow_framework/workflow/_search method: POST request: - payload: - query: - match: + payload: + query: + match: name: test response: status: 200 + - synopsis: Search workflow state. + path: /_plugins/_flow_framework/workflow/_search + method: POST + request: + payload: + query: + match: + state: test + response: + status: 200 - synopsis: Deprovision workflow with wrong workflow ID. path: /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision method: POST @@ -184,4 +194,4 @@ chapters: response: status: 200 payload: - result: not_found + result: not_found \ No newline at end of file From 14fe053acc69d4848d5020a60b1436975acdc0d5 Mon Sep 17 00:00:00 2001 From: Junwei Dai Date: Fri, 6 Sep 2024 08:10:06 -0700 Subject: [PATCH 08/11] add change log Signed-off-by: Junwei Dai --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ced75260..bc7835ba2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,6 +92,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added request and response schemas for `/_cluster/decommission/awareness` ([#524](https://github.com/opensearch-project/opensearch-api-specification/pull/524)) - Added `Bytes` component of type number ([#552](https://github.com/opensearch-project/opensearch-api-specification/pull/552)) - Added `xy_shape` query ([#531](https://github.com/opensearch-project/opensearch-api-specification/pull/531)) +- Added `/_plugins/_flow_framework/_search` ,`state/_search`,`_provision`,`_deprovision`,`_steps`,`_status`([#508](https://github.com/opensearch-project/opensearch-api-specification/issues/508)) ### Changed From 62fcc2e0f2a78e4217123446537211dd1a7e55ed Mon Sep 17 00:00:00 2001 From: Junwei Dai <59641585+junweid62@users.noreply.github.com> Date: Tue, 10 Sep 2024 14:57:29 -0700 Subject: [PATCH 09/11] Update tests/default/flow_framework/workflow.yaml Co-authored-by: Daniel (dB.) Doubrovkine Signed-off-by: Junwei Dai <59641585+junweid62@users.noreply.github.com> --- tests/default/flow_framework/workflow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/default/flow_framework/workflow.yaml b/tests/default/flow_framework/workflow.yaml index 7d17448c5..4ec58fc49 100644 --- a/tests/default/flow_framework/workflow.yaml +++ b/tests/default/flow_framework/workflow.yaml @@ -30,7 +30,7 @@ chapters: name: test_create_work_flow response: status: 400 - - synopsis: Update workflow With Provision and updateFields set true. + - synopsis: Update workflow With provision and update_fields set. path: /_plugins/_flow_framework/workflow/{workflow_id} method: PUT parameters: From 8543168edf7327e4c79958287c0d26b908138c83 Mon Sep 17 00:00:00 2001 From: Junwei Dai Date: Tue, 10 Sep 2024 14:26:11 -0700 Subject: [PATCH 10/11] change searchStateResponse spec. Signed-off-by: Junwei Dai --- CHANGELOG.md | 2 +- spec/namespaces/flow_framework.yaml | 2 +- spec/schemas/flow_framework.common.yaml | 67 +++++++++++++++++++++- tests/default/flow_framework/workflow.yaml | 21 ++++--- 4 files changed, 76 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc7835ba2..e60c16650 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -92,7 +92,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) - Added request and response schemas for `/_cluster/decommission/awareness` ([#524](https://github.com/opensearch-project/opensearch-api-specification/pull/524)) - Added `Bytes` component of type number ([#552](https://github.com/opensearch-project/opensearch-api-specification/pull/552)) - Added `xy_shape` query ([#531](https://github.com/opensearch-project/opensearch-api-specification/pull/531)) -- Added `/_plugins/_flow_framework/_search` ,`state/_search`,`_provision`,`_deprovision`,`_steps`,`_status`([#508](https://github.com/opensearch-project/opensearch-api-specification/issues/508)) +- Added `/_plugins/_flow_framework/` ,`_search`,`state/_search`,`_provision`,`_deprovision`,`_steps`,`_status`([#508](https://github.com/opensearch-project/opensearch-api-specification/issues/508)) ### Changed diff --git a/spec/namespaces/flow_framework.yaml b/spec/namespaces/flow_framework.yaml index 0d8c48797..e3a3e005c 100644 --- a/spec/namespaces/flow_framework.yaml +++ b/spec/namespaces/flow_framework.yaml @@ -542,7 +542,7 @@ components: flow_framework.search_state@200: content: application/json: - $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowSearchResponse' + $ref: '../schemas/flow_framework.common.yaml#/components/schemas/WorkflowSearchStateResponse' flow_framework.search_state@403: content: application/json: diff --git a/spec/schemas/flow_framework.common.yaml b/spec/schemas/flow_framework.common.yaml index bad3bdfc8..f14605f49 100644 --- a/spec/schemas/flow_framework.common.yaml +++ b/spec/schemas/flow_framework.common.yaml @@ -7,7 +7,7 @@ paths: { } components: schemas: AllowDelete: - description: A comma-separated list of resource IDs to be deprovisioned. Required if deleting resources of type index_name or pipeline_id. + description: Required when deleting resources involves a high potential for data loss. type: string WorkflowID: type: string @@ -83,7 +83,7 @@ components: result: type: string description: The result of the deletion operation. - enum: [deleted, not_found] + enum: [ deleted, not_found ] _shards: $ref: '#/components/schemas/shards' _seq_no: @@ -174,6 +174,41 @@ components: description: The score of the search hit. _source: $ref: '#/components/schemas/FlowFrameworkGetResponse' + StateItems: + type: object + properties: + _index: + type: string + description: The index name where the document resides. + _id: + type: string + description: The document ID. + _version: + type: integer + description: The version of the document. + _seq_no: + type: integer + description: The sequence number of the document. + _primary_term: + type: integer + description: The primary term of the document. + _score: + type: number + format: float + description: The score of the search hit. + _source: + $ref: '#/components/schemas/SearchStateResponse' + SearchStateResponse: + type: object + properties: + workflow_id: + type: string + provisioning_progress: + type: string + state: + type: string + user: + $ref: '#/components/schemas/user' user: type: object properties: @@ -207,6 +242,19 @@ components: type: array items: $ref: '#/components/schemas/itemsObject' + StateHits: + type: object + properties: + total: + $ref: '#/components/schemas/total' + max_score: + type: number + format: float + description: The maximum score of the search hits. + hits: + type: array + items: + $ref: '#/components/schemas/StateItems' WorkflowStep: type: object properties: @@ -302,4 +350,17 @@ components: _shards: $ref: '#/components/schemas/shards' hits: - $ref: '#/components/schemas/hits' \ No newline at end of file + $ref: '#/components/schemas/hits' + WorkflowSearchStateResponse: + type: object + properties: + took: + type: integer + description: Time in milliseconds that the request took to complete. + timed_out: + type: boolean + description: Indicates if the request timed out. + _shards: + $ref: '#/components/schemas/shards' + hits: + $ref: '#/components/schemas/StateHits' \ No newline at end of file diff --git a/tests/default/flow_framework/workflow.yaml b/tests/default/flow_framework/workflow.yaml index 4ec58fc49..be5c6b071 100644 --- a/tests/default/flow_framework/workflow.yaml +++ b/tests/default/flow_framework/workflow.yaml @@ -58,7 +58,7 @@ chapters: - synopsis: Create workflow With Reprovision set true. version: '>= 2.17' path: /_plugins/_flow_framework/workflow - method: PUT + method: POST parameters: reprovision: true request: @@ -69,7 +69,7 @@ chapters: - synopsis: Create workflow With Reprovision set true and UseCase Not null. version: '>= 2.17' path: /_plugins/_flow_framework/workflow - method: PUT + method: POST parameters: reprovision: true use_case: test_use_case @@ -80,9 +80,8 @@ chapters: status: 400 - synopsis: Update workflow With Reprovision set true and UseCase Not null. version: '>= 2.17' - id: create_flow_framework path: /_plugins/_flow_framework/workflow/{workflow_id} - method: POST + method: PUT parameters: workflow_id: ${create_flow_framework.test_workflow_id} reprovision: true @@ -113,7 +112,7 @@ chapters: path: /_plugins/_flow_framework/workflow/{workflow_id} method: PUT parameters: - workflow_id: test + workflow_id: invalid request: payload: name: test_update_work_flow @@ -138,11 +137,11 @@ chapters: workflow_id: ${create_flow_framework.test_workflow_id} response: status: 200 - - synopsis: Get workflow status with wrong workflow ID. + - synopsis: Get workflow status using an invalid ID. path: /_plugins/_flow_framework/workflow/{workflow_id}/_status method: GET parameters: - workflow_id: test + workflow_id: invalid response: status: 404 - synopsis: Deprovision workflow. @@ -172,11 +171,11 @@ chapters: state: test response: status: 200 - - synopsis: Deprovision workflow with wrong workflow ID. + - synopsis: Deprovision workflow using an invalid ID. path: /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision method: POST parameters: - workflow_id: test + workflow_id: invalid response: status: 404 - synopsis: Delete workflow. @@ -186,11 +185,11 @@ chapters: workflow_id: ${create_flow_framework.test_workflow_id} response: status: 200 - - synopsis: Delete workflow With Wrong ID. + - synopsis: Delete workflow using an invalid ID. path: /_plugins/_flow_framework/workflow/{workflow_id} method: DELETE parameters: - workflow_id: test + workflow_id: invalid response: status: 200 payload: From 7c7678724a1cf44ef95281eda0673e07d76855f3 Mon Sep 17 00:00:00 2001 From: Junwei Dai Date: Wed, 11 Sep 2024 11:07:20 -0700 Subject: [PATCH 11/11] update workflow.yaml add new test file seperate test senario Signed-off-by: Junwei Dai --- spec/schemas/flow_framework.common.yaml | 9 ++- tests/default/flow_framework/deprovision.yaml | 63 +++++++++++++++++++ tests/default/flow_framework/provision.yaml | 55 ++++++++++++++++ tests/default/flow_framework/search.yaml | 40 ++++++++++++ .../default/flow_framework/search_state.yaml | 31 +++++++++ tests/default/flow_framework/status.yaml | 37 +++++++++++ tests/default/flow_framework/steps.yaml | 33 ++++++++++ tests/default/flow_framework/workflow.yaml | 52 +-------------- 8 files changed, 267 insertions(+), 53 deletions(-) create mode 100644 tests/default/flow_framework/deprovision.yaml create mode 100644 tests/default/flow_framework/provision.yaml create mode 100644 tests/default/flow_framework/search.yaml create mode 100644 tests/default/flow_framework/search_state.yaml create mode 100644 tests/default/flow_framework/status.yaml create mode 100644 tests/default/flow_framework/steps.yaml diff --git a/spec/schemas/flow_framework.common.yaml b/spec/schemas/flow_framework.common.yaml index f14605f49..a8d7d7c00 100644 --- a/spec/schemas/flow_framework.common.yaml +++ b/spec/schemas/flow_framework.common.yaml @@ -83,7 +83,7 @@ components: result: type: string description: The result of the deletion operation. - enum: [ deleted, not_found ] + enum: [deleted, not_found] _shards: $ref: '#/components/schemas/shards' _seq_no: @@ -140,7 +140,7 @@ components: match: type: object match_all: - type: string + type: object total: type: object properties: @@ -304,6 +304,11 @@ components: state: type: string description: The current state of the workflow + enum: + - COMPLETED + - FAILED + - NOT_STARTED + - PROVISIONING resources_created: type: array items: diff --git a/tests/default/flow_framework/deprovision.yaml b/tests/default/flow_framework/deprovision.yaml new file mode 100644 index 000000000..43babeeb3 --- /dev/null +++ b/tests/default/flow_framework/deprovision.yaml @@ -0,0 +1,63 @@ +$schema: ../../../json_schemas/test_story.schema.yaml +description: Test flow_framework provision endpoints. +version: '>= 2.12' +prologues: + - path: /_plugins/_flow_framework/workflow + id: create_flow_framework + method: POST + parameters: + use_case: semantic_search_with_cohere_embedding_query_enricher + provision: true + request: + payload: + create_connector.credential.key: test_api_key + output: + test_workflow_id: payload.workflow_id +epilogues: + - path: /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision + method: POST + status: [200, 404] + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + - path: /_plugins/_flow_framework/workflow/{workflow_id} + method: DELETE + status: [200, 404] + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} +chapters: + - synopsis: Check the provision status before calling the deprovision API. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_status + method: GET + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + response: + status: 200 + payload: + workflow_id: ${create_flow_framework.test_workflow_id} + state: PROVISIONING + - synopsis: Deprovision workflow. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision + method: POST + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + response: + status: 200 + - synopsis: Check the provision status after calling the deprovision API. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_status + method: GET + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + response: + status: 200 + payload: + workflow_id: ${create_flow_framework.test_workflow_id} + state: NOT_STARTED + - synopsis: Deprovision workflow using an invalid ID. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision + method: POST + parameters: + workflow_id: invalid + response: + status: 404 + + diff --git a/tests/default/flow_framework/provision.yaml b/tests/default/flow_framework/provision.yaml new file mode 100644 index 000000000..2d52e83e1 --- /dev/null +++ b/tests/default/flow_framework/provision.yaml @@ -0,0 +1,55 @@ +$schema: ../../../json_schemas/test_story.schema.yaml +description: Test flow_framework provision endpoints. +version: '>= 2.12' +prologues: + - path: /_plugins/_flow_framework/workflow + id: create_flow_framework + method: POST + parameters: + use_case: semantic_search_with_cohere_embedding_query_enricher + request: + payload: + create_connector.credential.key: test_api_key + output: + test_workflow_id: payload.workflow_id +epilogues: + - path: /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision + method: POST + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + - path: /_plugins/_flow_framework/workflow/{workflow_id} + method: DELETE + status: [200, 404] + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} +chapters: + - synopsis: Check the status before calling provision API. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_status + method: GET + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + response: + status: 200 + payload: + workflow_id: ${create_flow_framework.test_workflow_id} + state: NOT_STARTED + - synopsis: Provision workflow. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_provision + method: POST + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + request: + payload: + openai_key: '1234556' + response: + status: 200 + - synopsis: Check the provision status after calling provision API. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_status + method: GET + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + response: + status: 200 + payload: + workflow_id: ${create_flow_framework.test_workflow_id} + state: PROVISIONING \ No newline at end of file diff --git a/tests/default/flow_framework/search.yaml b/tests/default/flow_framework/search.yaml new file mode 100644 index 000000000..3e3353079 --- /dev/null +++ b/tests/default/flow_framework/search.yaml @@ -0,0 +1,40 @@ +$schema: ../../../json_schemas/test_story.schema.yaml +description: Test flow_framework provision endpoints. +version: '>= 2.12' +prologues: + - path: /_plugins/_flow_framework/workflow + id: create_flow_framework + method: POST + parameters: + use_case: semantic_search_with_cohere_embedding_query_enricher + request: + payload: + create_connector.credential.key: test_api_key + output: + test_workflow_id: payload.workflow_id +epilogues: + - path: /_plugins/_flow_framework/workflow/{workflow_id} + method: DELETE + status: [200, 404] + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} +chapters: + - synopsis: Search workflow match the name. + path: /_plugins/_flow_framework/workflow/_search + method: POST + request: + payload: + query: + match: + name: test + response: + status: 200 + - synopsis: Search all workflow. + path: /_plugins/_flow_framework/workflow/_search + method: POST + request: + payload: + query: + match_all: {} + response: + status: 200 \ No newline at end of file diff --git a/tests/default/flow_framework/search_state.yaml b/tests/default/flow_framework/search_state.yaml new file mode 100644 index 000000000..7dc59ba04 --- /dev/null +++ b/tests/default/flow_framework/search_state.yaml @@ -0,0 +1,31 @@ +$schema: ../../../json_schemas/test_story.schema.yaml +description: Test flow_framework provision endpoints. +version: '>= 2.12' +prologues: + - path: /_plugins/_flow_framework/workflow + id: create_flow_framework + method: POST + parameters: + use_case: semantic_search_with_cohere_embedding_query_enricher + request: + payload: + create_connector.credential.key: test_api_key + output: + test_workflow_id: payload.workflow_id +epilogues: + - path: /_plugins/_flow_framework/workflow/{workflow_id} + method: DELETE + status: [200, 404] + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} +chapters: + - synopsis: Search workflow state. + path: /_plugins/_flow_framework/workflow/state/_search + method: POST + request: + payload: + query: + match: + state: NOT_STARTED + response: + status: 200 \ No newline at end of file diff --git a/tests/default/flow_framework/status.yaml b/tests/default/flow_framework/status.yaml new file mode 100644 index 000000000..fd9ea2300 --- /dev/null +++ b/tests/default/flow_framework/status.yaml @@ -0,0 +1,37 @@ +$schema: ../../../json_schemas/test_story.schema.yaml +description: Test flow_framework provision endpoints. +version: '>= 2.12' +prologues: + - path: /_plugins/_flow_framework/workflow + id: create_flow_framework + method: POST + parameters: + use_case: semantic_search_with_cohere_embedding_query_enricher + request: + payload: + create_connector.credential.key: test_api_key + output: + test_workflow_id: payload.workflow_id +epilogues: + - path: /_plugins/_flow_framework/workflow/{workflow_id} + method: DELETE + status: [200, 404] + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} +chapters: + - synopsis: Get workflow status. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_status + method: GET + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} + response: + status: 200 + payload: + state: NOT_STARTED + - synopsis: Get workflow status using an invalid ID. + path: /_plugins/_flow_framework/workflow/{workflow_id}/_status + method: GET + parameters: + workflow_id: invalid + response: + status: 404 \ No newline at end of file diff --git a/tests/default/flow_framework/steps.yaml b/tests/default/flow_framework/steps.yaml new file mode 100644 index 000000000..9e4bc1704 --- /dev/null +++ b/tests/default/flow_framework/steps.yaml @@ -0,0 +1,33 @@ +$schema: ../../../json_schemas/test_story.schema.yaml +description: Test flow_framework provision endpoints. +version: '>= 2.12' +prologues: + - path: /_plugins/_flow_framework/workflow + id: create_flow_framework + method: POST + parameters: + use_case: semantic_search_with_cohere_embedding_query_enricher + request: + payload: + create_connector.credential.key: test_api_key + output: + test_workflow_id: payload.workflow_id +epilogues: + - path: /_plugins/_flow_framework/workflow/{workflow_id} + method: DELETE + status: [200, 404] + parameters: + workflow_id: ${create_flow_framework.test_workflow_id} +chapters: + - synopsis: Get workflow steps. + path: /_plugins/_flow_framework/workflow/_steps + method: GET + response: + status: 200 + - synopsis: Get workflow steps with invalid query parameter. + path: /_plugins/_flow_framework/workflow/_steps + method: GET + parameters: + workflow_step: invalid_query + response: + status: 400 \ No newline at end of file diff --git a/tests/default/flow_framework/workflow.yaml b/tests/default/flow_framework/workflow.yaml index be5c6b071..d136c8047 100644 --- a/tests/default/flow_framework/workflow.yaml +++ b/tests/default/flow_framework/workflow.yaml @@ -118,51 +118,8 @@ chapters: name: test_update_work_flow response: status: 404 - - synopsis: Get workflow steps. - path: /_plugins/_flow_framework/workflow/_steps - method: GET - response: - status: 200 - - synopsis: Get workflow steps with bad request. - path: /_plugins/_flow_framework/workflow/_steps - method: GET - parameters: - workflow_step: wrong_query - response: - status: 400 - - synopsis: Get workflow status. - path: /_plugins/_flow_framework/workflow/{workflow_id}/_status - method: GET - parameters: - workflow_id: ${create_flow_framework.test_workflow_id} - response: - status: 200 - - synopsis: Get workflow status using an invalid ID. - path: /_plugins/_flow_framework/workflow/{workflow_id}/_status - method: GET - parameters: - workflow_id: invalid - response: - status: 404 - - synopsis: Deprovision workflow. - path: /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision - method: POST - parameters: - workflow_id: ${create_flow_framework.test_workflow_id} - response: - status: 200 - - synopsis: Search workflow. - path: /_plugins/_flow_framework/workflow/_search - method: POST - request: - payload: - query: - match: - name: test - response: - status: 200 - synopsis: Search workflow state. - path: /_plugins/_flow_framework/workflow/_search + path: /_plugins/_flow_framework/workflow/state/_search method: POST request: payload: @@ -171,13 +128,6 @@ chapters: state: test response: status: 200 - - synopsis: Deprovision workflow using an invalid ID. - path: /_plugins/_flow_framework/workflow/{workflow_id}/_deprovision - method: POST - parameters: - workflow_id: invalid - response: - status: 404 - synopsis: Delete workflow. path: /_plugins/_flow_framework/workflow/{workflow_id} method: DELETE