From d16fe89e4bb49d4421483ddf1280202d2c94441f Mon Sep 17 00:00:00 2001 From: Nabanita Paul Date: Mon, 11 Sep 2023 18:58:10 -0500 Subject: [PATCH 1/9] Adding pattern files for APIGw SageMaker integration --- apigw-rest-sagemaker/README.md | 89 +++++++++ apigw-rest-sagemaker/example-pattern.json | 56 ++++++ apigw-rest-sagemaker/template.yaml | 214 ++++++++++++++++++++++ 3 files changed, 359 insertions(+) create mode 100644 apigw-rest-sagemaker/README.md create mode 100644 apigw-rest-sagemaker/example-pattern.json create mode 100644 apigw-rest-sagemaker/template.yaml diff --git a/apigw-rest-sagemaker/README.md b/apigw-rest-sagemaker/README.md new file mode 100644 index 000000000..d0d9d81f5 --- /dev/null +++ b/apigw-rest-sagemaker/README.md @@ -0,0 +1,89 @@ +# AWS API Gateway REST Edge-Optimized API to Amazon SageMaker + +This pattern creates an Amazon API Gateway REST Edge-Optimized API with an Amazon SageMaker integration. It deploys a SageMaker Jumpstart model (Flan T5 XL) endpoint that is used for the integration. + +Learn more about this pattern at Serverless Land Patterns: + +Important: this application uses various AWS services and there are costs associated with these services after the Free Tier usage - please see the [AWS Pricing page](https://aws.amazon.com/pricing/) for details. You are responsible for any AWS costs incurred. No warranty is implied in this example. + +## Requirements + +* [Create an AWS account](https://portal.aws.amazon.com/gp/aws/developer/registration/index.html) if you do not already have one and log in. The IAM user that you use must have sufficient permissions to make necessary AWS service calls and manage AWS resources. +* [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html) installed and configured +* [Git Installed](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) +* [AWS Serverless Application Model](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html) (AWS SAM) installed + +## Deployment Instructions + +1. Create a new directory, navigate to that directory in a terminal and clone the GitHub repository: + ``` + git clone https://github.com/aws-samples/serverless-patterns + ``` +2. Change directory to the pattern directory: + ``` + cd apigw-rest-sagemaker + ``` +3. From the command line, use AWS SAM to deploy the AWS resources for the pattern as specified in the template.yml file: + ``` + sam deploy --guided + ``` +4. During the prompts: + * Enter a stack name + * Enter the desired AWS Region + * Allow SAM CLI to create IAM roles with the required permissions. + + Once you have run `sam deploy --guided` mode once and saved arguments to a configuration file (samconfig.toml), you can use `sam deploy` in future to use these defaults. + +5. Note the output from the SAM deployment process. It contains the API endpoint. + +## How it works + +1. This pattern deploys a SageMaker Jumpstart model (Flan T5 XL from HuggingFace) endpoint using Amazon SageMaker. To deploy the solution with a different model, replace the ModelData and ImageURI parameters in the template.yaml file. + +2. The pattern also adds an API Gateway query the endpoint. + +3. The API Gateway is protected using an API Key. To query the Api Gateway, ```x-api-key``` header needs to be added to the HTTP request. + +## Testing + +Once the application is deployed, retrieve the API URL provided as output and append the resource name "/text". Then make the request from Postman or from a terminal. The URL should look like this : https://[api-id].execute-api.[api-region].amazonaws.com/v1/text + +Postman Example + ```bash + https://aabbccddee.execute-api.eu-west-1.amazonaws.com/text + ``` + +OR open a terminal and execute the curl command + +Example + ```bash + curl -X POST 'https://aabbccddee.execute-api.eu-west-1.amazonaws.com/v1/text' \ +--header 'x-api-key: Q425Bv0mFe7s6C4jRrCAlazVkSlXXXXXXXXXXXXX' \ +--header 'Content-Type: application/json' \ +--data '{ + "text_inputs": "Translate to Spanish: My dog is very beautiful", + "max_length": 5000 +}' + ``` +The expected response is : 'Yo nac en Madrid' + + +## Cleanup + +1. Delete the stack + ```bash + aws cloudformation delete-stack --stack-name STACK_NAME + ``` +1. Confirm the stack has been deleted + ```bash + aws cloudformation list-stacks --query "StackSummaries[?contains(StackName,'STACK_NAME')].StackStatus" + ``` +## Author bio + +Nabanita Paul, +https://www.linkedin.com/in/nabanita-paul/ +Cloud Support Engineer II +---- +Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +SPDX-License-Identifier: MIT-0 \ No newline at end of file diff --git a/apigw-rest-sagemaker/example-pattern.json b/apigw-rest-sagemaker/example-pattern.json new file mode 100644 index 000000000..1b3ce342d --- /dev/null +++ b/apigw-rest-sagemaker/example-pattern.json @@ -0,0 +1,56 @@ +{ + "title": "API Gateway REST API to SageMaker", + "description": "Create a an Amazon API Gateway REST API to invoke a Sagemaker endpoint.", + "level": "300", + "framework": "SAM", + "introBox": { + "headline": "How it works", + "text": [ + "This sample project demonstrates how to use an use a API Gateway REST API to invoke a SageMaker endpoint. This pattern is leveraging the native AWS service integration between these 2 services.", + "The pattern deploys a SageMaker endpoint for the Flan-T5 foundation model in Amazon SageMaker JumpStart. To deploy the pattern with a different model, replace the ModelData and ImageURI parameters in the template.yaml file. The template also deploys a REST API with AWS service integration to invoke the SageMaker endpoint directly. " + ] + }, + "gitHub": { + "template": { + "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/apigw-rest-sagemaker", + "templateURL": "serverless-patterns/apigw-rest-sagemaker", + "projectFolder": "apigw-rest-sagemaker", + "templateFile": "apigw-rest-sagemaker/template.yaml" + } + }, + "resources": { + "bullets": [ + { + "text": "How do I use API Gateway as a proxy for another AWS service?", + "link": "https://repost.aws/knowledge-center/api-gateway-proxy-integrate-service" + }, + { + "text": "Zero-shot prompting for the Flan-T5 foundation model in Amazon SageMaker JumpStart", + "link": "https://aws.amazon.com/blogs/machine-learning/zero-shot-prompting-for-the-flan-t5-foundation-model-in-amazon-sagemaker-jumpstart/" + } + ] + }, + "deploy": { + "text": [ + "sam deploy" + ] + }, + "testing": { + "text": [ + "See the GitHub repo for detailed testing instructions." + ] + }, + "cleanup": { + "text": [ + "Delete the stack: aws cloudformation delete-stack --stack-name STACK_NAME" + ] + }, + "authors": [ + { + "name": "Nabanita Paul", + "image": "https://drive.google.com/file/d/1HBr3lD-QFPS4lw-46ZDx0i_4HYnJ0mrB/view?usp=drive_link", + "bio": "Cloud Support Engineer II @AWS", + "linkedin": "https://www.linkedin.com/in/nabanita-paul/" + } + ] +} diff --git a/apigw-rest-sagemaker/template.yaml b/apigw-rest-sagemaker/template.yaml new file mode 100644 index 000000000..334faa8eb --- /dev/null +++ b/apigw-rest-sagemaker/template.yaml @@ -0,0 +1,214 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: An Amazon API Gateway REST API that integrates with a SageMaker endpoint. +Parameters: + ApigatewayName: + Type: String + Default: sagemaker-api + Description: Name of application. + + ApigatewayStageName: + Type: String + Default: v1 + Description: Name of API stage. + + SageMakerImageURI: + Type: String + Default: 763104351884.dkr.ecr.us-east-1.amazonaws.com/huggingface-pytorch-inference:1.10.2-transformers4.17.0-gpu-py38-cu113-ubuntu20.04 + Description: Container URI for inference endpoint + + SageMakerModelData: + Type: String + Default: s3://jumpstart-cache-prod-us-east-1/huggingface-infer/prepack/v1.0.3/infer-prepack-huggingface-text2text-flan-t5-xl.tar.gz + Description: S3 location for SageMaker JumpStart model + + SageMakerInstanceType: + Type: String + Default: ml.p3.2xlarge + Description: Instance type for SageMaker endpoint + + SageMakerInstanceCount: + Type: String + Default: 1 + Description: Number of instances for SageMaker endpoint + +Resources: + # Execution IAM role for SageMaker Model + SageMakerExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + - sagemaker.amazonaws.com + Action: + - sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/AmazonSageMakerFullAccess + + # SageMaker Model + SageMakerModel: + Type: AWS::SageMaker::Model + Properties: + ModelName: apigateway-SageMakerModel + Containers: + - Image: !Ref SageMakerImageURI + ModelDataUrl: !Ref SageMakerModelData + Mode: SingleModel + Environment: + { + "MODEL_CACHE_ROOT": "/opt/ml/model", + "SAGEMAKER_ENV": "1", + "SAGEMAKER_MODEL_SERVER_TIMEOUT": "3600", + "SAGEMAKER_MODEL_SERVER_WORKERS": "1", + "SAGEMAKER_PROGRAM": "inference.py", + "SAGEMAKER_SUBMIT_DIRECTORY": "/opt/ml/model/code/", + "TS_DEFAULT_WORKERS_PER_MODEL": 1, + } + EnableNetworkIsolation: true + ExecutionRoleArn: !GetAtt SageMakerExecutionRole.Arn + + # SageMaker Endpoint Config + SageMakerEndpointConfig: + Type: AWS::SageMaker::EndpointConfig + Properties: + EndpointConfigName: apigateway-SageMakerEndpointConfig + ProductionVariants: + - ModelName: !GetAtt SageMakerModel.ModelName + VariantName: !Sub ${SageMakerModel.ModelName}-1 + InitialInstanceCount: !Ref SageMakerInstanceCount + InstanceType: !Ref SageMakerInstanceType + InitialVariantWeight: 1.0 + VolumeSizeInGB: 40 + + # SageMaker Endpoint + SageMakerEndpoint: + Type: AWS::SageMaker::Endpoint + Properties: + EndpointName: apigateway-sagemakerendpoint + EndpointConfigName: !GetAtt SageMakerEndpointConfig.EndpointConfigName + + # Execution IAM role for API gateway + APIGatewayRole: + Type: 'AWS::IAM::Role' + DependsOn: + - SageMakerEndpoint + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Action: + - 'sts:AssumeRole' + Effect: Allow + Principal: + Service: + - apigateway.amazonaws.com + Policies: + - PolicyName: APIGatewaySageMakerPolicy + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 'sagemaker:InvokeEndpoint' + Resource: !Sub + - '${varEndpointArn}' + - varEndpointArn: !Join ['', ['arn:aws:sagemaker:', !Ref AWS::Region, ':',!Ref AWS::AccountId, ':endpoint/', !GetAtt SageMakerEndpoint.EndpointName]] + + # REST API + Api: + Type: 'AWS::ApiGateway::RestApi' + DependsOn: + - APIGatewayRole + Properties: + Name: !Sub '${ApigatewayName}' + ApiKeySourceType: HEADER + + # API Resource + TextResource: + Type: 'AWS::ApiGateway::Resource' + Properties: + RestApiId: !Ref Api + ParentId: !GetAtt Api.RootResourceId + PathPart: 'text' + + # API method + TextMethodPost: + Type: 'AWS::ApiGateway::Method' + Properties: + RestApiId: !Ref Api + ResourceId: !Ref TextResource + HttpMethod: POST + ApiKeyRequired: true + AuthorizationType: NONE + Integration: + Type: AWS + Credentials: !GetAtt APIGatewayRole.Arn + # Should always be POST when integrating with AWS services + IntegrationHttpMethod: POST + # More info: https://docs.aws.amazon.com/apigateway/api-reference/resource/integration/ + Uri: !Join ['', ['arn:aws:apigateway:', !Ref AWS::Region, ':runtime.sagemaker:path/endpoints/', !GetAtt SageMakerEndpoint.EndpointName, '/invocations']] + PassthroughBehavior: WHEN_NO_MATCH + IntegrationResponses: + - StatusCode: '200' + MethodResponses: + - StatusCode: '200' + ResponseModels: { "application/json": "Empty" } + + # Deployment of API. Note: Change the logical ID every time when updating the API configuration as redeployment through CFN does not work if logical ID is the same + ApiDeployment: + Type: 'AWS::ApiGateway::Deployment' + DependsOn: + - TextMethodPost + Properties: + RestApiId: !Ref Api + StageName: !Sub '${ApigatewayStageName}' + + # API key that client will pass in 'x-api-key' header + ApiKey: + Type: 'AWS::ApiGateway::ApiKey' + DependsOn: + - ApiDeployment + Properties: + Enabled: true + Name: !Sub '${ApigatewayName}-apikey' + StageKeys: + - RestApiId: !Ref Api + StageName: !Sub '${ApigatewayStageName}' + + #API Usage plan to define Throttle limits + ApiUsagePlan: + Type: 'AWS::ApiGateway::UsagePlan' + DependsOn: + - ApiDeployment + Properties: + ApiStages: + - ApiId: !Ref Api + Stage: !Sub '${ApigatewayStageName}' + Throttle: + RateLimit: 500 + BurstLimit: 1000 + UsagePlanName: !Sub '${ApigatewayName}-usage-plan' + Quota: + Limit: 10000 + Period: MONTH + + #Associate API key with Usage plan + ApiUsagePlanKey: + Type: 'AWS::ApiGateway::UsagePlanKey' + Properties: + KeyType: API_KEY + KeyId: !Ref ApiKey + UsagePlanId: !Ref ApiUsagePlan + +Outputs: + ApiRootUrl: + Description: Root Url of the API + Value: !Sub + - 'https://${ApiId}.execute-api.${AWS::Region}.amazonaws.com/${ApigatewayStageName}' + - ApiId: !Ref Api + ApiKeyId: + Description: API Key Id + Value: !Ref ApiKey \ No newline at end of file From b61699623046ebc7f16459d400c14969314bd728 Mon Sep 17 00:00:00 2001 From: Nabanita Paul Date: Wed, 13 Sep 2023 10:25:43 -0500 Subject: [PATCH 2/9] Adding language property to example-pattern.json file --- apigw-rest-sagemaker/example-pattern.json | 1 + 1 file changed, 1 insertion(+) diff --git a/apigw-rest-sagemaker/example-pattern.json b/apigw-rest-sagemaker/example-pattern.json index 1b3ce342d..03aacbec2 100644 --- a/apigw-rest-sagemaker/example-pattern.json +++ b/apigw-rest-sagemaker/example-pattern.json @@ -1,6 +1,7 @@ { "title": "API Gateway REST API to SageMaker", "description": "Create a an Amazon API Gateway REST API to invoke a Sagemaker endpoint.", + "language": "", "level": "300", "framework": "SAM", "introBox": { From 4d785f6baf9747b58c27b5704e9bc3501e055682 Mon Sep 17 00:00:00 2001 From: Nabanita Paul <76739433+naba819@users.noreply.github.com> Date: Thu, 21 Sep 2023 12:06:52 -0500 Subject: [PATCH 3/9] Adding integration image to README.md --- apigw-rest-sagemaker/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apigw-rest-sagemaker/README.md b/apigw-rest-sagemaker/README.md index d0d9d81f5..75a9938ff 100644 --- a/apigw-rest-sagemaker/README.md +++ b/apigw-rest-sagemaker/README.md @@ -1,4 +1,7 @@ # AWS API Gateway REST Edge-Optimized API to Amazon SageMaker +

+ +

This pattern creates an Amazon API Gateway REST Edge-Optimized API with an Amazon SageMaker integration. It deploys a SageMaker Jumpstart model (Flan T5 XL) endpoint that is used for the integration. @@ -86,4 +89,4 @@ Cloud Support Engineer II ---- Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. -SPDX-License-Identifier: MIT-0 \ No newline at end of file +SPDX-License-Identifier: MIT-0 From 365c5e819b432ca57ecbc614820bd27256e69738 Mon Sep 17 00:00:00 2001 From: Nabanita Paul <76739433+naba819@users.noreply.github.com> Date: Wed, 18 Oct 2023 09:23:28 -0500 Subject: [PATCH 4/9] Update README.md Updated details about testing using Postman. --- apigw-rest-sagemaker/README.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/apigw-rest-sagemaker/README.md b/apigw-rest-sagemaker/README.md index 75a9938ff..aada62f51 100644 --- a/apigw-rest-sagemaker/README.md +++ b/apigw-rest-sagemaker/README.md @@ -52,10 +52,29 @@ Important: this application uses various AWS services and there are costs associ Once the application is deployed, retrieve the API URL provided as output and append the resource name "/text". Then make the request from Postman or from a terminal. The URL should look like this : https://[api-id].execute-api.[api-region].amazonaws.com/v1/text Postman Example - ```bash +1. URL: + ``` https://aabbccddee.execute-api.eu-west-1.amazonaws.com/text ``` +2. Method: +``` +POST +``` + +3. Request Header and value: +``` +x-api-key: Q425Bv0mFe7s6C4jRrCAlazVkSlXXXXXXXXXXXXX +``` + +4. Body: Choose "raw" radio button and "JSON" from dropdownlist. +``` +{ + "text_inputs": "A step by step recipe to make chicken noodles:", + "max_length": 5000 +} +``` + OR open a terminal and execute the curl command Example From 3d5be32cbe6fa188c0dcd022360ff066433b1c8f Mon Sep 17 00:00:00 2001 From: Nabanita Paul <76739433+naba819@users.noreply.github.com> Date: Wed, 18 Oct 2023 09:27:17 -0500 Subject: [PATCH 5/9] Corrected request URL for Postman in testing section README.md --- apigw-rest-sagemaker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apigw-rest-sagemaker/README.md b/apigw-rest-sagemaker/README.md index aada62f51..6fb1686e8 100644 --- a/apigw-rest-sagemaker/README.md +++ b/apigw-rest-sagemaker/README.md @@ -54,7 +54,7 @@ Once the application is deployed, retrieve the API URL provided as output and ap Postman Example 1. URL: ``` - https://aabbccddee.execute-api.eu-west-1.amazonaws.com/text + https://aabbccddee.execute-api.eu-west-1.amazonaws.com/v1/text ``` 2. Method: From a0ae8f2255b997ebc058cab42c590dcfb35e2bb0 Mon Sep 17 00:00:00 2001 From: Ben Smith Date: Mon, 23 Oct 2023 11:28:22 -0500 Subject: [PATCH 6/9] Update apigw-rest-sagemaker/example-pattern.json --- apigw-rest-sagemaker/example-pattern.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apigw-rest-sagemaker/example-pattern.json b/apigw-rest-sagemaker/example-pattern.json index 03aacbec2..79286979a 100644 --- a/apigw-rest-sagemaker/example-pattern.json +++ b/apigw-rest-sagemaker/example-pattern.json @@ -7,7 +7,7 @@ "introBox": { "headline": "How it works", "text": [ - "This sample project demonstrates how to use an use a API Gateway REST API to invoke a SageMaker endpoint. This pattern is leveraging the native AWS service integration between these 2 services.", + "This pattern shows how to use an use a API Gateway REST API to invoke a SageMaker endpoint using using a native AWS service integration.", "The pattern deploys a SageMaker endpoint for the Flan-T5 foundation model in Amazon SageMaker JumpStart. To deploy the pattern with a different model, replace the ModelData and ImageURI parameters in the template.yaml file. The template also deploys a REST API with AWS service integration to invoke the SageMaker endpoint directly. " ] }, From d8626ddb63f03435821055490577c22c1cd3ace1 Mon Sep 17 00:00:00 2001 From: Ben Smith Date: Mon, 23 Oct 2023 11:28:28 -0500 Subject: [PATCH 7/9] Update apigw-rest-sagemaker/example-pattern.json --- apigw-rest-sagemaker/example-pattern.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apigw-rest-sagemaker/example-pattern.json b/apigw-rest-sagemaker/example-pattern.json index 79286979a..179d5820d 100644 --- a/apigw-rest-sagemaker/example-pattern.json +++ b/apigw-rest-sagemaker/example-pattern.json @@ -49,7 +49,7 @@ "authors": [ { "name": "Nabanita Paul", - "image": "https://drive.google.com/file/d/1HBr3lD-QFPS4lw-46ZDx0i_4HYnJ0mrB/view?usp=drive_link", + "image": "nabanita-paul.jpeg", "bio": "Cloud Support Engineer II @AWS", "linkedin": "https://www.linkedin.com/in/nabanita-paul/" } From 33ede1ca86651e7ef63b36d25d33f70398fa54ec Mon Sep 17 00:00:00 2001 From: Ben Smith Date: Mon, 23 Oct 2023 11:28:33 -0500 Subject: [PATCH 8/9] Update apigw-rest-sagemaker/example-pattern.json --- apigw-rest-sagemaker/example-pattern.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apigw-rest-sagemaker/example-pattern.json b/apigw-rest-sagemaker/example-pattern.json index 179d5820d..c2501ee13 100644 --- a/apigw-rest-sagemaker/example-pattern.json +++ b/apigw-rest-sagemaker/example-pattern.json @@ -16,7 +16,7 @@ "repoURL": "https://github.com/aws-samples/serverless-patterns/tree/main/apigw-rest-sagemaker", "templateURL": "serverless-patterns/apigw-rest-sagemaker", "projectFolder": "apigw-rest-sagemaker", - "templateFile": "apigw-rest-sagemaker/template.yaml" + "templateFile": "template.yaml" } }, "resources": { From 7517f2073f2d03960340719c5d73d820e26aebd0 Mon Sep 17 00:00:00 2001 From: Ben Smith Date: Mon, 23 Oct 2023 11:28:38 -0500 Subject: [PATCH 9/9] Update apigw-rest-sagemaker/example-pattern.json --- apigw-rest-sagemaker/example-pattern.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apigw-rest-sagemaker/example-pattern.json b/apigw-rest-sagemaker/example-pattern.json index c2501ee13..d48c623f1 100644 --- a/apigw-rest-sagemaker/example-pattern.json +++ b/apigw-rest-sagemaker/example-pattern.json @@ -51,7 +51,7 @@ "name": "Nabanita Paul", "image": "nabanita-paul.jpeg", "bio": "Cloud Support Engineer II @AWS", - "linkedin": "https://www.linkedin.com/in/nabanita-paul/" + "linkedin": "nabanita-paul" } ] }