Skip to content

Commit 021076e

Browse files
leandrodamascenaCavalcante Damascena
leandrodamascena
and
Cavalcante Damascena
authored
feat(parser): add support for Pydantic v2 (aws-powertools#2733)
* pydantic v2: initial tests * pydantic v2: comment * pydantic v2: new workflow * pydantic v2: comment * pydantic v2: mypy fix * pydantic v2: fix v2 compability * pydantic v2: fix last things * pydantic v2: improving comments * pydantic v2: addressing Heitor's feedback * pydantic v2: creating pydantic v2 specific test * pydantic v2: using fixture to clean the code * pydanticv2: reverting Optional fields * Removing the validators. Pydantic bug was fixed Signed-off-by: Cavalcante Damascena <[email protected]> * Adding pytest ignore messages for Pydantic v2 Signed-off-by: Cavalcante Damascena <[email protected]> * Adding pytest ignore messages for Pydantic v2 Signed-off-by: Cavalcante Damascena <[email protected]> * pydanticv2: removing duplicated workflow + disabling warning * pydanticv2: adding documentation * Adding cache to disable pydantic warnings Signed-off-by: Cavalcante Damascena <[email protected]> * Adjusting workflow Signed-off-by: Cavalcante Damascena <[email protected]> * Addressing Heitor's feedback Signed-off-by: Cavalcante Damascena <[email protected]> * Removed codecov upload Signed-off-by: Cavalcante Damascena <[email protected]> --------- Signed-off-by: Cavalcante Damascena <[email protected]> Co-authored-by: Cavalcante Damascena <[email protected]>
1 parent 1dd46b6 commit 021076e

27 files changed

+299
-106
lines changed

‎.github/workflows/quality_check_pydanticv2.yml

+76
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
name: Code quality - Pydanticv2
2+
3+
# PROCESS
4+
#
5+
# 1. Install all dependencies and spin off containers for all supported Python versions
6+
# 2. Run code formatters and linters (various checks) for code standard
7+
# 3. Run static typing checker for potential bugs
8+
# 4. Run entire test suite for regressions except end-to-end (unit, functional, performance)
9+
# 5. Run static analysis (in addition to CodeQL) for common insecure code practices
10+
# 6. Run complexity baseline to avoid error-prone bugs and keep maintenance lower
11+
# 7. Collect and report on test coverage
12+
13+
# USAGE
14+
#
15+
# Always triggered on new PRs, PR changes and PR merge.
16+
17+
18+
on:
19+
pull_request:
20+
paths:
21+
- "aws_lambda_powertools/**"
22+
- "tests/**"
23+
- "pyproject.toml"
24+
- "poetry.lock"
25+
- "mypy.ini"
26+
branches:
27+
- develop
28+
push:
29+
paths:
30+
- "aws_lambda_powertools/**"
31+
- "tests/**"
32+
- "pyproject.toml"
33+
- "poetry.lock"
34+
- "mypy.ini"
35+
branches:
36+
- develop
37+
38+
permissions:
39+
contents: read
40+
41+
jobs:
42+
quality_check:
43+
runs-on: ubuntu-latest
44+
strategy:
45+
max-parallel: 4
46+
matrix:
47+
python-version: ["3.7", "3.8", "3.9", "3.10"]
48+
env:
49+
PYTHON: "${{ matrix.python-version }}"
50+
permissions:
51+
contents: read # checkout code only
52+
steps:
53+
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
54+
- name: Install poetry
55+
run: pipx install poetry
56+
- name: Set up Python ${{ matrix.python-version }}
57+
uses: actions/setup-python@bd6b4b6205c4dbad673328db7b31b7fab9e241c0 # v4.6.1
58+
with:
59+
python-version: ${{ matrix.python-version }}
60+
cache: "poetry"
61+
- name: Removing dev dependencies locked to Pydantic v1
62+
run: poetry remove cfn-lint
63+
- name: Replacing Pydantic v1 with v2 > 2.0.3
64+
run: poetry add "pydantic=^2.0.3"
65+
- name: Install dependencies
66+
run: make dev
67+
- name: Formatting and Linting
68+
run: make lint
69+
- name: Static type checking
70+
run: make mypy
71+
- name: Test with pytest
72+
run: make test
73+
- name: Security baseline
74+
run: make security-baseline
75+
- name: Complexity baseline
76+
run: make complexity-baseline

‎aws_lambda_powertools/utilities/batch/base.py

+19-4
Original file line numberDiff line numberDiff line change
@@ -348,6 +348,11 @@ def _to_batch_type(self, record: dict, event_type: EventType) -> EventSourceData
348348

349349
def _to_batch_type(self, record: dict, event_type: EventType, model: Optional["BatchTypeModels"] = None):
350350
if model is not None:
351+
# If a model is provided, we assume Pydantic is installed and we need to disable v2 warnings
352+
from aws_lambda_powertools.utilities.parser.compat import disable_pydantic_v2_warning
353+
354+
disable_pydantic_v2_warning()
355+
351356
return model.parse_obj(record)
352357
return self._DATA_CLASS_MAPPING[event_type](record)
353358

@@ -500,8 +505,13 @@ def _process_record(self, record: dict) -> Union[SuccessResponse, FailureRespons
500505
# we need to handle that exception differently.
501506
# We check for a public attr in validation errors coming from Pydantic exceptions (subclass or not)
502507
# and we compare if it's coming from the same model that trigger the exception in the first place
503-
model = getattr(exc, "model", None)
504-
if model == self.model:
508+
509+
# Pydantic v1 raises a ValidationError with ErrorWrappers and store the model instance in a class variable.
510+
# Pydantic v2 simplifies this by adding a title variable to store the model name directly.
511+
model = getattr(exc, "model", None) or getattr(exc, "title", None)
512+
model_name = getattr(self.model, "__name__", None)
513+
514+
if model == self.model or model == model_name:
505515
return self._register_model_validation_error_record(record)
506516

507517
return self.failure_handler(record=data, exception=sys.exc_info())
@@ -644,8 +654,13 @@ async def _async_process_record(self, record: dict) -> Union[SuccessResponse, Fa
644654
# we need to handle that exception differently.
645655
# We check for a public attr in validation errors coming from Pydantic exceptions (subclass or not)
646656
# and we compare if it's coming from the same model that trigger the exception in the first place
647-
model = getattr(exc, "model", None)
648-
if model == self.model:
657+
658+
# Pydantic v1 raises a ValidationError with ErrorWrappers and store the model instance in a class variable.
659+
# Pydantic v2 simplifies this by adding a title variable to store the model name directly.
660+
model = getattr(exc, "model", None) or getattr(exc, "title", None)
661+
model_name = getattr(self.model, "__name__", None)
662+
663+
if model == self.model or model == model_name:
649664
return self._register_model_validation_error_record(record)
650665

651666
return self.failure_handler(record=data, exception=sys.exc_info())

‎aws_lambda_powertools/utilities/parser/compat.py

+34
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
import functools
2+
3+
4+
@functools.lru_cache(maxsize=None)
5+
def disable_pydantic_v2_warning():
6+
"""
7+
Disables the Pydantic version 2 warning by filtering out the related warnings.
8+
9+
This function checks the version of Pydantic currently installed and if it is version 2,
10+
it filters out the PydanticDeprecationWarning and PydanticDeprecatedSince20 warnings
11+
to suppress them.
12+
13+
Since we only need to run the code once, we are using lru_cache to improve performance.
14+
15+
Note: This function assumes that Pydantic is installed.
16+
17+
Usage:
18+
disable_pydantic_v2_warning()
19+
"""
20+
try:
21+
from pydantic import __version__
22+
23+
version = __version__.split(".")
24+
25+
if int(version[0]) == 2:
26+
import warnings
27+
28+
from pydantic import PydanticDeprecatedSince20, PydanticDeprecationWarning
29+
30+
warnings.filterwarnings("ignore", category=PydanticDeprecationWarning)
31+
warnings.filterwarnings("ignore", category=PydanticDeprecatedSince20)
32+
33+
except ImportError:
34+
pass

‎aws_lambda_powertools/utilities/parser/envelopes/base.py

+3
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
from abc import ABC, abstractmethod
33
from typing import Any, Dict, Optional, Type, TypeVar, Union
44

5+
from aws_lambda_powertools.utilities.parser.compat import disable_pydantic_v2_warning
56
from aws_lambda_powertools.utilities.parser.types import Model
67

78
logger = logging.getLogger(__name__)
@@ -26,6 +27,8 @@ def _parse(data: Optional[Union[Dict[str, Any], Any]], model: Type[Model]) -> Un
2627
Any
2728
Parsed data
2829
"""
30+
disable_pydantic_v2_warning()
31+
2932
if data is None:
3033
logger.debug("Skipping parsing as event is None")
3134
return data

‎aws_lambda_powertools/utilities/parser/models/__init__.py

+4
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
from aws_lambda_powertools.utilities.parser.compat import disable_pydantic_v2_warning
2+
3+
disable_pydantic_v2_warning()
4+
15
from .alb import AlbModel, AlbRequestContext, AlbRequestContextData
26
from .apigw import (
37
APIGatewayEventAuthorizer,

‎aws_lambda_powertools/utilities/parser/models/apigw.py

+36-36
Original file line numberDiff line numberDiff line change
@@ -21,74 +21,74 @@ class ApiGatewayUserCert(BaseModel):
2121

2222

2323
class APIGatewayEventIdentity(BaseModel):
24-
accessKey: Optional[str]
25-
accountId: Optional[str]
26-
apiKey: Optional[str]
27-
apiKeyId: Optional[str]
28-
caller: Optional[str]
29-
cognitoAuthenticationProvider: Optional[str]
30-
cognitoAuthenticationType: Optional[str]
31-
cognitoIdentityId: Optional[str]
32-
cognitoIdentityPoolId: Optional[str]
33-
principalOrgId: Optional[str]
24+
accessKey: Optional[str] = None
25+
accountId: Optional[str] = None
26+
apiKey: Optional[str] = None
27+
apiKeyId: Optional[str] = None
28+
caller: Optional[str] = None
29+
cognitoAuthenticationProvider: Optional[str] = None
30+
cognitoAuthenticationType: Optional[str] = None
31+
cognitoIdentityId: Optional[str] = None
32+
cognitoIdentityPoolId: Optional[str] = None
33+
principalOrgId: Optional[str] = None
3434
# see #1562, temp workaround until API Gateway fixes it the Test button payload
3535
# removing it will not be considered a regression in the future
3636
sourceIp: Union[IPvAnyNetwork, Literal["test-invoke-source-ip"]]
37-
user: Optional[str]
38-
userAgent: Optional[str]
39-
userArn: Optional[str]
40-
clientCert: Optional[ApiGatewayUserCert]
37+
user: Optional[str] = None
38+
userAgent: Optional[str] = None
39+
userArn: Optional[str] = None
40+
clientCert: Optional[ApiGatewayUserCert] = None
4141

4242

4343
class APIGatewayEventAuthorizer(BaseModel):
44-
claims: Optional[Dict[str, Any]]
45-
scopes: Optional[List[str]]
44+
claims: Optional[Dict[str, Any]] = None
45+
scopes: Optional[List[str]] = None
4646

4747

4848
class APIGatewayEventRequestContext(BaseModel):
4949
accountId: str
5050
apiId: str
51-
authorizer: Optional[APIGatewayEventAuthorizer]
51+
authorizer: Optional[APIGatewayEventAuthorizer] = None
5252
stage: str
5353
protocol: str
5454
identity: APIGatewayEventIdentity
5555
requestId: str
5656
requestTime: str
5757
requestTimeEpoch: datetime
58-
resourceId: Optional[str]
58+
resourceId: Optional[str] = None
5959
resourcePath: str
60-
domainName: Optional[str]
61-
domainPrefix: Optional[str]
62-
extendedRequestId: Optional[str]
60+
domainName: Optional[str] = None
61+
domainPrefix: Optional[str] = None
62+
extendedRequestId: Optional[str] = None
6363
httpMethod: Literal["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"]
6464
path: str
65-
connectedAt: Optional[datetime]
66-
connectionId: Optional[str]
67-
eventType: Optional[Literal["CONNECT", "MESSAGE", "DISCONNECT"]]
68-
messageDirection: Optional[str]
69-
messageId: Optional[str]
70-
routeKey: Optional[str]
71-
operationName: Optional[str]
65+
connectedAt: Optional[datetime] = None
66+
connectionId: Optional[str] = None
67+
eventType: Optional[Literal["CONNECT", "MESSAGE", "DISCONNECT"]] = None
68+
messageDirection: Optional[str] = None
69+
messageId: Optional[str] = None
70+
routeKey: Optional[str] = None
71+
operationName: Optional[str] = None
7272

73-
@root_validator(allow_reuse=True)
73+
@root_validator(allow_reuse=True, skip_on_failure=True)
7474
def check_message_id(cls, values):
7575
message_id, event_type = values.get("messageId"), values.get("eventType")
7676
if message_id is not None and event_type != "MESSAGE":
77-
raise TypeError("messageId is available only when the `eventType` is `MESSAGE`")
77+
raise ValueError("messageId is available only when the `eventType` is `MESSAGE`")
7878
return values
7979

8080

8181
class APIGatewayProxyEventModel(BaseModel):
82-
version: Optional[str]
82+
version: Optional[str] = None
8383
resource: str
8484
path: str
8585
httpMethod: Literal["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"]
8686
headers: Dict[str, str]
8787
multiValueHeaders: Dict[str, List[str]]
88-
queryStringParameters: Optional[Dict[str, str]]
89-
multiValueQueryStringParameters: Optional[Dict[str, List[str]]]
88+
queryStringParameters: Optional[Dict[str, str]] = None
89+
multiValueQueryStringParameters: Optional[Dict[str, List[str]]] = None
9090
requestContext: APIGatewayEventRequestContext
91-
pathParameters: Optional[Dict[str, str]]
92-
stageVariables: Optional[Dict[str, str]]
91+
pathParameters: Optional[Dict[str, str]] = None
92+
stageVariables: Optional[Dict[str, str]] = None
9393
isBase64Encoded: bool
94-
body: Optional[Union[str, Type[BaseModel]]]
94+
body: Optional[Union[str, Type[BaseModel]]] = None

‎aws_lambda_powertools/utilities/parser/models/apigwv2.py

+15-15
Original file line numberDiff line numberDiff line change
@@ -14,13 +14,13 @@ class RequestContextV2AuthorizerIamCognito(BaseModel):
1414

1515

1616
class RequestContextV2AuthorizerIam(BaseModel):
17-
accessKey: Optional[str]
18-
accountId: Optional[str]
19-
callerId: Optional[str]
20-
principalOrgId: Optional[str]
21-
userArn: Optional[str]
22-
userId: Optional[str]
23-
cognitoIdentity: Optional[RequestContextV2AuthorizerIamCognito]
17+
accessKey: Optional[str] = None
18+
accountId: Optional[str] = None
19+
callerId: Optional[str] = None
20+
principalOrgId: Optional[str] = None
21+
userArn: Optional[str] = None
22+
userId: Optional[str] = None
23+
cognitoIdentity: Optional[RequestContextV2AuthorizerIamCognito] = None
2424

2525

2626
class RequestContextV2AuthorizerJwt(BaseModel):
@@ -29,8 +29,8 @@ class RequestContextV2AuthorizerJwt(BaseModel):
2929

3030

3131
class RequestContextV2Authorizer(BaseModel):
32-
jwt: Optional[RequestContextV2AuthorizerJwt]
33-
iam: Optional[RequestContextV2AuthorizerIam]
32+
jwt: Optional[RequestContextV2AuthorizerJwt] = None
33+
iam: Optional[RequestContextV2AuthorizerIam] = None
3434
lambda_value: Optional[Dict[str, Any]] = Field(None, alias="lambda")
3535

3636

@@ -45,7 +45,7 @@ class RequestContextV2Http(BaseModel):
4545
class RequestContextV2(BaseModel):
4646
accountId: str
4747
apiId: str
48-
authorizer: Optional[RequestContextV2Authorizer]
48+
authorizer: Optional[RequestContextV2Authorizer] = None
4949
domainName: str
5050
domainPrefix: str
5151
requestId: str
@@ -61,11 +61,11 @@ class APIGatewayProxyEventV2Model(BaseModel):
6161
routeKey: str
6262
rawPath: str
6363
rawQueryString: str
64-
cookies: Optional[List[str]]
64+
cookies: Optional[List[str]] = None
6565
headers: Dict[str, str]
66-
queryStringParameters: Optional[Dict[str, str]]
67-
pathParameters: Optional[Dict[str, str]]
68-
stageVariables: Optional[Dict[str, str]]
66+
queryStringParameters: Optional[Dict[str, str]] = None
67+
pathParameters: Optional[Dict[str, str]] = None
68+
stageVariables: Optional[Dict[str, str]] = None
6969
requestContext: RequestContextV2
70-
body: Optional[Union[str, Type[BaseModel]]]
70+
body: Optional[Union[str, Type[BaseModel]]] = None
7171
isBase64Encoded: bool

‎aws_lambda_powertools/utilities/parser/models/dynamodb.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,10 @@
77

88

99
class DynamoDBStreamChangedRecordModel(BaseModel):
10-
ApproximateCreationDateTime: Optional[date]
10+
ApproximateCreationDateTime: Optional[date] = None
1111
Keys: Dict[str, Dict[str, Any]]
12-
NewImage: Optional[Union[Dict[str, Any], Type[BaseModel], BaseModel]]
13-
OldImage: Optional[Union[Dict[str, Any], Type[BaseModel], BaseModel]]
12+
NewImage: Optional[Union[Dict[str, Any], Type[BaseModel], BaseModel]] = None
13+
OldImage: Optional[Union[Dict[str, Any], Type[BaseModel], BaseModel]] = None
1414
SequenceNumber: str
1515
SizeBytes: int
1616
StreamViewType: Literal["NEW_AND_OLD_IMAGES", "KEYS_ONLY", "NEW_IMAGE", "OLD_IMAGE"]
@@ -40,7 +40,7 @@ class DynamoDBStreamRecordModel(BaseModel):
4040
awsRegion: str
4141
eventSourceARN: str
4242
dynamodb: DynamoDBStreamChangedRecordModel
43-
userIdentity: Optional[UserIdentity]
43+
userIdentity: Optional[UserIdentity] = None
4444

4545

4646
class DynamoDBStreamModel(BaseModel):

‎aws_lambda_powertools/utilities/parser/models/kafka.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ class KafkaRecordModel(BaseModel):
1919
value: Union[str, Type[BaseModel]]
2020
headers: List[Dict[str, bytes]]
2121

22-
# validators
23-
_decode_key = validator("key", allow_reuse=True)(base64_decode)
22+
# Added type ignore to keep compatibility between Pydantic v1 and v2
23+
_decode_key = validator("key", allow_reuse=True)(base64_decode) # type: ignore[type-var, unused-ignore]
2424

2525
@validator("value", pre=True, allow_reuse=True)
2626
def data_base64_decode(cls, value):

‎aws_lambda_powertools/utilities/parser/models/kinesis_firehose.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ class KinesisFirehoseRecord(BaseModel):
1717
data: Union[bytes, Type[BaseModel]] # base64 encoded str is parsed into bytes
1818
recordId: str
1919
approximateArrivalTimestamp: PositiveInt
20-
kinesisRecordMetadata: Optional[KinesisFirehoseRecordMetadata]
20+
kinesisRecordMetadata: Optional[KinesisFirehoseRecordMetadata] = None
2121

2222
@validator("data", pre=True, allow_reuse=True)
2323
def data_base64_decode(cls, value):
@@ -28,5 +28,5 @@ class KinesisFirehoseModel(BaseModel):
2828
invocationId: str
2929
deliveryStreamArn: str
3030
region: str
31-
sourceKinesisStreamArn: Optional[str]
31+
sourceKinesisStreamArn: Optional[str] = None
3232
records: List[KinesisFirehoseRecord]

0 commit comments

Comments
 (0)