Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add FastAPI Redis Caching #4195

Closed
wants to merge 20 commits into from
Closed
Changes from 1 commit
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
1849fd3
feat: get logins working with drill
ThomasLaPiana Sep 29, 2023
1d60191
feat: add more endpoints
ThomasLaPiana Sep 29, 2023
5229381
feat: separate each endpoint into its own task
ThomasLaPiana Sep 29, 2023
220f060
feat: optimize the Drill test
ThomasLaPiana Sep 29, 2023
eb0d7e7
feat: make the privacy-experience endpoint async
ThomasLaPiana Sep 29, 2023
800a944
feat: sprinkle some `async sleep` magic into the privacy-experience e…
ThomasLaPiana Sep 29, 2023
abe119c
feat: remove an unused requirement
ThomasLaPiana Sep 29, 2023
2ae8afb
fix: static checks
ThomasLaPiana Sep 29, 2023
1f1f12b
Add FastAPI Redis Caching
ThomasLaPiana Sep 29, 2023
6b33377
Merge branch 'add-more-drill-endpoints' into ThomasLaPiana-add-fastap…
ThomasLaPiana Sep 30, 2023
ffe5a7e
Merge branch 'main' into ThomasLaPiana-add-fastapi-redis-caching
ThomasLaPiana Oct 11, 2023
1347246
fix: remove outdated redis import
ThomasLaPiana Oct 11, 2023
161e271
feat: add caching to the privacy-experience endpoint
ThomasLaPiana Oct 11, 2023
38f32bc
feat: rough POC of custom caching for the privacy experience endpoint
ThomasLaPiana Oct 12, 2023
4d50fd9
feat: additional cleanup
ThomasLaPiana Oct 12, 2023
fdf1d95
Revert "feat: additional cleanup"
ThomasLaPiana Oct 12, 2023
cdb6e49
feat: cleanup, passing more tests
ThomasLaPiana Oct 12, 2023
be70d0c
feat: add headers to the privacy-experience response object indicatin…
ThomasLaPiana Oct 13, 2023
2b64527
checkin: add headers and cache tests, but not passing
ThomasLaPiana Oct 13, 2023
a8a45a2
fix: more tests and almost all static checks
ThomasLaPiana Oct 13, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Revert "feat: additional cleanup"
This reverts commit 4d50fd9.
ThomasLaPiana committed Oct 12, 2023
commit fdf1d9568a0fa0d95e3084487d8efbfde6269095
1 change: 1 addition & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -11,6 +11,7 @@ deepdiff==6.3.0
defusedxml==0.7.1
expandvars==0.9.0
fastapi[all]==0.89.1
fastapi-cache2[redis]==0.2.1
fastapi-pagination[sqlalchemy]==0.11.4
fideslang==2.1.0
fideslog==1.2.10
18 changes: 8 additions & 10 deletions src/fides/api/api/v1/endpoints/privacy_experience_endpoints.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import asyncio
import uuid
from html import escape, unescape
from typing import Dict, List, Optional
from typing import Dict, List, Optional, Union

from fastapi import Depends, HTTPException
from fastapi import Query as FastAPIQuery
@@ -16,6 +16,7 @@
HTTP_404_NOT_FOUND,
HTTP_422_UNPROCESSABLE_ENTITY,
)
from functools import lru_cache


from fides.api.api import deps
@@ -48,6 +49,7 @@
router = APIRouter(tags=["Privacy Experience"], prefix=urls.V1_URL_PREFIX)


@lru_cache(maxsize=20, typed=True)
def get_privacy_experience_or_error(
db: Session, experience_id: str
) -> PrivacyExperience:
@@ -65,6 +67,7 @@ def get_privacy_experience_or_error(
return privacy_experience


@lru_cache(maxsize=20, typed=True)
def _filter_experiences_by_region_or_country(
db: Session, region: Optional[str], experience_query: Query
) -> Query:
@@ -120,19 +123,16 @@ def _filter_experiences_by_region_or_country(
return db.query(PrivacyExperience).filter(False)


EPIC_CACHE: Dict[str, AbstractPage[PrivacyExperience]] = {}
EPIC_CACHE: Dict[str, List] = {}


@router.get(
urls.PRIVACY_EXPERIENCE,
status_code=HTTP_200_OK,
response_model=Page[PrivacyExperienceResponse],
)
@fides_limiter.limit(CONFIG.security.public_request_rate_limit)
async def privacy_experience_list(
*,
db: Session = Depends(deps.get_db),
params: Params = Depends(),
show_disabled: Optional[bool] = True,
region: Optional[str] = None,
component: Optional[ComponentType] = None,
@@ -144,7 +144,7 @@ async def privacy_experience_list(
include_meta: Optional[bool] = False,
request: Request, # required for rate limiting
response: Response, # required for rate limiting
) -> AbstractPage[PrivacyExperience]:
) -> List:
"""
Public endpoint that returns a list of PrivacyExperience records for individual regions with
relevant privacy notices or tcf contents embedded in the response.
@@ -153,7 +153,6 @@ async def privacy_experience_list(
notices as well.
:param db:
:param params: Special case used for Pagination
:param show_disabled: If False, returns only enabled Experiences and Notices
:param region: Return the Experiences for the given region
:param component: Returns Experiences of the given component type
@@ -281,10 +280,9 @@ async def privacy_experience_list(
)

results.append(privacy_experience)
EPIC_CACHE[cache_hash] = results

paginated_results = fastapi_paginate(results, params=params)
EPIC_CACHE[cache_hash] = paginated_results
return paginated_results
return results


def embed_experience_details(
24 changes: 24 additions & 0 deletions src/fides/api/app_setup.py
Original file line number Diff line number Diff line change
@@ -14,6 +14,10 @@
from slowapi.extension import _rate_limit_exceeded_handler # type: ignore
from slowapi.middleware import SlowAPIMiddleware # type: ignore
from starlette.middleware.cors import CORSMiddleware
from fastapi_cache import FastAPICache
from fastapi_cache.backends.redis import RedisBackend

from redis import asyncio as aioredis

import fides
from fides.api.api.deps import get_api_session
@@ -75,6 +79,26 @@
)


async def init_fastapi_redis_cache():
"""Initialize a Redis-backed cache to store API responses."""
redis_url = "{}://{}:{}@{}:{}/{}".format(
"rediss" if CONFIG.redis.ssl else "redis",
CONFIG.redis.user,
CONFIG.redis.password,
CONFIG.redis.host,
CONFIG.redis.port,
CONFIG.redis.db_index,
)
redis = aioredis.from_url(
url=redis_url,
charset=CONFIG.redis.charset,
decode_responses=CONFIG.redis.decode_responses,
ssl_ca_certs=CONFIG.redis.ssl_ca_certs,
ssl_cert_reqs=CONFIG.redis.ssl_cert_reqs,
)
FastAPICache.init(RedisBackend(redis), prefix="fastapi-cache")


def create_fides_app(
cors_origins: List[AnyUrl] = CONFIG.security.cors_origins,
cors_origin_regex: Optional[Pattern] = CONFIG.security.cors_origin_regex,
2 changes: 2 additions & 0 deletions src/fides/api/main.py
Original file line number Diff line number Diff line change
@@ -24,6 +24,7 @@
create_fides_app,
log_startup,
run_database_startup,
init_fastapi_redis_cache,
)
from fides.api.common_exceptions import MalisciousUrlException
from fides.api.middleware import handle_audit_log_resource
@@ -260,6 +261,7 @@ async def setup_server() -> None:
await run_database_startup(app)

check_redis()
await init_fastapi_redis_cache()

if not scheduler.running:
scheduler.start()