Skip to content

Commit

Permalink
use elastic in wire expiry (#1093)
Browse files Browse the repository at this point in the history 
there were timeouts from mongo

CPCN-849
  • Loading branch information
@petrjasek
petrjasek authored Sep 25, 2024
1 parent eedc299 commit 378cc5b
Show file tree
Hide file tree
Showing 6 changed files with 109 additions and 64 deletions.
108 changes: 59 additions & 49 deletions dev-requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ amqp==5.2.0
# via kombu
arabic-reshaper==3.0.0
# via xhtml2pdf
arrow==0.13.0
arrow==1.3.0
# via
# eve-elastic
# superdesk-core
Expand All @@ -21,49 +21,49 @@ async-timeout==4.0.3
# via redis
authlib==0.14.3
# via superdesk-core
babel==2.15.0
babel==2.16.0
# via flask-babel
bcrypt==4.1.3
# via superdesk-core
behave==1.2.6
# via wooper
billiard==3.6.4.0
billiard==4.2.1
# via celery
black==23.12.1
# via -r black-requirements.txt
blinker==1.7.0
blinker==1.8.2
# via
# elastic-apm
# flask
# flask-mail
# raven
# sentry-sdk
# superdesk-core
boto3==1.34.154
boto3==1.35.25
# via superdesk-core
botocore==1.34.154
botocore==1.35.25
# via
# boto3
# s3transfer
cachelib==0.9.0
# via flask-caching
cachetools==5.4.0
cachetools==5.5.0
# via
# flask-oidc-ex
# google-auth
celery[redis]==5.2.7
celery[redis]==5.4.0
# via superdesk-core
cerberus==1.3.5
# via
# eve
# superdesk-core
certifi==2024.7.4
certifi==2024.8.30
# via
# elastic-apm
# elasticsearch
# requests
# sentry-sdk
cffi==1.16.0
cffi==1.17.1
# via cryptography
chardet==5.2.0
# via
Expand Down Expand Up @@ -92,23 +92,23 @@ click-repl==0.3.0
# via celery
coverage[toml]==7.6.1
# via pytest-cov
croniter==2.0.7
croniter==3.0.3
# via superdesk-core
cryptography==43.0.0
cryptography==43.0.1
# via
# authlib
# jwcrypto
# pyhanko
# pyhanko-certvalidator
cssselect2==0.7.0
# via svglib
deepdiff==7.0.1
deepdiff==8.0.1
# via superdesk-planning
deprecated==1.2.14
# via limits
dnspython==2.6.1
# via email-validator
draftjs-exporter[lxml]==2.1.7
draftjs-exporter[lxml]==2.1.0
# via superdesk-core
ecs-logging==2.2.0
# via elastic-apm
Expand Down Expand Up @@ -158,9 +158,9 @@ flask-caching==2.3.0
# via -r requirements.txt
flask-limiter==0.9.5.1
# via -r requirements.txt
flask-mail==0.9.1
flask-mail==0.10.0
# via superdesk-core
flask-oidc-ex==0.5.5
flask-oidc-ex==0.6.2
# via superdesk-core
flask-pymongo==2.3.0
# via -r requirements.txt
Expand All @@ -176,9 +176,9 @@ google-auth==2.29.0
# via -r requirements.txt
gunicorn==22.0.0
# via -r requirements.txt
hachoir==3.0a3
hachoir==3.3.0
# via superdesk-core
hermescache==0.10.0
hermescache==1.0.0
# via superdesk-core
honcho==1.1.0
# via -r requirements.txt
Expand All @@ -190,11 +190,11 @@ httplib2==0.22.0
# via oauth2client
icalendar==4.0.9
# via superdesk-planning
idna==3.7
idna==3.10
# via
# email-validator
# requests
importlib-resources==6.4.0
importlib-resources==6.4.5
# via limits
iniconfig==2.0.0
# via pytest
Expand All @@ -219,21 +219,24 @@ jwcrypto==1.5.6
# via
# flask-oidc-ex
# python-jwt
kombu==5.2.4
kombu==5.4.2
# via
# celery
# superdesk-core
ldap3==2.5.2
ldap3==2.9.1
# via superdesk-core
limits==3.13.0
# via flask-limiter
lxml==4.6.5
lxml==5.2.2
# via
# draftjs-exporter
# lxml-html-clean
# python3-saml
# superdesk-core
# svglib
# xmlsec
lxml-html-clean==0.2.2
# via superdesk-core
markupsafe==2.0.1
# via
# -r requirements.txt
Expand All @@ -245,7 +248,7 @@ mccabe==0.7.0
# via flake8
mongolock @ git+https://github.com/superdesk/mongolock.git@v1
# via superdesk-core
mypy==1.11.1
mypy==1.11.2
# via -r mypy-requirements.txt
mypy-extensions==1.0.0
# via
Expand All @@ -255,7 +258,7 @@ oauth2client==4.1.3
# via flask-oidc-ex
oauthlib==3.2.2
# via requests-oauthlib
ordered-set==4.1.0
orderly-set==5.2.2
# via deepdiff
oscrypto==1.3.0
# via pyhanko-certvalidator
Expand All @@ -269,28 +272,28 @@ parse==1.20.2
# via
# behave
# parse-type
parse-type==0.6.2
parse-type==0.6.3
# via behave
pathspec==0.12.1
# via black
pillow==9.2.0
pillow==10.4.0
# via
# reportlab
# superdesk-core
# xhtml2pdf
platformdirs==4.2.2
platformdirs==4.3.6
# via black
pluggy==1.5.0
# via pytest
prompt-toolkit==3.0.47
# via click-repl
pyasn1==0.6.0
pyasn1==0.6.1
# via
# ldap3
# oauth2client
# pyasn1-modules
# rsa
pyasn1-modules==0.4.0
pyasn1-modules==0.4.1
# via
# google-auth
# oauth2client
Expand All @@ -306,7 +309,7 @@ pyhanko-certvalidator==0.22.0
# via
# pyhanko
# xhtml2pdf
pyjwt==2.4.0
pyjwt==2.8.0
# via superdesk-core
pymemcache==4.0.0
# via superdesk-core
Expand All @@ -316,17 +319,17 @@ pymongo==3.11.4
# flask-pymongo
# mongolock
# superdesk-core
pyparsing==3.1.2
pyparsing==3.1.4
# via
# httplib2
# pyrtf3
pypdf==4.3.1
pypdf==5.0.0
# via xhtml2pdf
pypng==0.20220715.0
# via qrcode
pyrtf3==0.47.5
# via -r requirements.txt
pytest==8.3.2
pytest==8.3.3
# via
# -r dev-requirements.in
# pytest-cov
Expand All @@ -343,6 +346,7 @@ python-dateutil==2.9.0.post0
# via
# arrow
# botocore
# celery
# croniter
# icalendar
# superdesk-core
Expand All @@ -354,17 +358,16 @@ python-twitter==3.5
# via superdesk-core
python3-saml==1.16.0
# via -r requirements.txt
pytz==2024.1
pytz==2024.2
# via
# celery
# croniter
# eve-elastic
# flask-babel
# icalendar
# pyhanko
# superdesk-core
# tzlocal
pyyaml==6.0.1
pyyaml==6.0.2
# via
# pyhanko
# responses
Expand All @@ -377,9 +380,9 @@ redis==5.0.8
# via
# celery
# superdesk-core
regex==2020.7.14
regex==2024.7.24
# via superdesk-core
reportlab==3.6.13
reportlab==4.2.2
# via
# -r requirements.txt
# superdesk-core
Expand All @@ -405,11 +408,11 @@ rsa==4.9
# oauth2client
s3transfer==0.10.2
# via boto3
sentry-sdk[flask]==2.12.0
sentry-sdk[flask]==2.13.0
# via -r requirements.txt
sgmllib3k==1.0.0
# via feedparser
simplejson==3.19.2
simplejson==3.19.3
# via eve
six==1.16.0
# via
Expand All @@ -422,7 +425,7 @@ six==1.16.0
# parse-type
# python-bidi
# python-dateutil
superdesk-core @ git+https://github.com/superdesk/superdesk-core.git@v2.7.3
superdesk-core @ git+https://github.com/superdesk/superdesk-core.git@v2.8.0rc2
# via -r requirements.txt
superdesk-planning @ git+https://github.com/superdesk/[email protected]
# via -r requirements.txt
Expand Down Expand Up @@ -450,11 +453,13 @@ types-jinja2==2.11.9
# types-flask
types-markupsafe==1.1.10
# via types-jinja2
types-protobuf==5.27.0.20240626
types-protobuf==5.28.0.20240924
# via -r mypy-requirements.txt
types-python-dateutil==2.9.0.20240316
# via -r mypy-requirements.txt
types-pytz==2024.1.0.20240417
types-python-dateutil==2.9.0.20240906
# via
# -r mypy-requirements.txt
# arrow
types-pytz==2024.2.0.20240913
# via
# -r mypy-requirements.txt
# types-tzlocal
Expand All @@ -478,6 +483,10 @@ typing-extensions==4.12.2
# pypdf
# qrcode
# superdesk-core
tzdata==2024.2
# via
# celery
# kombu
tzlocal==2.1
# via
# pyhanko
Expand All @@ -486,8 +495,9 @@ unidecode==1.3.8
# via superdesk-core
uritools==4.0.3
# via pyhanko-certvalidator
urllib3==1.26.19
urllib3==1.26.20
# via
# -r requirements.txt
# botocore
# elastic-apm
# elasticsearch
Expand Down Expand Up @@ -524,9 +534,9 @@ wtforms[email]==3.1.2
# -r requirements.txt
# flask-wtf
# wtforms
xhtml2pdf==0.2.11
xhtml2pdf==0.2.16
# via -r requirements.txt
xmlsec==1.3.13
xmlsec==1.3.14
# via
# python3-saml
# superdesk-core
Expand Down
1 change: 1 addition & 0 deletions newsroom/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ def query_resource(
req.max_results = max_results
req.sort = sort
req.projection = json.dumps(projection) if projection else None
req.args = {}
cursor, count = app.data.find(resource, req, lookup, perform_count=False)
return cursor

Expand Down
5 changes: 3 additions & 2 deletions newsroom/web/default_settings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -471,8 +471,9 @@
},
}

MAX_EXPIRY_QUERY_LIMIT = os.environ.get("MAX_EXPIRY_QUERY_LIMIT", 100)
CONTENT_API_EXPIRY_DAYS = os.environ.get("CONTENT_API_EXPIRY_DAYS", 180)
MAX_EXPIRY_QUERY_LIMIT = int(os.environ.get("MAX_EXPIRY_QUERY_LIMIT", 100))
CONTENT_API_EXPIRY_DAYS = int(os.environ.get("CONTENT_API_EXPIRY_DAYS", 180))
CONTENT_API_EXPIRY_QUERY_LIMIT = int(os.environ.get("CONTENT_API_EXPIRY_QUERY_LIMIT", 1000))

NEWS_API_ENABLED = strtobool(env("NEWS_API_ENABLED", "false"))

Expand Down
Loading

0 comments on commit 378cc5b

Please sign in to comment.