Merge branch 'main' into bad-bots

GSA · Feb 12, 2024 · 3ba013b · 3ba013b
2 parents 7a31a2d + d104589
commit 3ba013b
Show file tree

Hide file tree

Showing 28 changed files with 52,784 additions and 12 deletions.
diff --git a/.github/workflows/egress_check.yml b/.github/workflows/egress_check.yml
@@ -3,7 +3,7 @@ name: 6 - Check Egress Operation
 
 on:
   schedule:
-    - cron: '40 * * * *'
+    - cron: '40 02,14 * * *'
   workflow_dispatch:
 
 jobs:

diff --git a/.github/workflows/scale_web.yml b/.github/workflows/scale_web.yml
@@ -0,0 +1,16 @@
+---
+name: 7 - Scale catalog-web
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '*/5 * * * *'
+
+jobs:
+  scale-prod-web:
+    name: scale prod catalog-web
+    uses: gsa/data.gov/.github/workflows/scale-web-template.yml@main
+    with:
+      environ: prod
+      app_names: "{\"include\":[{\"app\":\"catalog-web\"},]}"
+    secrets: inherit
diff --git a/Makefile b/Makefile
@@ -59,6 +59,7 @@ validate-proxy:
 	sed -i 's/{{env "PUBLIC_ROUTE"}}/test.com/g' proxy/nginx-cloudfront.conf proxy/nginx-authy.conf
 	sed -i 's#{{env "S3_URL"}}#http://test.com#g' proxy/nginx-common.conf
 	sed -i 's#{{env "S3_BUCKET"}}#somebucket#g' proxy/nginx-common.conf
+	sed -i 's#{{env "DENY_PACKAGE_CREATE"}}#truetodeny#g' proxy/nginx-common.conf
 	docker run --rm -e nameservers=127.0.0.1 -v $(shell pwd)/proxy:/proxy nginx nginx -t -c /proxy/nginx.conf
 	sed -i 's/127.0.0.1/{{nameservers}}/g' proxy/nginx.conf
 	sed -i 's/127.0.0.2/{{env "EXTERNAL_ROUTE"}}/g' proxy/nginx.conf proxy/nginx-cloudfront.conf
@@ -67,10 +68,10 @@ validate-proxy:
 	sed -i 's/127.0.0.5/{{env "INTERNAL_ROUTE_ADMIN"}}/g' proxy/nginx.conf
 	sed -i 's/127.0.0.6/{{env "PUBLIC_ROUTE"}}/g' proxy/nginx.conf proxy/nginx-cloudfront.conf
 	sed -i 's/1111/{{port}}/g' proxy/nginx.conf proxy/nginx-common.conf
-	sed -i 's/test.com/{{env "PUBLIC_ROUTE"}}/g' proxy/nginx-cloudfront.conf
+	sed -i 's/test.com/{{env "PUBLIC_ROUTE"}}/g' proxy/nginx-cloudfront.conf proxy/nginx-authy.conf
 	sed -i 's#http://test.com#{{env "S3_URL"}}#g' proxy/nginx-common.conf
 	sed -i 's#somebucket#{{env "S3_BUCKET"}}#g' proxy/nginx-common.conf
-	sed -i 's/test.com/{{env "PUBLIC_ROUTE"}}/g' proxy/nginx-authy.conf
+	sed -i 's/truetodeny/{{env "DENY_PACKAGE_CREATE"}}/g' proxy/nginx-common.conf
 
 quick-bat-test:
 	# if local environment is already build and running

diff --git a/ckan/requirements.in b/ckan/requirements.in
@@ -123,5 +123,5 @@ requests==2.31.0
 # avoid ImportError error https://github.com/GSA/data.gov/issues/4396
 importlib-resources<6.0
 gevent>=23.9.0
-cryptography>=42
 jinja2>=3.1.3
+cryptography>=42.0.2
diff --git a/ckan/requirements.txt b/ckan/requirements.txt
@@ -4,16 +4,16 @@ Babel==2.10.3
 Beaker==1.11.0
 bleach==3.3.0
 blinker==1.5
-boto3==1.34.29
-botocore==1.34.29
+boto3==1.34.32
+botocore==1.34.32
 certifi==2023.11.17
 cffi==1.16.0
 chardet==5.2.0
 charset-normalizer==3.3.2
 ckan==2.10.1
 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver
 ckanext-datagovcatalog==0.1.0
-ckanext-datagovtheme==0.2.12
+ckanext-datagovtheme==0.2.13
 ckanext-datajson==0.1.23
 ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@1109205069dd105dda27e3486898e4ca1525a808
 ckanext-envvars==0.0.3
@@ -26,7 +26,7 @@ ckanext-saml2auth @ git+https://github.com/GSA/ckanext-saml2auth.git@4d59366423e
 -e git+https://github.com/ckan/ckanext-spatial.git@938308469892e4bcf7389cb4adee5ccdd5a0ccca#egg=ckanext_spatial
 ckantoolkit==0.0.7
 click==8.1.3
-cryptography==42.0.1
+cryptography==42.0.2
 defusedxml==0.7.1
 dominate==2.7.0
 elementpath==4.1.5
@@ -57,7 +57,7 @@ jmespath==1.0.1
 json-table-schema==0.2.1
 jsonschema==2.4.0
 lxml==5.1.0
-Mako==1.3.0
+Mako==1.3.2
 Markdown==3.4.1
 MarkupSafe==2.1.4
 messytables==0.15.2

diff --git a/manifest.yml b/manifest.yml
@@ -79,6 +79,7 @@ applications:
       INTERNAL_ROUTE: ((route-internal))
       EXTERNAL_ROUTE_ADMIN: ((route-external-admin))
       INTERNAL_ROUTE_ADMIN: ((route-internal-admin))
+      DENY_PACKAGE_CREATE: ((deny_package_create))
 
   - name: ((app_name))-gather
     buildpacks:

diff --git a/proxy/nginx-common.conf b/proxy/nginx-common.conf
@@ -83,7 +83,11 @@ location = /500.html {
 
 # prevent users from accessing: '/dataset/new' route, 'package_create' and 'resource_create' API routes
 location ~ ^/(dataset\/new|api\/action\/package_create|api\/action\/resource_create)/?$ {
-  deny all;
+  set $deny {{env "DENY_PACKAGE_CREATE"}};
+  if ($deny = 'true') {
+    return 403;
+  }
+  try_files $uri @proxy_to_app;
 }
 
 # use local path for map tiles so that they