Skip to content

Commit

Permalink
Merge pull request #1288 from GSA/snyk-fix
Browse files Browse the repository at this point in the history 
update snyk expirations and remove outdated
  • Loading branch information
@btylerburton
btylerburton authored Apr 2, 2024
2 parents 59655be + efc6635 commit abbdf87
Show file tree
Hide file tree
Showing 5 changed files with 48 additions and 47 deletions.
54 changes: 27 additions & 27 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,42 +9,42 @@ all: build
# ###############################################

build:
docker-compose build --parallel
docker compose build --parallel

ci:
docker-compose up -d
docker compose up -d

clean:
docker-compose down -v --remove-orphans
docker compose down -v --remove-orphans

cypress:
# Turn on local system, and open cypress in interactive mode
# If you haven't remapped localhost > ckan, you should change baseURL to "http://localhost:5000" in `e2e/cypress.json`
docker-compose up -d && cd e2e && CYPRESS_USER=admin CYPRESS_USER_PASSWORD=password npx [email protected] open
docker compose up -d && cd e2e && CYPRESS_USER=admin CYPRESS_USER_PASSWORD=password npx [email protected] open

dev:
docker build -t ghcr.io/gsa/catalog.data.gov:latest ckan/
docker-compose build
docker-compose up
docker compose build
docker compose up

debug:
docker build -t ghcr.io/gsa/catalog.data.gov:latest ckan/
docker-compose build
docker-compose run --service-ports ckan
docker compose build
docker compose run --service-ports ckan

up:
docker-compose up $(ARGS)
docker compose up $(ARGS)

update-dependencies:
docker-compose run --rm -T ckan /app/ckan/freeze-requirements.sh $(shell id -u) $(shell id -g)
docker compose run --rm -T ckan /app/ckan/freeze-requirements.sh $(shell id -u) $(shell id -g)

# ###############################################
# Test commands
# ###############################################

test: build
# docker build -t ghcr.io/gsa/catalog.data.gov:latest ckan/
docker-compose -f docker-compose.yml -f docker-compose.test.yml up --abort-on-container-exit test
docker compose -f docker-compose.yml -f docker-compose.test.yml up --abort-on-container-exit test

# everytime you added some new variables, you need to swap it with some test values
# and swap it back after the test. This is because "nginx -t" test cannot read env variables.
Expand Down Expand Up @@ -79,31 +79,31 @@ validate-proxy:

quick-bat-test:
# if local environment is already build and running
docker-compose -f docker-compose.yml -f docker-compose.test.yml up --abort-on-container-exit test
docker compose -f docker-compose.yml -f docker-compose.test.yml up --abort-on-container-exit test

test-extensions:
# test our extensions

# deal with the CKAN path
docker-compose exec ckan bash -c "ln -sf $(CKAN_HOME)/src/ckan $(CKAN_HOME)/ckan"
docker compose exec ckan bash -c "ln -sf $(CKAN_HOME)/src/ckan $(CKAN_HOME)/ckan"

# full test datajson
docker-compose exec ckan bash -c \
docker compose exec ckan bash -c \
"cd $(CKAN_HOME)/src/ckanext-datajson && \
nosetests --ckan --with-pylons=$(CKAN_HOME)/src/ckan/test-catalog-next.ini ckanext/datajson/tests --debug=ckanext"

# full test datagovtheme
docker-compose exec ckan bash -c \
docker compose exec ckan bash -c \
"cd $(CKAN_HOME)/src/ckanext-datagovtheme && \
nosetests --ckan --with-pylons=$(CKAN_HOME)/src/ckan/test-catalog-next.ini ckanext/datagovtheme/tests --debug=ckanext"

# full test geodatagov
docker-compose exec ckan bash -c \
docker compose exec ckan bash -c \
"cd $(CKAN_HOME)/src/ckanext-geodatagov && \
nosetests --ckan --with-pylons=$(CKAN_HOME)/src/ckan/test-catalog-next.ini ckanext/geodatagov/tests --debug=ckanext"

# full test geodatagov
docker-compose exec ckan bash -c \
docker compose exec ckan bash -c \
"cd $(CKAN_HOME)/src/ckanext-datagovdatalog && \
nosetests --ckan --with-pylons=$(CKAN_HOME)/src/ckan/test-catalog-next.ini ckanext/datagovdatalog/tests --debug=ckanext"

Expand All @@ -119,10 +119,10 @@ clear-solr-volume:

unlock-solr-volume:
# Corruptible
docker-compose run solr /bin/bash -c "rm -rf /var/solr/data/ckan/data/index/write.lock"
docker compose run solr /bin/bash -c "rm -rf /var/solr/data/ckan/data/index/write.lock"

search-index-rebuild:
docker-compose exec ckan /bin/bash -c "ckan search-index rebuild"
docker compose exec ckan /bin/bash -c "ckan search-index rebuild"

copy-src:
docker cp catalog-app_ckan_1:$(CKAN_HOME)/src .
Expand All @@ -136,7 +136,7 @@ test-import-tool:
python -m pytest --vcr-record=none tests/

lint-all:
docker-compose exec -T ckan \
docker compose exec -T ckan \
bash -c "cd $(CKAN_HOME)/src && \
pip install pip==20.3.3 && \
pip install flake8 && \
Expand All @@ -146,31 +146,31 @@ lint-all:
# qa:
# ifeq (${PARAMS}, all)
# # PARAMS=all make qa
# docker-compose exec ckan ckan report generate
# docker compose exec ckan ckan report generate
# else ifeq (${PARAMS}, openness)
# # PARAMS=openness make qa
# # generate report at /report/openness
# docker-compose exec ckan ckan report generate openness
# docker compose exec ckan ckan report generate openness
# else ifeq (${PARAMS}, update)
# # PARAMS=update make qa
# # QA is performed when a dataset/resource is archived, or you can run it manually using a ckan command:
# docker-compose exec ckan ckan qa update
# docker compose exec ckan ckan qa update
# else ifeq (${PARAMS}, archive)
# # PARAMS=archive make qa
# # Archive datasets to perform QA
# docker-compose exec ckan ckan archiver update --queue bulk
# docker compose exec ckan ckan archiver update --queue bulk
# else ifeq (${PARAMS}, worker)
# # PARAMS=worker make qa
# docker-compose exec ckan ckan jobs worker bulk
# docker compose exec ckan ckan jobs worker bulk
# endif

update-tracking-info:
# https://docs.ckan.org/en/2.8/maintaining/tracking.html
docker-compose exec ckan ckan tracking update
docker compose exec ckan ckan tracking update

harvest:
# Pass any of the following arguments to run them
# ARGS=run make harvest
# ARGS=gather-consumer make harvest
# ARGS=fetch-consumer make harvest
docker-compose exec ckan ckan harvester $(ARGS)
docker compose exec ckan ckan harvester $(ARGS)
6 changes: 3 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ To test extensions locally you can run
_TODO: update this for pytest_

```
docker-compose exec ckan bash
docker compose exec ckan bash
nosetests --ckan --with-pylons=src/ckan/test-catalog-next.ini src/ckanext-datagovtheme/ckanext/datagovtheme/
nosetests --ckan --with-pylons=src/ckan/test-catalog-next.ini src/ckanext-datagovtheme/ckanext/datajson/
nosetests --ckan --with-pylons=src/ckan/test-catalog-next.ini src/ckanext-datagovtheme/ckanext/geodatagov/
Expand Down Expand Up @@ -207,7 +207,7 @@ You can use the ckan template in much the same way as a source install, only
executing the command inside the CKAN container and setting the mounted `src/`
folder as output:

$ docker-compose exec ckan /bin/bash -c \
$ docker compose exec ckan /bin/bash -c \
"ckan generate extension"

The new extension will be created in the `src/` folder. You might need to change
Expand All @@ -219,7 +219,7 @@ the owner of its folder to have the appropriate permissions.
To run a container and be able to add a breakpoint with `pdb` or `ipdb`, run the
`ckan-dev` container with the `--service-ports` option:

docker-compose run --service-ports ckan
docker compose run --service-ports ckan

This will start a new container, displaying the standard output in your
terminal. If you add a breakpoint in a source file in the `src` folder (`import
Expand Down
31 changes: 16 additions & 15 deletions ckan/.snyk
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,51 +3,52 @@ version: v1.25.0
# ignores vulnerabilities until expiry date; change duration by modifying expiry date
ignore:
SNYK-PYTHON-BEAKER-575115:
- '*':
- "*":
reason: >-
No remediation available yet; Not affecting us since the storage is
not accessible to any other client
expires: 2024-03-31T16:20:58.017Z
expires: 2024-05-31T16:20:58.017Z
created: 2022-12-08T16:20:58.023Z
SNYK-PYTHON-WERKZEUG-6035177:
- '*':
- "*":
reason: >-
Upgrade path is complex, Issue tracked in github:
https://github.com/GSA/data.gov/issues/4217
expires: 2024-03-31T16:20:58.017Z
expires: 2024-05-31T16:20:58.017Z
created: 2023-10-30T16:50:58.023Z
SNYK-PYTHON-WERKZEUG-3319936:
- '*':
- "*":
reason: >-
Upgrade path is complex, Issue tracked in github:
https://github.com/GSA/data.gov/issues/4217
expires: 2024-03-31T16:20:58.017Z
expires: 2024-05-31T16:20:58.017Z
created: 2023-02-15T16:20:58.023Z
SNYK-PYTHON-WERKZEUG-3319935:
- '*':
- "*":
reason: >-
Upgrade path is complex, Issue tracked in github:
https://github.com/GSA/data.gov/issues/4217
expires: 2024-03-31T16:20:58.017Z
expires: 2024-05-31T16:20:58.017Z
created: 2023-02-15T16:20:58.023Z
SNYK-PYTHON-FLASK-5490129:
- '*':
- "*":
reason: >-
Upgrade path is complex, Issue tracked in github:
https://github.com/GSA/data.gov/issues/4303
expires: 2024-03-31T16:20:58.017Z
expires: 2024-05-31T16:20:58.017Z
created: 2023-05-08T16:20:58.023Z
SNYK-PYTHON-PYOPENSSL-6149520:
- '*':
- "*":
reason: >-
No remediation available yet; Issue tracked in github:
https://github.com/GSA/data.gov/issues/4532
expires: 2024-04-10T19:29:54.032Z
created: 2024-01-11T19:29:54.039Z
expires: 2024-05-31T19:29:54.032Z
created: 2024-01-08T00:00:00.000Z
SNYK-PYTHON-PYOPENSSL-6157250:
- '*':
- "*":
reason: >-
No remediation available yet; Issue tracked in github:
https://github.com/GSA/data.gov/issues/4591
expires: 2024-04-10T19:29:54.032Z
expires: 2024-05-31T19:29:54.032Z
created: 2024-01-14T00:00:00.000Z
patch: {}
2 changes: 1 addition & 1 deletion src/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@ This folder is used to install local extensions for development.
Clone here any extension that you want to work on. This folder in mounted on
the `ckan-dev` service, so any extension located here will be installed when
booting up Docker Compose in development mode
(`docker-compose -f docker-compose.dev.yml up`).
(`docker compose -f docker-compose.dev.yml up`).
If running the CKAN container standalone you will need to mount the folder
yourself.
2 changes: 1 addition & 1 deletion tools/harvest_source_import/check_harvest_sources.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@
f'Config: {config}'
print(info)

command = (['docker-compose', 'exec', 'ckan',
command = (['docker compose', 'exec', 'ckan',
'ckan', 'harvester', 'run_test',
hs.get("id"), '--config=/app/ckan/setup/ckan.ini'])
out = subprocess.Popen(
Expand Down

0 comments on commit abbdf87

Please sign in to comment.