diff --git a/.env.tmpl b/.env.tmpl index 9732596f..10d1364e 100644 --- a/.env.tmpl +++ b/.env.tmpl @@ -16,6 +16,5 @@ AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} AWS_BUCKET_NAME=${AWS_BUCKET_NAME} ANALYTICS=${ANALYTICS} BROKER_URL=${BROKER_URL} -TX_USR=${TX_USR} -TX_PWD=${TX_PWD} +TX_TOKEN=${TX_TOKEN} HTPASSWORDS=${HTPASSWORDS} diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ded8e845..fcfd8b4d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,8 +13,7 @@ jobs: env: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - TX_USR: ${{ secrets.TX_USR }} - TX_PWD: ${{ secrets.TX_PWD }} + TX_TOKEN: ${{ secrets.TX_TOKEN }} steps: - uses: actions/checkout@v3 - name: Build diff --git a/.tx/config b/.tx/config index 31be074b..4105fc2d 100644 --- a/.tx/config +++ b/.tx/config @@ -1,16 +1,20 @@ [main] host = https://www.transifex.com -[gfdrr-thinkhazard.ui] -source_file = /tmp/thinkhazard.pot +[o:gfdrr-labs:p:gfdrr-thinkhazard:r:ui] type = PO +source_file = /tmp/thinkhazard.pot source_lang = en -trans.es = thinkhazard/locale/es/LC_MESSAGES/thinkhazard.po -trans.fr = thinkhazard/locale/fr/LC_MESSAGES/thinkhazard.po +file_filter = thinkhazard/locale//LC_MESSAGES/thinkhazard.po -[gfdrr-thinkhazard.database] +[o:gfdrr-labs:p:gfdrr-thinkhazard:r:database] +type = PO source_file = /tmp/thinkhazard-database.pot +source_lang = en +file_filter = /tmp/thinkhazard-database-.po + +[o:gfdrr-labs:p:gfdrr-thinkhazard:r:test--database] type = PO +source_file = /tmp/thinkhazard-database.pot source_lang = en -trans.es = /tmp/thinkhazard-database-es.po -trans.fr = /tmp/thinkhazard-database-fr.po +file_filter = /tmp/thinkhazard-database-.po diff --git a/Dockerfile b/Dockerfile index b107a365..43820f65 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ ENV PYTHONUNBUFFERED 1 RUN apt-get update && apt-get install -y \ # Install postgis for shp2pgsql as ogr2ogr from distrib is not compatible with PostgreSQL 12 postgis \ - git python-numpy gdal-bin libgdal-dev tidy gnupg2 unzip \ + curl git python-numpy gdal-bin libgdal-dev tidy gnupg2 unzip \ # pyppeteer dependencies (cf https://github.com/puppeteer/puppeteer/issues/1345) gconf-service libasound2 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget \ && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ @@ -20,6 +20,11 @@ RUN apt-get update && apt-get install -y \ && apt-get update && apt-get install -y postgresql-client-12 \ && rm -rf /var/lib/apt/lists/* +RUN curl -L -o /tmp/tx-linux-amd64.tar.gz https://github.com/transifex/cli/releases/download/v1.6.7/tx-linux-amd64.tar.gz \ + && tar -C /usr/local/bin -xf /tmp/tx-linux-amd64.tar.gz tx \ + && chmod +x /usr/local/bin/tx \ + && rm -f /tmp/tx-linux-amd64.tar.gz + ENV HOME=/home/user \ NODE_PATH=/opt/thinkhazard/node_modules @@ -91,10 +96,8 @@ RUN pip install --no-cache-dir -r /app/requirements-dev.txt \ WORKDIR /app COPY . /app/ -ARG TX_USR -ARG TX_PWD -RUN TX_USR=$TX_USR \ - TX_PWD=$TX_PWD \ +ARG TX_TOKEN +RUN TX_TOKEN=$TX_TOKEN \ make -f docker.mk build RUN pip install --no-deps -e . diff --git a/Makefile b/Makefile index bbb9f2ec..ec9fa553 100644 --- a/Makefile +++ b/Makefile @@ -95,7 +95,8 @@ compile_catalog: .PHONY: test test: ## Run automated tests - $(DOCKER_CMD) nosetests -v + # $(DOCKER_CMD) nosetests -v + $(DOCKER_CMD) pytest -vv --cov=thinkhazard tests .PHONY: bash test-bash: ## Open bash in a test container @@ -133,15 +134,13 @@ cleanall: clean .PHONY: docker_build_thinkhazard docker_build_thinkhazard: docker build \ - --build-arg TX_USR=${TX_USR} \ - --build-arg TX_PWD=${TX_PWD} \ + --build-arg TX_TOKEN=${TX_TOKEN} \ --target app -t camptocamp/thinkhazard . .PHONY: docker_build_builder docker_build_builder: docker build \ - --build-arg TX_USR=${TX_USR} \ - --build-arg TX_PWD=${TX_PWD} \ + --build-arg TX_TOKEN=${TX_TOKEN} \ --target builder -t camptocamp/thinkhazard-builder . .PHONY: docker_build_testdb diff --git a/docker-compose.yaml b/docker-compose.yaml index a8b47a9e..29389abe 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -25,8 +25,7 @@ services: - hazardsets:/tmp/hazardsets environment: &env - INI_FILE - - TX_USR - - TX_PWD + - TX_TOKEN - PGHOST - PGPORT - PGUSER_PUBLIC diff --git a/docker.mk b/docker.mk index afbd959e..eb3e1f11 100644 --- a/docker.mk +++ b/docker.mk @@ -52,16 +52,15 @@ compile_catalog: \ msgfmt -o $@ $< thinkhazard/locale/%/LC_MESSAGES/thinkhazard.po: $(HOME)/.transifexrc - tx pull -r gfdrr-thinkhazard.ui + tx pull --translations --languages=$* --resources=gfdrr-thinkhazard.ui --force touch `find thinkhazard/locale/ -name '*.po' 2> /dev/null` .INTERMEDIATE: $(HOME)/.transifexrc $(HOME)/.transifexrc: echo "[https://www.transifex.com]" > $@ - echo "hostname = https://www.transifex.com" >> $@ - echo "username = $(TX_USR)" >> $@ - @echo "password = $(TX_PWD)" >> $@ - echo "token =" >> $@ + echo "rest_hostname = https://rest.api.transifex.com" >> $@ + @echo "token = $(TX_TOKEN)" >> $@ + cat $@ check: flake8 jshint diff --git a/requirements-dev.txt b/requirements-dev.txt index 8d31bb75..21abf87e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,4 +7,6 @@ mock nose pep8-naming pydevd +pytest +pytest-cov WebTest diff --git a/requirements.txt b/requirements.txt index 197e3b16..b9869cdd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,19 @@ APScheduler Babel PyPDF2 -SQLAlchemy +SQLAlchemy<2 alembic asyncio boto3 +chameleon # needed by lingua colorlog geoalchemy2 gunicorn httplib2 -jinja2 +jinja2<3.1 lingua markdown +numpy<1.24 # for compatibility with rasterio 1.3.7 papyrus passlib paste @@ -35,7 +37,6 @@ shapely shapely simplejson transaction -transifex-client waitress zope.sqlalchemy celery diff --git a/thinkhazard/scripts/tx-init b/thinkhazard/scripts/tx-init index 08c8203b..42ecec42 100755 --- a/thinkhazard/scripts/tx-init +++ b/thinkhazard/scripts/tx-init @@ -2,8 +2,6 @@ cat < ${HOME}/.transifexrc [https://www.transifex.com] -hostname = https://www.transifex.com -username = ${TX_USR} -password = ${TX_PWD} -token = +rest_hostname = https://rest.api.transifex.com +token = ${TX_TOKEN} EOF diff --git a/thinkhazard/scripts/tx-pull-db b/thinkhazard/scripts/tx-pull-db index 2b2f7f38..5626c398 100755 --- a/thinkhazard/scripts/tx-pull-db +++ b/thinkhazard/scripts/tx-pull-db @@ -1,7 +1,7 @@ -#!/bin/bash +#!/bin/bash -e $(dirname $0)/tx-init -tx pull -s -r gfdrr-thinkhazard.database ${TX_BRANCH:+-b "$TX_BRANCH"} +tx pull --translations --languages=es,fr --resources=gfdrr-thinkhazard.database --force import_l10n -v diff --git a/thinkhazard/scripts/tx-push-db b/thinkhazard/scripts/tx-push-db index 4c98bf34..c6265103 100755 --- a/thinkhazard/scripts/tx-push-db +++ b/thinkhazard/scripts/tx-push-db @@ -1,7 +1,7 @@ -#!/bin/bash +#!/bin/bash -e $(dirname $0)/tx-init pot-create -c lingua.cfg -o /tmp/thinkhazard-database.pot thinkhazard/dont_remove_me.db-i18n -tx push -s -r gfdrr-thinkhazard.database ${TX_BRANCH:+-b "$TX_BRANCH"} +tx push --source --resources=gfdrr-thinkhazard.database