diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9835d42..c1fb4b9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,77 +10,76 @@ jobs: runs-on: ubuntu-latest env: - PUBLIC_IMAGE: datastewardshipwizard/nanopub-submission-service - PRIVATE_IMAGE: ${{ secrets.PRIVATE_REGISTRY_URL }}/nanopub-submission-service - TAG_DEVELOP: develop - TAG_LATEST: latest + PUBLIC_IMAGE_PREFIX: 'datastewardshipwizard' + DOCKER_IMAGE_NAME: 'nanopub-submission-service' + DOCKER_META_CONTEXT: '.' + DOCKER_META_FILE: 'Dockerfile' + DOCKER_META_PLATFORMS: 'linux/amd64,linux/arm64' steps: - - uses: actions/checkout@v2 + - name: '[setup] Check out repository' + uses: actions/checkout@v4 + with: + fetch-depth: 0 -# (1) -> Build Docker image - - name: Update build info - run: | - ./scripts/build-info.sh + - name: '[setup] Set up QEMU' + uses: docker/setup-qemu-action@v3 - - name: Docker build - run: | - docker build -t $PRIVATE_IMAGE:$GITHUB_SHA . + - name: '[setup] Set up Docker Buildx' + id: buildx + uses: docker/setup-buildx-action@v3 -# (2) -> Docker image tagging - - name: Docker login - if: github.event_name == 'push' + - name: '[setup ]Update build info' run: | - docker login -u "$DOCKER_HUB_USERNAME" -p "$DOCKER_HUB_PASSWORD" - docker login -u "$PRIVATE_REGISTRY_USERNAME" -p "$PRIVATE_REGISTRY_PASSWORD" "$PRIVATE_REGISTRY_URL" - env: - DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} - DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} - PRIVATE_REGISTRY_URL: ${{ secrets.PRIVATE_REGISTRY_URL }} - PRIVATE_REGISTRY_USERNAME: ${{ secrets.PRIVATE_REGISTRY_USERNAME }} - PRIVATE_REGISTRY_PASSWORD: ${{ secrets.PRIVATE_REGISTRY_PASSWORD }} + ./scripts/build-info.sh - - name: Docker push - commit SHA (private) - if: github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/') - run: | - docker push $PRIVATE_IMAGE:$GITHUB_SHA + - name: '[docker] Docker meta' + id: meta-test + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.PUBLIC_IMAGE_PREFIX }}/${{ env.DOCKER_IMAGE_NAME }} + tags: | + type=sha - - name: Docker tag and push - branch (private) - if: github.event_name == 'push' && startsWith(github.ref, 'refs/heads/') && !contains(github.ref, 'release') - run: | - GITHUB_BRANCH=`echo $GITHUB_REF | cut -d/ -f3- | sed 's#/#-#g'` - docker image tag $PRIVATE_IMAGE:$GITHUB_SHA $PRIVATE_IMAGE:$GITHUB_BRANCH - docker push $PRIVATE_IMAGE:$GITHUB_BRANCH + - name: '[docker] Docker build' + uses: docker/build-push-action@v4 + with: + context: ${{ env.DOCKER_META_CONTEXT }} + file: ${{ env.DOCKER_META_FILE }} + platforms: ${{ env.DOCKER_META_PLATFORMS }} + push: false + tags: ${{ steps.meta-test.outputs.tags }} + labels: ${{ steps.meta-test.outputs.labels }} - - name: Docker tag and push - develop (public) - if: github.event_name == 'push' && github.ref == 'refs/heads/develop' - run: | - docker image tag $PRIVATE_IMAGE:$GITHUB_SHA $PUBLIC_IMAGE:$TAG_DEVELOP - docker push $PUBLIC_IMAGE:$TAG_DEVELOP + - name: '[docker-hub] Docker login' + if: github.event_name != 'pull_request' && github.actor != 'dependabot[bot]' + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} - - name: Docker tag and push - latest (public) - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - run: | - docker image tag $PRIVATE_IMAGE:$GITHUB_SHA $PUBLIC_IMAGE:$TAG_LATEST - docker push $PUBLIC_IMAGE:$TAG_LATEST + - name: '[docker-hub] Docker meta' + id: meta-public + if: github.event_name != 'pull_request' && github.actor != 'dependabot[bot]' + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.PUBLIC_IMAGE_PREFIX }}/${{ env.DOCKER_IMAGE_NAME }} + tags: | + type=ref,event=branch + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.') }} - - name: Docker tag and push - version tag (public) - if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') - run: | - GITHUB_TAG=`echo $GITHUB_REF | cut -d/ -f3` - # Release vX.Y.Z - if [[ $GITHUB_TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - IMAGE_TAG_MAJOR="$PUBLIC_IMAGE:"`echo $GITHUB_TAG | sed -E "s/v(.*)\..*\..*/\1/g"` - IMAGE_TAG_MINOR="$PUBLIC_IMAGE:"`echo $GITHUB_TAG | sed -E "s/v(.*)\..*/\1/g"` - IMAGE_TAG_PATCH="$PUBLIC_IMAGE:"`echo $GITHUB_TAG | sed -E "s/v//g"` - echo "Publishing release: $IMAGE_TAG_PATCH"; - docker image tag $PRIVATE_IMAGE:$GITHUB_SHA $IMAGE_TAG_MAJOR && docker push $IMAGE_TAG_MAJOR; - docker image tag $PRIVATE_IMAGE:$GITHUB_SHA $IMAGE_TAG_MINOR && docker push $IMAGE_TAG_MINOR; - docker image tag $PRIVATE_IMAGE:$GITHUB_SHA $IMAGE_TAG_PATCH && docker push $IMAGE_TAG_PATCH; - fi - # Release candidate vX.Y.Z-rc.R - if [[ $GITHUB_TAG =~ ^v[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+$ ]]; then - IMAGE_TAG_RC="$PUBLIC_IMAGE:"`echo $GITHUB_TAG | sed -E "s/v//g"` - echo "Publishing release candidate: $IMAGE_TAG_RC"; - docker image tag $PRIVATE_IMAGE:$GITHUB_SHA $IMAGE_TAG_RC && docker push $IMAGE_TAG_RC; - fi + - name: '[docker-hub] Docker build+push' + uses: docker/build-push-action@v4 + if: github.event_name != 'pull_request' && steps.meta-public.outputs.tags != '' + with: + context: ${{ env.DOCKER_META_CONTEXT }} + file: ${{ env.DOCKER_META_FILE }} + platforms: ${{ env.DOCKER_META_PLATFORMS }} + push: true + tags: ${{ steps.meta-public.outputs.tags }} + labels: ${{ steps.meta-public.outputs.labels }} diff --git a/Dockerfile b/Dockerfile index 36c6917..713a1da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ -FROM python:3.9-slim-buster +FROM python:3.11-slim-bookworm RUN apt-get update && \ - apt install -y --no-install-recommends bash default-jre && \ + apt install -y --no-install-recommends bash default-jre build-essential gcc && \ rm -rf /var/lib/apt/lists/* WORKDIR /app diff --git a/bin/nanopub-1.36-jar-with-dependencies.jar b/bin/nanopub-1.36-jar-with-dependencies.jar deleted file mode 100644 index f483cc0..0000000 Binary files a/bin/nanopub-1.36-jar-with-dependencies.jar and /dev/null differ diff --git a/bin/nanopub-1.55-jar-with-dependencies.jar b/bin/nanopub-1.55-jar-with-dependencies.jar new file mode 100644 index 0000000..42d4cbb Binary files /dev/null and b/bin/nanopub-1.55-jar-with-dependencies.jar differ diff --git a/bin/np b/bin/np old mode 100644 new mode 100755 index fa5c620..6dd038c --- a/bin/np +++ b/bin/np @@ -61,4 +61,3 @@ fi >&2 echo "ERROR: Failed to find or download nanopub jar file." exit 1 - diff --git a/nanopub_submitter/api.py b/nanopub_submitter/api.py index 4463117..32f8ec0 100644 --- a/nanopub_submitter/api.py +++ b/nanopub_submitter/api.py @@ -6,8 +6,8 @@ from typing import Tuple -from nanopub_submitter.config import cfg_parser -from nanopub_submitter.consts import NICE_NAME, VERSION, BUILD_INFO,\ +from nanopub_submitter.config import cfg_parser, RequestConfig +from nanopub_submitter.consts import NICE_NAME, VERSION, BUILD_INFO, \ ENV_CONFIG, DEFAULT_CONFIG, DEFAULT_ENCODING from nanopub_submitter.logger import LOG, init_default_logging, init_config_logging from nanopub_submitter.mailer import Mailer @@ -43,6 +43,12 @@ def _extract_content_type(header: str) -> Tuple[str, str]: return input_format, DEFAULT_ENCODING +def _extract_servers(header: str) -> list[str]: + if header == '': + return [] + return list(map(lambda x: x.strip(), header.split(','))) + + @app.get(path='/') async def get_info(): return fastapi.responses.JSONResponse( @@ -62,7 +68,11 @@ async def submit_nanopub(request: fastapi.Request): # (2) Extract data submission_id = str(uuid.uuid4()) data = await request.body() - input_format, encoding = _extract_content_type(request.headers.get('Content-Type')) + input_format, encoding = _extract_content_type(request.headers.get('Content-Type', '')) + req_cfg = RequestConfig( + servers=_extract_servers(request.headers.get('X-NP-Servers', '')), + uri_replace=request.headers.get('X-URI-Replace', None) + ) if input_format != 'application/trig': return fastapi.responses.Response( status_code=fastapi.status.HTTP_400_BAD_REQUEST, @@ -73,6 +83,7 @@ async def submit_nanopub(request: fastapi.Request): try: result = process( cfg=cfg, + req_cfg=req_cfg, submission_id=submission_id, data=data.decode(encoding), ) @@ -85,16 +96,17 @@ async def submit_nanopub(request: fastapi.Request): LOG.error(f'[{submission_id}] Unexpected processing error: {str(e)}') return fastapi.responses.PlainTextResponse( status_code=fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR, - content='Failed to process the nanopublication', + content=f'Failed to process the nanopublication: {str(e)}', ) # (4) Mail Mailer.get().notice(nanopub_uri=result.location) # (5) Return + headers = dict() + if result.location is not None: + headers['Location'] = result.location return fastapi.responses.Response( status_code=fastapi.status.HTTP_201_CREATED, - headers={ - 'Location': result.location, - }, + headers=headers, content=str(result), ) diff --git a/nanopub_submitter/config.py b/nanopub_submitter/config.py index 3e66d4a..70c8fff 100644 --- a/nanopub_submitter/config.py +++ b/nanopub_submitter/config.py @@ -260,3 +260,10 @@ def config(self) -> SubmitterConfig: cfg_parser = SubmitterConfigParser() + + +class RequestConfig: + + def __init__(self, servers: list[str], uri_replace: Optional[str]): + self.servers = servers + self.uri_replace = uri_replace diff --git a/nanopub_submitter/consts.py b/nanopub_submitter/consts.py index 97100c9..002b5c2 100644 --- a/nanopub_submitter/consts.py +++ b/nanopub_submitter/consts.py @@ -1,6 +1,6 @@ PACKAGE_NAME = 'nanopub_submitter' NICE_NAME = 'DSW Nanopublication Submission Service' -PACKAGE_VERSION = '1.1.0' +PACKAGE_VERSION = '1.2.0' ENV_CONFIG = 'SUBMISSION_CONFIG' LOGGER_NAME = 'DSW_SUBMITTER' diff --git a/nanopub_submitter/nanopub.py b/nanopub_submitter/nanopub.py index 53b2c03..091178f 100644 --- a/nanopub_submitter/nanopub.py +++ b/nanopub_submitter/nanopub.py @@ -4,7 +4,7 @@ from typing import Optional, Tuple -from nanopub_submitter.config import SubmitterConfig +from nanopub_submitter.config import SubmitterConfig, RequestConfig from nanopub_submitter.consts import DEFAULT_ENCODING, PACKAGE_NAME, PACKAGE_VERSION from nanopub_submitter.logger import LOG from nanopub_submitter.triple_store import store_to_triple_store @@ -40,9 +40,11 @@ def __str__(self): class NanopubProcessingContext: - def __init__(self, submission_id: str, cfg: SubmitterConfig): + def __init__(self, submission_id: str, cfg: SubmitterConfig, + req_cfg: RequestConfig): self.id = submission_id self.cfg = cfg + self.req_cfg = req_cfg self.uri = None def cleanup(self): @@ -54,6 +56,20 @@ def cleanup(self): except Exception: pass + @property + def target_servers(self) -> list[str]: + if len(self.req_cfg.servers) > 0: + return self.req_cfg.servers + return self.cfg.nanopub.target_servers + + @property + def uri_replace(self) -> Optional[str]: + if self.req_cfg.uri_replace is None: + return self.cfg.nanopub.uri_replace + if '|' in self.req_cfg.uri_replace: + return self.req_cfg.uri_replace + return None + @property def input_file(self) -> str: return f'{self.id}.trig' @@ -100,23 +116,29 @@ def _split_nanopubs(nanopub_bundle: str) -> list[str]: def _publish_nanopub(nanopub_bundle: str, ctx: NanopubProcessingContext) -> list[str]: success = [] nanopubs = _split_nanopubs(nanopub_bundle) - for server in ctx.cfg.nanopub.target_servers: + for server in ctx.target_servers: ctx.debug(f'Submitting to: {server}') ok = True for nanopub in nanopubs: - r = requests.post( - url=server, - data=nanopub, - headers={ - 'Content-Type': f'application/trig; charset={DEFAULT_ENCODING}', - 'User-Agent': f'{PACKAGE_NAME}/{PACKAGE_VERSION}', - } - ) - if not r.ok: + try: + r = requests.post( + url=server, + data=nanopub.encode(encoding=DEFAULT_ENCODING), + headers={ + 'Content-Type': f'application/trig; charset={DEFAULT_ENCODING}', + 'User-Agent': f'{PACKAGE_NAME}/{PACKAGE_VERSION}', + }, + timeout=10, + ) + if not r.ok: + ok = False + ctx.warn(f'Failed to publish nanopub via {server}') + ctx.debug(f'status={r.status_code}') + ctx.debug(r.text) + break + except Exception as e: ok = False - ctx.warn(f'Failed to publish nanopub via {server}') - ctx.debug(f'status={r.status_code}') - ctx.debug(r.text) + ctx.warn(f'Failed to publish nanopub via {server}: {str(e)}') break if ok: ctx.info(f'Nanopub published via {server}') @@ -173,10 +195,10 @@ def _run_np_sign(ctx: NanopubProcessingContext) -> str: ctx=ctx, ) if exit_code != EXIT_SUCCESS: - LOG.warn(f'Failed to make TrustyURI ({exit_code}):\n{stdout}\n\n{stderr}') + LOG.warn(f'Failed to sign the nanopub ({exit_code}):\n{stdout}\n\n{stderr}') raise NanopubProcessingError( status_code=500, - message='Failed to make TrustyURI for nanopub.' + message='Failed to sign the nanopub.' ) return ctx.signed_file @@ -192,8 +214,13 @@ def _extract_np_uri(nanopub: str) -> Optional[str]: return last_this_prefix -def process(cfg: SubmitterConfig, submission_id: str, data: str) -> NanopubSubmissionResult: - ctx = NanopubProcessingContext(submission_id=submission_id, cfg=cfg) +def process(cfg: SubmitterConfig, req_cfg: RequestConfig, + submission_id: str, data: str) -> NanopubSubmissionResult: + ctx = NanopubProcessingContext( + submission_id=submission_id, + cfg=cfg, + req_cfg=req_cfg, + ) ctx.debug('Preprocessing nanopublication as RDF') try: graph = rdflib.ConjunctiveGraph() @@ -234,8 +261,8 @@ def process(cfg: SubmitterConfig, submission_id: str, data: str) -> NanopubSubmi ctx.cleanup() raise NanopubProcessingError(400, 'Failed to get nanopub URI') - if cfg.nanopub.uri_replace is not None: - old, new = cfg.nanopub.uri_replace.split('|', maxsplit=1) + if ctx.uri_replace is not None: + old, new = ctx.uri_replace.split('|', maxsplit=1) new_uri = nanopub_uri.replace(old, new) LOG.debug(f'Replacing {nanopub_uri} with {new_uri}') nanopub_uri = new_uri diff --git a/nanopub_submitter/triple_store.py b/nanopub_submitter/triple_store.py index ced3ea1..57e88e9 100644 --- a/nanopub_submitter/triple_store.py +++ b/nanopub_submitter/triple_store.py @@ -6,7 +6,7 @@ from typing import List from nanopub_submitter.config import SubmitterConfig -from nanopub_submitter.consts import COMMENT_INSTRUCTION_DELIMITER,\ +from nanopub_submitter.consts import COMMENT_INSTRUCTION_DELIMITER, \ COMMENT_POST_QUERY_PREFIX, COMMENT_PRE_QUERY_PREFIX @@ -22,6 +22,17 @@ def create_graph(class_name: str) -> rdflib.Graph: return GRAPH_CLASSES.get(class_name, rdflib.Graph)() +def _n3(node: rdflib.term.Node) -> str: + if isinstance(node, rdflib.URIRef): + return node.n3() + elif isinstance(node, rdflib.Literal): + return node.n3() + elif isinstance(node, rdflib.BNode): + return node.n3() + else: + raise ValueError(f'Unknown node type: {type(node)}') + + class QueryBuilder: def __init__(self): @@ -89,7 +100,7 @@ def basic_query_builder(cfg: SubmitterConfig, data: str, input_format: str) -> s g = create_graph(cfg.triple_store.graph_class) g.parse(data=data, format=input_format) triples = [ - f'{s.n3()} {p.n3()} {o.n3()} .' for s, p, o in g + f'{_n3(s)} {_n3(p)} {_n3(o)} .' for s, p, o in g ] if cfg.triple_store.graph_named is True and cfg.triple_store.graph_type: t = rdflib.URIRef(cfg.triple_store.graph_type) @@ -115,7 +126,7 @@ def multi_graph_query_builder(cfg: SubmitterConfig, data: str, input_format: str qb.insert_multigraph_start() for ctx in cg.contexts(): triples = [ - f'{s.n3()} {p.n3()} {o.n3()} .' + f'{_n3(s)} {_n3(p)} {_n3(o)} .' for s, p, o in cg.triples((None, None, None), context=ctx) ] qb.insert_multigraph(triples=triples, graph_node=ctx.identifier) diff --git a/requirements.txt b/requirements.txt index 4729b9f..d43684d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,35 +1,27 @@ -asgiref==3.4.1 -certifi==2021.5.30 -charset-normalizer==2.0.6 -click==8.0.1 -colorama==0.4.4 -fastapi==0.68.1 -flake8==3.9.2 -h11==0.12.0 -httptools==0.2.0 -idna==3.2 -isodate==0.6.0 -mccabe==0.6.1 -mypy==0.910 -mypy-extensions==0.4.3 -pycodestyle==2.7.0 -pydantic==1.8.2 -pyflakes==2.3.1 -pyparsing==2.4.7 -python-dotenv==0.19.0 -PyYAML==5.4.1 -rdflib==6.0.1 -requests==2.26.0 +annotated-types==0.6.0 +anyio==4.2.0 +certifi==2023.11.17 +charset-normalizer==3.3.2 +click==8.1.7 +fastapi==0.109.0 +h11==0.14.0 +httptools==0.6.1 +idna==3.6 +isodate==0.6.1 +pydantic==2.5.3 +pydantic_core==2.14.6 +pyparsing==3.1.1 +python-dotenv==1.0.0 +PyYAML==6.0.1 +rdflib==7.0.0 +requests==2.31.0 six==1.16.0 -SPARQLWrapper==1.8.5 -starlette==0.14.2 -toml==0.10.2 -types-aiofiles==0.1.9 -types-orjson==3.6.0 -types-PyYAML==5.4.10 -types-ujson==0.1.1 -typing-extensions==3.10.0.2 -urllib3==1.26.7 -uvicorn==0.15.0 -watchgod==0.7 -websockets==10.0 +sniffio==1.3.0 +SPARQLWrapper==2.0.0 +starlette==0.35.1 +typing_extensions==4.9.0 +urllib3==2.1.0 +uvicorn==0.26.0 +uvloop==0.19.0 +watchfiles==0.21.0 +websockets==12.0 diff --git a/setup.py b/setup.py index 27c4cb4..3c04eb5 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='nanopub_submitter', - version='1.1.0', + version='1.2.0', keywords='dsw submission document nanopublication', description='Submission service for publishing nanopublications from DSW', long_description=long_description,