From 8f477ec274f223e4028abc486a231deb614a2b7e Mon Sep 17 00:00:00 2001 From: Ewan Cahen Date: Fri, 16 Feb 2024 17:45:50 +0100 Subject: [PATCH 1/4] feat: add CodeMeta server --- codemeta/go.mod | 8 ++ codemeta/main.go | 293 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 301 insertions(+) create mode 100644 codemeta/go.mod create mode 100644 codemeta/main.go diff --git a/codemeta/go.mod b/codemeta/go.mod new file mode 100644 index 000000000..f9979a58a --- /dev/null +++ b/codemeta/go.mod @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +module codemeta + +go 1.22 diff --git a/codemeta/main.go b/codemeta/main.go new file mode 100644 index 000000000..3206a29de --- /dev/null +++ b/codemeta/main.go @@ -0,0 +1,293 @@ +// SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +// SPDX-FileCopyrightText: 2024 Netherlands eScience Center +// +// SPDX-License-Identifier: Apache-2.0 + +// https://research-software-directory.org/api/v1/software?slug=eq.rsd-ng&select=brand_name,concept_doi,short_statement,contributor(family_names,given_names,affiliation,role,orcid),license_for_software(license),repository_url(url) + +package main + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "regexp" +) + +type RsdSoftware struct { + BrandName string `json:"brand_name"` + ConceptDoi *string `json:"concept_doi"` + ShortStatement string `json:"short_statement"` + Contributor []struct { + FamilyNames string `json:"family_names"` + GivenNames string `json:"given_names"` + Affiliation *string `json:"affiliation"` + EmailAddress *string `json:"email_address"` + Orcid *string `json:"orcid"` + Role *string `json:"role"` + } `json:"contributor"` + Keyword []struct { + Value string `json:"value"` + } `json:"keyword"` + License []struct { + License string `json:"license"` + } `json:"license_for_software"` + RepositoryURL *struct { + URL string `json:"url"` + } `json:"repository_url"` +} + +type Organization struct { + Type string `json:"@type"` + Name string `json:"name"` +} + +type Person struct { + Type string `json:"@type"` + Id *string `json:"@id"` + GivenName string `json:"givenName"` + FamilyName string `json:"familyName"` + Email *string `json:"email"` + RoleName *string `json:"roleName"` + Affiliation *Organization `json:"affiliation,omitempty"` +} + +type CreativeWork struct { + Type string `json:"@type"` + Name string `json:"name"` +} + +type SoftwareApplication struct { + Context string `json:"@context"` + Type string `json:"@type"` + Id *string `json:"@id"` + Name string `json:"name"` + Description string `json:"description"` + CodeRepository *string `json:"codeRepository"` + Author []Person `json:"author"` + Keyword []string `json:"keyword"` + License []CreativeWork `json:"license"` +} + +var slugRegex *regexp.Regexp = regexp.MustCompile("^[a-z0-9]+(-[a-z0-9]+)*$") + +func main() { + postgrestUrl := os.Getenv("POSTGREST_URL") + if postgrestUrl == "" { + log.Print("No value found for POSTGREST_URL, defaulting to http://backend:3500") + postgrestUrl = "http://backend:3500" + } + + http.HandleFunc("GET /", func(writer http.ResponseWriter, request *http.Request) { + http.Redirect(writer, request, "/v3/", http.StatusMovedPermanently) + }) + + http.HandleFunc("GET /v3/", func(writer http.ResponseWriter, request *http.Request) { + writer.WriteHeader(http.StatusBadRequest) + _, err := writer.Write([]byte("Please provide a slug")) + if err != nil { + log.Print("Couldn't write response: ", err) + } + }) + + http.HandleFunc("GET /v3/{slug}/{other}/", func(writer http.ResponseWriter, request *http.Request) { + writer.WriteHeader(http.StatusBadRequest) + _, err := writer.Write([]byte("The slug is not valid")) + if err != nil { + log.Print("Couldn't write response: ", err) + } + }) + + http.HandleFunc("GET /favicon.ico", func(writer http.ResponseWriter, request *http.Request) { + writer.WriteHeader(http.StatusNoContent) + }) + + http.HandleFunc("GET /v3/{slug}/", func(writer http.ResponseWriter, request *http.Request) { + slug := request.PathValue("slug") + + if len(slug) > 200 { + writer.WriteHeader(http.StatusBadRequest) + _, err := writer.Write([]byte("The slug should be at most 200 characters")) + if err != nil { + log.Print("Couldn't write response: ", err) + } + return + } + + if !slugRegex.MatchString(slug) { + writer.WriteHeader(http.StatusBadRequest) + _, err := writer.Write([]byte("The slug is not valid")) + if err != nil { + log.Print("Couldn't write response: ", err) + } + return + } + + urlUnformatted := "%v/software?slug=eq.%v&select=brand_name,concept_doi,short_statement,contributor(family_names,given_names,affiliation,role,orcid,email_address),keyword(value),license_for_software(license),repository_url(url)" + url := fmt.Sprintf(urlUnformatted, postgrestUrl, slug) + + resp, err := http.Get(url) + if err != nil { + log.Print("Unknown error when downloading data for slug "+slug+": ", err) + writer.WriteHeader(http.StatusInternalServerError) + _, err := writer.Write([]byte("Server error")) + if err != nil { + log.Print("Couldn't write response: ", err) + } + return + } + + var bytes []byte + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + log.Print("Unknown error when closing response body for slug "+slug+": ", err) + writer.WriteHeader(http.StatusInternalServerError) + _, err := writer.Write([]byte("Server error")) + if err != nil { + log.Print("Couldn't write response: ", err) + } + return + } + }(resp.Body) + bytes, err = io.ReadAll(resp.Body) + if err != nil { + log.Print("Unknown error for reading response body for slug "+slug+": ", err) + writer.WriteHeader(http.StatusInternalServerError) + _, err := writer.Write([]byte("Server error")) + if err != nil { + log.Print("Couldn't write response: ", err) + } + return + } + + jsonBytes, err := convertRsdToCodeMeta(bytes) + if err != nil { + log.Print("Unknown error for slug "+slug+": ", err) + writer.WriteHeader(http.StatusInternalServerError) + _, err := writer.Write([]byte("Server error")) + if err != nil { + log.Print("Couldn't write response: ", err) + } + return + } + if jsonBytes == nil { + writer.WriteHeader(http.StatusNotFound) + _, err := writer.Write([]byte("No software found with slug " + slug)) + if err != nil { + log.Print("Couldn't write response: ", err) + } + return + } + + writer.Header().Add("Content-Type", "application/ld+json") + _, err = writer.Write(jsonBytes) + if err != nil { + log.Print("Couldn't write response: ", err) + } + }) + err := http.ListenAndServe(":8000", nil) + + if err != nil { + log.Fatal("Couldn't start HTTP server: ", err) + } +} + +func convertRsdToCodeMeta(bytes []byte) ([]byte, error) { + var rsdResponseArray []RsdSoftware + err := json.Unmarshal(bytes, &rsdResponseArray) + if err != nil { + return nil, err + } + + if len(rsdResponseArray) == 0 { + return nil, nil + } + + rsdResponse := rsdResponseArray[0] + + var codemetaData = SoftwareApplication{ + Context: "https://w3id.org/codemeta/v3.0", + Id: rsdResponse.ConceptDoi, + Type: "SoftwareApplication", + Name: rsdResponse.BrandName, + Description: rsdResponse.ShortStatement, + Author: extractContributors(rsdResponse), + Keyword: extractKeywords(rsdResponse), + License: extractLicenses(rsdResponse), + } + + if rsdResponse.RepositoryURL != nil { + codemetaData.CodeRepository = &rsdResponse.RepositoryURL.URL + } + + jsonBytes, err := json.Marshal(codemetaData) + if err != nil { + log.Fatal(err) + return nil, err + } + return jsonBytes, nil +} + +func extractKeywords(rsdSoftware RsdSoftware) []string { + result := []string{} + + for _, jsonKeyword := range rsdSoftware.Keyword { + keyword := jsonKeyword.Value + + result = append(result, keyword) + } + + return result +} + +func extractLicenses(rsdSoftware RsdSoftware) []CreativeWork { + result := []CreativeWork{} + + for _, jsonLicense := range rsdSoftware.License { + license := CreativeWork{ + Type: "creativeWork", + Name: jsonLicense.License, + } + + result = append(result, license) + } + + return result +} + +func extractContributors(rsdSoftware RsdSoftware) []Person { + result := []Person{} + + for _, jsonContributor := range rsdSoftware.Contributor { + person := Person{ + Type: "Person", + Id: jsonContributor.Orcid, + GivenName: jsonContributor.GivenNames, + FamilyName: jsonContributor.FamilyNames, + Email: jsonContributor.EmailAddress, + RoleName: jsonContributor.Role, + } + + if jsonContributor.Orcid != nil { + const orcidUrlPrefix string = "https://orcid.org/" + var orcid string = *jsonContributor.Orcid + var orcidUrl string = orcidUrlPrefix + orcid + person.Id = &orcidUrl + } + + if jsonContributor.Affiliation != nil { + person.Affiliation = &Organization{ + Type: "Organization", + Name: *jsonContributor.Affiliation, + } + } + + result = append(result, person) + } + + return result +} From f8e123ef0ad052c40d450b22d2e14704bc38df52 Mon Sep 17 00:00:00 2001 From: Ewan Cahen Date: Wed, 21 Feb 2024 17:19:02 +0100 Subject: [PATCH 2/4] docs: add readme docs --- codemeta/README.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 codemeta/README.md diff --git a/codemeta/README.md b/codemeta/README.md new file mode 100644 index 000000000..bcd6aa75b --- /dev/null +++ b/codemeta/README.md @@ -0,0 +1,34 @@ + + +# CodeMeta server + +This module implements a [CodeMeta](https://codemeta.github.io/) server, using [V3](https://w3id.org/codemeta/v3.0) of the schema. + +If your instance of the RSD is hosted on https://example.com/, you can access the CodeMeta server on https://example.com/codemeta/v3/. To get the CodeMeta data for a software page, just append its slug to this URL. For example, to get the CodeMeta data for software page on https://example.com/software/some-software, visit https://example.com/codemeta/v3/some-software/. + +## Developing + +This module is written in Go, version 1.22. You can find the installation instruction [here](https://go.dev/doc/install). If this page lists a newer version than the one we use, you can find all versions [here](https://go.dev/dl/). We currently don't have any external dependencies. + +When developing and running it locally, you need a locally running RSD instance, and you should set the `POSTGREST_URL` environment variable to a value at which the PostgREST backend can be reached. This will probably be `http://localhost/api/v1`. You can also point it to a dev or production server instead. + +To build the module, open a terminal in the `codemeta` directory and run + +```shell +go build +``` + +This should produce an executable called `codemeta`. Run it with + +```shell +POSTGREST_URL=http://localhost/api/v1 ./codemeta +``` + +You can then access the service on http://localhost:8000/v3/ + +You can also use an IDE to build and run it more easily. From 7802770895c4d1061db3a6fade8da7b700f6be11 Mon Sep 17 00:00:00 2001 From: Ewan Cahen Date: Thu, 22 Feb 2024 11:22:44 +0100 Subject: [PATCH 3/4] build: integrate with docker --- Makefile | 6 +++--- codemeta/Dockerfile | 17 +++++++++++++++++ deployment/docker-compose.yml | 15 +++++++++++++++ docker-compose.yml | 14 ++++++++++++++ nginx/nginx.conf | 7 ++++++- 5 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 codemeta/Dockerfile diff --git a/Makefile b/Makefile index 14f6af7d1..4597a6f97 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,9 @@ # SPDX-FileCopyrightText: 2022 - 2023 Dusan Mijatovic (dv4all) -# SPDX-FileCopyrightText: 2022 - 2023 Ewan Cahen (Netherlands eScience Center) -# SPDX-FileCopyrightText: 2022 - 2023 Netherlands eScience Center # SPDX-FileCopyrightText: 2022 - 2023 dv4all # SPDX-FileCopyrightText: 2022 - 2024 Christian Meeßen (GFZ) +# SPDX-FileCopyrightText: 2022 - 2024 Ewan Cahen (Netherlands eScience Center) # SPDX-FileCopyrightText: 2022 - 2024 Helmholtz Centre Potsdam - GFZ German Research Centre for Geosciences +# SPDX-FileCopyrightText: 2022 - 2024 Netherlands eScience Center # SPDX-FileCopyrightText: 2022 Jesús García Gonzalez (Netherlands eScience Center) # SPDX-FileCopyrightText: 2023 Dusan Mijatovic (Netherlands eScience Center) # @@ -34,7 +34,7 @@ start: clean # open http://localhost to see the application running install: clean - docker compose build database backend auth scrapers nginx # exclude frontend and wait for the build to finish + docker compose build database backend auth codemeta scrapers nginx # exclude frontend and wait for the build to finish docker compose up --scale scrapers=0 --detach cd frontend && yarn install cd documentation && yarn install diff --git a/codemeta/Dockerfile b/codemeta/Dockerfile new file mode 100644 index 000000000..897570610 --- /dev/null +++ b/codemeta/Dockerfile @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2024 Netherlands eScience Center +# +# SPDX-License-Identifier: Apache-2.0 + +FROM golang:1.22.0-bullseye + +WORKDIR /usr/src/app +COPY go.mod . +COPY **.go . + +RUN go build -v -o /usr/local/bin/app + +RUN useradd user +USER user + +CMD ["app"] diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml index 2a1e7cdc1..e406edec0 100644 --- a/deployment/docker-compose.yml +++ b/deployment/docker-compose.yml @@ -162,6 +162,18 @@ services: - net restart: unless-stopped + codemeta: + image: ghcr.io/research-software-directory/rsd-saas/codemeta:latest + expose: + - "8000" + environment: + - POSTGREST_URL + depends_on: + - backend + networks: + - net + restart: unless-stopped + swagger: image: swaggerapi/swagger-ui:v4.15.0 expose: @@ -196,6 +208,9 @@ services: - backend - auth - frontend + - codemeta + - swagger + - documentation networks: - net restart: unless-stopped diff --git a/docker-compose.yml b/docker-compose.yml index a879bdde6..fd2c35a28 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -176,6 +176,18 @@ services: networks: - net + codemeta: + build: ./codemeta + image: rsd/codemeta:v1.0.0 + expose: + - "8000" + environment: + - POSTGREST_URL + depends_on: + - backend + networks: + - net + swagger: image: swaggerapi/swagger-ui:v4.15.0 expose: @@ -209,7 +221,9 @@ services: - backend - auth - frontend + - codemeta - swagger + - documentation networks: - net volumes: diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 685ca76a4..db4b61044 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -12,7 +12,7 @@ upstream authentication { server auth:7000; } -server{ +server { listen 80; server_name www.localhost; return 301 $scheme://localhost$request_uri; @@ -79,6 +79,11 @@ server { proxy_pass http://backend/; } + location /metadata/codemeta/ { + proxy_pass http://codemeta:8000/; + proxy_redirect ~(.*) /metadata/codemeta$1; + } + location /swagger/ { proxy_pass http://swagger:8080/; } From 1ad742b2d3fd3138e4759d0f32c85e266f466d9f Mon Sep 17 00:00:00 2001 From: Ewan Cahen Date: Thu, 22 Feb 2024 14:01:57 +0100 Subject: [PATCH 4/4] ci: add GitHub workflow actions --- .github/workflows/codemeta_scan_build.yml | 42 +++++++++++++++++++++++ .github/workflows/e2e_tests_chrome.yml | 4 +-- .github/workflows/release.yml | 31 ++++++++++++----- codemeta/sonar-project.properties | 13 +++++++ 4 files changed, 80 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/codemeta_scan_build.yml create mode 100644 codemeta/sonar-project.properties diff --git a/.github/workflows/codemeta_scan_build.yml b/.github/workflows/codemeta_scan_build.yml new file mode 100644 index 000000000..41149141a --- /dev/null +++ b/.github/workflows/codemeta_scan_build.yml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2024 Netherlands eScience Center +# +# SPDX-License-Identifier: Apache-2.0 + +name: codemeta code quality scan and build + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - "codemeta/**" + pull_request: + paths: + - "codemeta/**" + +jobs: + codemeta-scan-build: + runs-on: ubuntu-22.04 + strategy: + matrix: + go-version: [ '1.22.0' ] + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis + - name: "Build application with Go" + uses: actions/setup-go@v5 + with: + go-version: '1.22.0' + - run: | + go build -v + working-directory: codemeta + - name: SonarCloud Scan codemeta + uses: SonarSource/sonarcloud-github-action@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + with: + projectBaseDir: codemeta diff --git a/.github/workflows/e2e_tests_chrome.yml b/.github/workflows/e2e_tests_chrome.yml index ec662164d..f52e2fe52 100644 --- a/.github/workflows/e2e_tests_chrome.yml +++ b/.github/workflows/e2e_tests_chrome.yml @@ -62,11 +62,11 @@ jobs: working-directory: . run: | cp e2e/.env.e2e .env - docker compose build --parallel database backend auth frontend documentation nginx + docker compose build --parallel database backend auth frontend documentation nginx codemeta swagger - name: start rsd working-directory: . run: | - docker compose up --detach database backend auth frontend documentation nginx swagger + docker compose up --detach database backend auth frontend documentation nginx codemeta swagger sleep 5 - name: run e2e tests in chrome working-directory: e2e diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0990731a0..c29ae0434 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -55,7 +55,7 @@ jobs: echo skipped=${{needs.release_tag.outputs.skipped}} echo tag=${{needs.release_tag.outputs.tag}} auth: - # it needs to be check on string value + # it needs to be checked on string value if: needs.release_tag.outputs.skipped == 'false' needs: release_tag name: auth @@ -70,7 +70,7 @@ jobs: token: ${{secrets.GITHUB_TOKEN}} database: - # it needs to be check on string value + # it needs to be checked on string value if: needs.release_tag.outputs.skipped == 'false' needs: release_tag name: database @@ -85,7 +85,7 @@ jobs: token: ${{secrets.GITHUB_TOKEN}} backend: - # it needs to be check on string value + # it needs to be checked on string value if: needs.release_tag.outputs.skipped == 'false' needs: release_tag name: backend api @@ -100,7 +100,7 @@ jobs: token: ${{secrets.GITHUB_TOKEN}} frontend: - # it needs to be check on string value + # it needs to be checked on string value if: needs.release_tag.outputs.skipped == 'false' needs: release_tag name: frontend @@ -115,7 +115,7 @@ jobs: token: ${{secrets.GITHUB_TOKEN}} nginx: - # it needs to be check on string value + # it needs to be checked on string value if: needs.release_tag.outputs.skipped == 'false' needs: release_tag name: nginx @@ -129,8 +129,23 @@ jobs: secrets: token: ${{secrets.GITHUB_TOKEN}} + codemeta: + # it needs to be checked on string value + if: needs.release_tag.outputs.skipped == 'false' + needs: release_tag + name: codemeta + uses: ./.github/workflows/_ghcr.yml + with: + ghcr_user: ${{github.actor}} + base_image_name: ghcr.io/research-software-directory/rsd-saas/codemeta + image_tag: ${{needs.release_tag.outputs.tag}} + dockerfile: codemeta/Dockerfile + docker_context: ./codemeta + secrets: + token: ${{secrets.GITHUB_TOKEN}} + scrapers: - # it needs to be check on string value + # it needs to be checked on string value if: needs.release_tag.outputs.skipped == 'false' needs: release_tag name: scrapers @@ -160,7 +175,7 @@ jobs: token: ${{secrets.GITHUB_TOKEN}} deployment_files: - # it needs to be check on string value + # it needs to be checked on string value if: needs.release_tag.outputs.skipped == 'false' needs: [ release_tag,auth,database,backend,frontend,nginx,scrapers,documentation ] name: create deployment.zip @@ -241,7 +256,7 @@ jobs: files: deployment.zip citation: - # it needs to be check on string value + # it needs to be checked on string value if: needs.release_tag.outputs.skipped == 'false' needs: [ release_tag, deployment_files, release_draft ] name: citations diff --git a/codemeta/sonar-project.properties b/codemeta/sonar-project.properties new file mode 100644 index 000000000..e27748d75 --- /dev/null +++ b/codemeta/sonar-project.properties @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2024 Ewan Cahen (Netherlands eScience Center) +# SPDX-FileCopyrightText: 2024 Netherlands eScience Center +# +# SPDX-License-Identifier: Apache-2.0 + +sonar.projectKey=rsd-codemeta +sonar.organization=research-software-directory + +sonar.sources=. +sonar.exclusions=**/*_test.go + +sonar.tests=. +sonar.test.inclusions=**/*_test.go