From cf838bb31845c68b141ccfcf354fb6bc2dab6ad8 Mon Sep 17 00:00:00 2001 From: Raido Kaju Date: Tue, 11 Jun 2024 09:43:08 +0300 Subject: [PATCH 1/2] feat: migrate from Akka to JAVA virtual threads based tasks (#27) Ref: XRDCAT-12 --- README.md | 2 +- doc/xroad_catalog_installation_guide.md | 60 +- gradle/libs.versions.toml | 11 +- xroad-catalog-collector/README.md | 17 +- xroad-catalog-collector/build.gradle | 5 +- .../redhat/SPECS/xroad-catalog-collector.spec | 2 - .../collector/XRoadCatalogCollector.java | 104 +- .../collector/actors/CatalogSupervisor.java | 125 --- .../collector/actors/FetchOpenApiActor.java | 102 -- .../collector/actors/FetchRestActor.java | 103 -- .../collector/actors/FetchWsdlActor.java | 90 -- .../collector/actors/ListClientsActor.java | 137 --- .../collector/actors/ListMethodsActor.java | 153 --- .../collector/actors/OrganizationsActor.java | 193 ---- .../actors/OrganizationsSupervisor.java | 160 --- .../collector/actors/XRoadCatalogActor.java | 43 - .../ApplicationConfiguration.java | 82 +- .../configuration/TaskPoolConfiguration.java | 136 +++ .../extension/SpringActorProducer.java | 56 -- .../collector/extension/SpringExtension.java | 40 - .../collector/tasks/BaseFetchTask.java | 111 +++ .../FetchCompaniesTask.java} | 118 ++- .../collector/tasks/FetchOpenApiTask.java | 97 ++ .../FetchOrganizationsTask.java} | 137 ++- .../collector/tasks/FetchRestTask.java | 80 ++ .../collector/tasks/FetchWsdlsTask.java | 84 ++ .../collector/tasks/ListClientsTask.java | 144 +++ .../collector/tasks/ListMethodsTask.java | 172 ++++ .../collector/util/ClientListUtil.java | 21 +- .../collector/util/CollectorUtils.java | 74 ++ .../collector/util/MethodListUtil.java | 44 +- .../collector/util/OrganizationUtil.java | 45 +- .../catalog/collector/util/XRoadClient.java | 17 +- .../src/main/resources/application.conf | 20 - .../src/main/resources/collector.properties | 4 +- .../actors/CatalogSupervisorTest.java | 58 -- .../actors/FetchCompanyActorTest.java | 78 -- .../actors/FetchOpenApiActorTest.java | 81 -- .../actors/FetchOrganizationActorTest.java | 75 -- .../collector/actors/FetchRestActorTest.java | 79 -- .../collector/actors/FetchWsdlActorTest.java | 70 -- .../actors/ListClientsActorTest.java | 170 ---- .../collector/actors/ListClientsRestTest.java | 149 --- .../actors/ListMethodsActorTest.java | 85 -- .../actors/OrganizationsActorTest.java | 93 -- .../actors/OrganizationsSupervisorTest.java | 61 -- .../tasks/FetchCompaniesTaskTest.java | 141 +++ .../collector/tasks/FetchOpenApiTaskTest.java | 136 +++ .../tasks/FetchOrganizationTaskTest.java | 142 +++ .../collector/tasks/FetchRestTaskTest.java | 97 ++ .../collector/tasks/FetchWsdlsTaskTest.java | 95 ++ .../collector/tasks/ListClientsTaskTest.java | 236 +++++ .../collector/tasks/ListMethodsTaskTest.java | 115 +++ .../collector/util/XRoadClientTest.java | 12 +- .../resources/mock/companies/company.json | 166 ++++ .../mock/companies/getCompanies.json | 39 + .../mock/organizations/organizationsById.json | 926 ++++++++++++++++++ .../resources/mock/xroad/openapi/openapi.json | 68 ++ 58 files changed, 3408 insertions(+), 2553 deletions(-) delete mode 100644 xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/CatalogSupervisor.java delete mode 100644 xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/FetchOpenApiActor.java delete mode 100644 xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/FetchRestActor.java delete mode 100644 xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/FetchWsdlActor.java delete mode 100644 xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/ListClientsActor.java delete mode 100644 xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/ListMethodsActor.java delete mode 100644 xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/OrganizationsActor.java delete mode 100644 xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/OrganizationsSupervisor.java delete mode 100644 xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/XRoadCatalogActor.java create mode 100644 xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/configuration/TaskPoolConfiguration.java delete mode 100644 xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/extension/SpringActorProducer.java delete mode 100644 xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/extension/SpringExtension.java create mode 100644 xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/BaseFetchTask.java rename xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/{actors/FetchCompaniesActor.java => tasks/FetchCompaniesTask.java} (69%) create mode 100644 xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchOpenApiTask.java rename xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/{actors/FetchOrganizationsActor.java => tasks/FetchOrganizationsTask.java} (78%) create mode 100644 xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchRestTask.java create mode 100644 xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchWsdlsTask.java create mode 100644 xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/ListClientsTask.java create mode 100644 xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/ListMethodsTask.java create mode 100644 xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/CollectorUtils.java delete mode 100644 xroad-catalog-collector/src/main/resources/application.conf delete mode 100644 xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/CatalogSupervisorTest.java delete mode 100644 xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/FetchCompanyActorTest.java delete mode 100644 xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/FetchOpenApiActorTest.java delete mode 100644 xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/FetchOrganizationActorTest.java delete mode 100644 xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/FetchRestActorTest.java delete mode 100644 xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/FetchWsdlActorTest.java delete mode 100644 xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/ListClientsActorTest.java delete mode 100644 xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/ListClientsRestTest.java delete mode 100644 xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/ListMethodsActorTest.java delete mode 100644 xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/OrganizationsActorTest.java delete mode 100644 xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/OrganizationsSupervisorTest.java create mode 100644 xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchCompaniesTaskTest.java create mode 100644 xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchOpenApiTaskTest.java create mode 100644 xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchOrganizationTaskTest.java create mode 100644 xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchRestTaskTest.java create mode 100644 xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchWsdlsTaskTest.java create mode 100644 xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/ListClientsTaskTest.java create mode 100644 xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/ListMethodsTaskTest.java create mode 100644 xroad-catalog-collector/src/test/resources/mock/companies/company.json create mode 100644 xroad-catalog-collector/src/test/resources/mock/companies/getCompanies.json create mode 100644 xroad-catalog-collector/src/test/resources/mock/organizations/organizationsById.json create mode 100644 xroad-catalog-collector/src/test/resources/mock/xroad/openapi/openapi.json diff --git a/README.md b/README.md index b18f3e23..87a7a396 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ The X-Road Catalog software consists of three modules: - [X-Road Catalog Collector](xroad-catalog-collector/README.md) * Collects information from the X-Road ecosystem (possibly also from external APIs) and stores it to the postgresql database. - * Implemented using concurrent Akka actors. + * Implemented using JAVA virtual threads. - [X-Road Catalog Lister](xroad-catalog-lister/README.md) * Provides REST and SOAP interfaces that offer information collected by the Collector. * Can be used as an X-Road service (X-Road headers are in place). diff --git a/doc/xroad_catalog_installation_guide.md b/doc/xroad_catalog_installation_guide.md index 27b2dca9..2abe8d77 100644 --- a/doc/xroad_catalog_installation_guide.md +++ b/doc/xroad_catalog_installation_guide.md @@ -1,5 +1,5 @@ # X-Road Catalog Installation Guide -Version: 1.3.0 +Version: 1.3.1 Doc. ID: IG-XRDCAT --- @@ -11,6 +11,7 @@ Doc. ID: IG-XRDCAT | 16.08.2023 | 1.1.0 | Add instructions to install and configure the `xroad-conflient` module | Petteri Kivimäki | | 09.09.2023 | 1.2.0 | Remove instructions to install the `xroad-conflient` module manually | Petteri Kivimäki | | 24.09.2023 | 1.3.0 | Add instructions to disable the automatic backup job run by the `xroad-conflient` module | Petteri Kivimäki | +| 10.06.2024 | 1.3.1 | Add information about default values for configurable properties | Raido Kaju | ## Table of Contents @@ -150,8 +151,8 @@ The application log of the `xroad-confclient` module is available in `/var/log/x ## 2.5 Initial Configuration -Configure the below parameters in `/etc/xroad/xroad-catalog/collector-production.properties`, especially X-Road instance -information and URL of Security Server. +The following parameters must be manually configured in `/etc/xroad/xroad-catalog/collector-production.properties`, +especially X-Road instance information and URL of Security Server. ```properties xroad-catalog.xroad-instance= @@ -161,30 +162,43 @@ xroad-catalog.subsystem-code= xroad-catalog.security-server-host= ``` -In addition, configure also parameters related to behaviour of X-Road Catalog Collector: +When using the `xroad-catalog-collector` module with the `FI` profile, the following additional parameters must be +configured in the same file: ```properties -xroad-catalog.flush-log-time-after-hour= -xroad-catalog.flush-log-time-before-hour= -xroad-catalog.error-log-length-in-days= -xroad-catalog.fetch-run-unlimited= -xroad-catalog.fetch-time-after-hour= -xroad-catalog.fetch-time-before-hour= -xroad-catalog.collector-interval-min= +xroad-catalog.fetch-organizations-url= +xroad-catalog.fetch-companies-url= ``` -The parameters are explained in the table below. - -| Parameter | Description | -|-----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `ERROR_LOGS_FLUSH_IN_DB_TIME_INTERVAL_AFTER` | A parameter for setting the start of time interval during which the error logs in the db will be deleted when those exceed the amount in days set by `ERROR_LOGS_KEPT_IN_DB_LENGTH_IN_DAYS` parameter, e.g. value `18` means starting from `18:00`. | -| `ERROR_LOGS_FLUSH_IN_DB_TIME_INTERVAL_AFTER` | A parameter for setting the start of time interval during which the error logs in the db will be deleted when those exceed the amount in days set by `ERROR_LOGS_KEPT_IN_DB_LENGTH_IN_DAYS` parameter, e.g. value `18` means starting from `18:00`. | -|`ERROR_LOGS_FLUSH_IN_DB_TIME_INTERVAL_BEFORE` | A parameter for setting the end of time interval during which the error logs in the db will be deleted when those exceed the amount in days set by `ERROR_LOGS_KEPT_IN_DB_LENGTH_IN_DAYS` parameter, e.g. value `23` means ending at `23:00`. | -| `ERROR_LOGS_KEPT_IN_DB_LENGTH_IN_DAYS` | A parameter for setting the amount in days for how long the errors logs should be kept in the db, e.g. value `90` means `for 90 days`. | -| `XROAD_CATALOG_COLLECTOR_FETCH_UNLIMITED` | A parameter for setting whether the X-Road Catalog Collector should try to fetch data from Security Server continuously during a day or only between certain hours, e.g. value `true` means `continously`. | -| `XROAD_CATALOG_COLLECTOR_FETCH_INTERVAL_AFTER` | A parameter for setting the start of time interval during which the X-Road Catalog Collector should try to fetch data from Security Server continuously (this parameter will be ignored if the parameter `XROAD_CATALOG_COLLECTOR_FETCH_UNLIMITED` is set to `true`), e.g. value `18` means starting from `18:00`. | -| `XROAD_CATALOG_COLLECTOR_FETCH_INTERVAL_BEFORE` | A parameter for setting the end of time interval during which the X-Road Catalog Collector should try to fetch data from Security Server continuously (this parameter will be ignored if the parameter `XROAD_CATALOG_COLLECTOR_FETCH_UNLIMITED` is set to `true`), e.g. value `23` means ending at `23:00`. | -| `XROAD_CATALOG_COLLECTOR_FETCH_INTERVAL_MINUTES` | A parameter for setting the amount of time in minutes after which the X-Road Catalog Collector should start re-fetching data from Security Server, e.g. value `60` means `every 60 minutes`. | +Optional parameters which can be configured in the same file are described below along with their default values: + +| Parameter | Defaults | Description | +|--------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `xroad-catalog.fetch-run-unlimited` | false | A parameter for setting whether the X-Road Catalog Collector should try to fetch data from Security Server continuously during a day or only between certain hours, e.g. value `true` means `continously`. | +| `xroad-catalog.fetch-time-after-hour` | 3 | A parameter for setting the start of time interval during which the X-Road Catalog Collector should try to fetch data from Security Server continuously (this parameter will be ignored if the parameter `xroad-catalog.fetch-run-unlimited` is set to `true`), e.g. value `18` means starting from `18:00`. | +| `xroad-catalog.fetch-time-before-hour` | 4 | A parameter for setting the end of time interval during which the X-Road Catalog Collector should try to fetch data from Security Server continuously (this parameter will be ignored if the parameter `xroad-catalog.fetch-run-unlimited` is set to `true`), e.g. value `23` means ending at `23:00`. | +| `xroad-catalog.collector-interval-min` | 20 | A parameter for setting the amount of time in minutes after which the X-Road Catalog Collector should start re-fetching data from Security Server, e.g. value `20` means `every 20 minutes`. | +| `xroad-catalog.list-methods-pool-size` | 50 | A parameter for setting the amount of virtual threads in the pool for fetching methods metadata from Security Server, e.g. value `50` means `50 virtual threads`. | +| `xroad-catalog.fetch-wsdl-pool-size` | 10 | A parameter for setting the amount of virtual threads in the pool for fetching WSDLs from Security Server, e.g. value `10` means `10 virtual threads`. | +| `xroad-catalog.fetch-rest-pool-size` | 10 | A parameter for setting the amount of virtual threads in the pool for fetching REST services from Security Server, e.g. value `10` means `10 virtual threads`. | +| `xroad-catalog.fetch-openapi-pool-size` | 10 | A parameter for setting the amount of virtual threads in the pool for fetching OpenAPI services from Security Server, e.g. value `10` means `10 virtual threads`. | +| `xroad-catalog.flush-log-time-after-hour` | 3 | A parameter for setting the start of time interval during which the error logs in the db will be deleted when those exceed the amount in days set by `xroad-catalog.error-log-length-in-days` parameter, e.g. value `18` means starting from `18:00`. | +| `xroad-catalog.flush-log-time-before-hour` | 4 | A parameter for setting the end of time interval during which the error logs in the db will be deleted when those exceed the amount in days set by `xroad-catalog.error-log-length-in-days` parameter, e.g. value `23` means ending at `23:00`. | +| `xroad-catalog.error-log-length-in-days` | 90 | A parameter for setting the amount in days for how long the errors logs should be kept in the db, e.g. value `90` means `for 90 days`. | + +When using the `xroad-catalog-collector` module with the `FI` profile, the following additional optional parameters are +in effect: + +| Parameter | Defaults | Description | +|--------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `xroad-catalog.max-organizations-per-request` | 100 | A parameter for setting the maximum amount of organizations that can be fetched from the organizations API in one request, e.g. value `100` means `100 organizations`. | +| `xroad-catalog.fetch-companies-limit` | 1000 | A parameter for setting the maximum amount of companies that can be fetched from the companies API, e.g. value `1000` means `1000 companies`. | +| `xroad-catalog.fetch-organizations-limit` | 2000 | A parameter for setting the maximum amount of organizations that can be fetched from the organizations API, e.g. value `2000` means `2000 organizations`. | +| `xroad-catalog.fetch-companies-run-unlimited` | false | A parameter for setting whether the X-Road Catalog Collector should try to fetch data from the companies API continuously during a day or only between certain hours, e.g. value `true` means `continously`. | +| `xroad-catalog.fetch-companies-time-after-hour` | 3 | A parameter for setting the start of time interval during which the X-Road Catalog Collector should try to fetch data from the companies API continuously (this parameter will be ignored if the parameter `xroad-catalog.fetch-companies-run-unlimited` is set to `true`), e.g. value `18` means starting from `18:00`. | +| `xroad-catalog.fetch-companies-time-before-hour` | 4 | A parameter for setting the end of time interval during which the X-Road Catalog Collector should try to fetch data from the companies API continuously (this parameter will be ignored if the parameter `xroad-catalog.fetch-companies-run-unlimited` is set to `true`), e.g. value `23` means ending at `23:00`. | +| `xroad-catalog.fetch-organizations-pool-size` | 10 | A parameter for setting the amount of virtual threads in the pool for fetching organizations from the organizations API, e.g. value `10` means `10 virtual threads`. | +| `xroad-catalog.fetch-companies-pool-size` | 10 | A parameter for setting the amount of virtual threads in the pool for fetching companies from the companies API, e.g. value `10` means `10 virtual threads`. | In addition, update the `xroad-catalog.shared-params-file` property value in `/etc/xroad/xroad-catalog/lister-production.properties`. The value must point to the `/etc/xroad/globalconf//shared-params.xml` X-Road global configuration file: diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 834c82ee..1ac07659 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,5 +1,4 @@ [versions] -akka = "2.6.21" commons-beanutils = "1.9.4" commons-csv = "1.11.0" cxf = "4.0.4" @@ -14,7 +13,7 @@ wsImport = "4.0.2" jaxb = "4.0.2" json = "20240303" lombok = "1.18.32" -mockito-inline = "4.6.1" +mockito = "5.12.0" postgresql = "42.7.3" spring-boot = "3.2.5" spring-doc = "1.8.0" @@ -22,11 +21,9 @@ wsdl4j = "1.6.3" wsdl2java = "2.0.2" sonarqube = "5.0.0.4638" hierynomus-license = "0.16.1" +awaitility = "4.2.1" [libraries] -akka-actor = { module = "com.typesafe.akka:akka-actor_3", version.ref = "akka" } -akka-slf4j = { module = "com.typesafe.akka:akka-slf4j_3", version.ref = "akka" } -akka-testkit = { module = "com.typesafe.akka:akka-testkit_3", version.ref = "akka" } commons-bean-utils = { module = "commons-beanutils:commons-beanutils", version.ref = "commons.beanutils" } commons-csv = { module = "org.apache.commons:commons-csv", version.ref = "commons.csv" } cxf-core = { module = "org.apache.cxf:cxf-core", version.ref = "cxf" } @@ -46,16 +43,16 @@ jaxb-tools = { module = "com.sun.xml.ws:jaxws-tools", version.ref = "jaxb" } jaxb-ws-api = { module = "jakarta.xml.ws:jakarta.xml.ws-api", version.ref = "jaxb" } json = { module = "org.json:json", version.ref = "json" } lombok = { module = "org.projectlombok:lombok", version.ref = "lombok" } -mockito-inline = { module = "org.mockito:mockito-inline", version.ref = "mockito.inline" } +mockito = { module = "org.mockito:mockito-core", version.ref = "mockito" } postgresql = { module = "org.postgresql:postgresql", version.ref = "postgresql" } spring-boot-jpa = { module = "org.springframework.boot:spring-boot-starter-data-jpa", version.ref = "spring.boot" } spring-boot-test = { module = "org.springframework.boot:spring-boot-starter-test", version.ref = "spring.boot" } spring-boot-ws = { module = "org.springframework.boot:spring-boot-starter-web-services", version.ref = "spring.boot" } spring-doc = { module = "org.springdoc:springdoc-openapi-ui", version.ref = "spring.doc" } wsdl4j = { module = "wsdl4j:wsdl4j", version.ref = "wsdl4j" } +awaitility = { module = "org.awaitility:awaitility", version.ref = "awaitility" } [bundles] -akka = [ "akka-actor", "akka-slf4j" ] # Don't include akka-testkit in this since it is not used in implementation jaxb = [ "jaxb-rt", "jaxb-tools", "jaxb-ws-api" ] # Don't include jaxb-maven in this since it is not used in implementation [plugins] diff --git a/xroad-catalog-collector/README.md b/xroad-catalog-collector/README.md index e9dd601a..41efbe5b 100644 --- a/xroad-catalog-collector/README.md +++ b/xroad-catalog-collector/README.md @@ -3,14 +3,15 @@ The purpose of this module is to collect members, subsystems and services from the X-Road ecosystem and store them to a database. -The module is implemented using concurrent Akka actors: - -* `FetchWsdlActor` - fetches WSDL descriptions of SOAP services from the X-Road instance and stores them to the db. -* `FetchOpenApiActor` - fetches OpenAPI descriptions of Rest services from the X-Road instance and stores them to the db. -* `ListClientsActor` - fetches a list of clients from the X-Road instance and stores them to the db. -* `ListMethodsActor` - fetches a list of services from the X-Road instance and stores them to the db. -* `FetchOrganizationsActor` - fetches a list of public organizations from an external API and stores them to the db. -* `FetchCompaniesActor` - fetches a list of private companies from an external API and stores them to the db. +The module is implemented using JAVA virtual threads: + +* `FetchWsdlTask` - fetches WSDL descriptions of SOAP services from the X-Road instance and stores them to the db. +* `FetchRestTask` - fetches REST services without descriptions from the X-Road instance and stores them to the db. +* `FetchOpenApiTask` - fetches OpenAPI descriptions of Rest services from the X-Road instance and stores them to the db. +* `ListClientsTask` - fetches a list of clients from the X-Road instance and stores them to the db. +* `ListMethodsTask` - fetches a list of services from the X-Road instance and stores them to the db. +* `FetchOrganizationsTask` - fetches a list of public organizations from an external API and stores them to the db. +* `FetchCompaniesTask` - fetches a list of private companies from an external API and stores them to the db. See also the [Installation Guide](../doc/xroad_catalog_installation_guide.md) and [User Guide](../doc/xroad_catalog_user_guide.md). diff --git a/xroad-catalog-collector/build.gradle b/xroad-catalog-collector/build.gradle index ce67a1f3..9bfe2348 100644 --- a/xroad-catalog-collector/build.gradle +++ b/xroad-catalog-collector/build.gradle @@ -84,7 +84,6 @@ dependencies { implementation (libs.json) implementation (libs.spring.boot.ws) implementation (libs.jackson) - implementation (libs.bundles.akka) // Note that bundle seems to break something with CXF and being able top find api bindings, so using separate dependencies here implementation (libs.cxf.jaxws) implementation (libs.cxf.transport) @@ -97,12 +96,12 @@ dependencies { testImplementation (libs.spring.boot.test) { exclude group: 'org.mockito', module :'mockito-core' } - testImplementation (libs.mockito.inline) - testImplementation (libs.akka.testkit) + testImplementation (libs.mockito) testImplementation (libs.lombok) testAnnotationProcessor (libs.lombok) testImplementation (libs.jakarta.annotation) testImplementation (libs.jakarta.jaxb) + testImplementation (libs.awaitility) } compileJava.dependsOn wsImport diff --git a/xroad-catalog-collector/packages/xroad-catalog-collector/redhat/SPECS/xroad-catalog-collector.spec b/xroad-catalog-collector/packages/xroad-catalog-collector/redhat/SPECS/xroad-catalog-collector.spec index 155b39c1..3caf3d15 100644 --- a/xroad-catalog-collector/packages/xroad-catalog-collector/redhat/SPECS/xroad-catalog-collector.spec +++ b/xroad-catalog-collector/packages/xroad-catalog-collector/redhat/SPECS/xroad-catalog-collector.spec @@ -38,7 +38,6 @@ cp -p %{src}/../../../build/libs/xroad-catalog-collector-%{version}.jar %{buildr cp -p %{src}/../../../build/resources/main/collector-production.properties %{buildroot}%{conf} cp -p %{src}/../../../build/resources/main/catalogdb-production.properties %{buildroot}%{conf} cp -p catalog-profile.properties %{buildroot}%{conf} -cp -p %{src}/../../../build/resources/main/application.conf %{buildroot}%{conf} cp -p ../../../../../xroad-catalog-persistence/src/main/sql/init_database.sql %{buildroot}/usr/share/xroad/sql cp -p ../../../../../xroad-catalog-persistence/src/main/sql/create_tables_%{profile}.sql %{buildroot}/usr/share/xroad/sql cp -p %{src}/SOURCES/%{name} %{buildroot}/usr/share/xroad/bin @@ -51,7 +50,6 @@ rm -rf %{buildroot} %files %defattr(600,xroad-catalog,xroad-catalog,-) -%config(noreplace) %{conf}/application.conf %config(noreplace) %{conf}/collector-production.properties %config(noreplace) %{conf}/catalogdb-production.properties diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/XRoadCatalogCollector.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/XRoadCatalogCollector.java index 1017ed7d..32f7dd7c 100644 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/XRoadCatalogCollector.java +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/XRoadCatalogCollector.java @@ -12,29 +12,41 @@ */ package fi.vrk.xroad.catalog.collector; -import fi.vrk.xroad.catalog.collector.actors.XRoadCatalogActor; -import fi.vrk.xroad.catalog.collector.extension.SpringExtension; -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import akka.event.Logging; -import akka.event.LoggingAdapter; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.ApplicationContext; import org.springframework.core.env.Environment; -import scala.concurrent.duration.Duration; -import java.util.Arrays; -import java.util.concurrent.TimeUnit; +import fi.vrk.xroad.catalog.collector.configuration.TaskPoolConfiguration; +import fi.vrk.xroad.catalog.collector.tasks.FetchCompaniesTask; +import fi.vrk.xroad.catalog.collector.tasks.FetchOpenApiTask; +import fi.vrk.xroad.catalog.collector.tasks.FetchOrganizationsTask; +import fi.vrk.xroad.catalog.collector.tasks.FetchRestTask; +import fi.vrk.xroad.catalog.collector.tasks.FetchWsdlsTask; +import fi.vrk.xroad.catalog.collector.tasks.ListClientsTask; +import fi.vrk.xroad.catalog.collector.tasks.ListMethodsTask; +import fi.vrk.xroad.catalog.collector.util.XRoadRestServiceIdentifierType; +import fi.vrk.xroad.catalog.collector.wsimport.ClientType; +import fi.vrk.xroad.catalog.collector.wsimport.XRoadServiceIdentifierType; +import lombok.extern.slf4j.Slf4j; +@Slf4j @SpringBootApplication public class XRoadCatalogCollector { - private static final String CATALOG_SUPERVISOR = "CatalogSupervisor"; - private static final String ORGANIZATIONS_SUPERVISOR = "OrganizationsSupervisor"; private static final String FI_PROFILE = "fi"; - public static void main(String[] args) { + public static void main(String[] args) throws MalformedURLException, URISyntaxException { ApplicationContext context = SpringApplication.run(XRoadCatalogCollector.class, args); @@ -43,29 +55,61 @@ public static void main(String[] args) { final String keystore = env.getProperty("xroad-catalog.ssl-keystore"); final String keystorePw = env.getProperty("xroad-catalog.ssl-keystore-password"); - if (keystore != null && keystorePw != null) { - System.setProperty("javax.net.ssl.keyStore", keystore); - System.setProperty("javax.net.ssl.keyStorePassword", keystorePw); + if (keystore != null && !keystore.isEmpty() && keystorePw != null) { + if (!Path.of(keystore).toFile().exists()) { + log.warn("Keystore file at {} is not accessible or does not exist, not using keystore", keystore); + } else { + log.info("Using keystore at {}", keystore); + System.setProperty("javax.net.ssl.keyStore", keystore); + System.setProperty("javax.net.ssl.keyStorePassword", keystorePw); + } } - ActorSystem system = context.getBean(ActorSystem.class); - final LoggingAdapter log = Logging.getLogger(system, "Application"); - Long collectorInterval = (Long) context.getBean("getCollectorInterval"); + final boolean isFIProfile = Arrays.stream(env.getActiveProfiles()) + .anyMatch(str -> str.equalsIgnoreCase(FI_PROFILE)); - log.info("Starting up catalog collector with collector interval of {}", collectorInterval); + final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); + final BlockingQueue listMethodsQueue = new LinkedBlockingQueue<>(); + final BlockingQueue fetchCompaniesQueue = isFIProfile ? new LinkedBlockingQueue<>() : null; + final BlockingQueue fetchOrganizationsQueue = isFIProfile ? new LinkedBlockingQueue<>() : null; + final BlockingQueue fetchWsdlsQueue = new LinkedBlockingQueue<>(); + final BlockingQueue fetchRestQueue = new LinkedBlockingQueue<>(); + final BlockingQueue fetchOpenApiQueue = new LinkedBlockingQueue<>(); - SpringExtension ext = context.getBean(SpringExtension.class); - String supervisorBeanName = CATALOG_SUPERVISOR; - if (Arrays.stream(env.getActiveProfiles()).anyMatch(str -> str.equalsIgnoreCase(FI_PROFILE))) { - supervisorBeanName = ORGANIZATIONS_SUPERVISOR; + if (isFIProfile) { + log.info("FI profile detected, starting up organizations and companies fetchers"); + final FetchCompaniesTask fetchCompaniesTask = new FetchCompaniesTask(context, fetchCompaniesQueue); + Thread.ofVirtual().start(fetchCompaniesTask::run); + + final FetchOrganizationsTask fetchOrganizationsTask = new FetchOrganizationsTask(context, + fetchOrganizationsQueue); + Thread.ofVirtual().start(fetchOrganizationsTask::run); } - ActorRef supervisor = system.actorOf(ext.props(supervisorBeanName)); - system.scheduler().scheduleWithFixedDelay(Duration.Zero(), - Duration.create(collectorInterval, TimeUnit.MINUTES), - supervisor, - XRoadCatalogActor.START_COLLECTING, - system.dispatcher(), - ActorRef.noSender()); + + final FetchWsdlsTask fetchWsdlsTask = new FetchWsdlsTask(context, fetchWsdlsQueue); + Thread.ofVirtual().start(fetchWsdlsTask::run); + + final FetchRestTask fetchRestTask = new FetchRestTask(context, fetchRestQueue); + Thread.ofVirtual().start(fetchRestTask::run); + + final FetchOpenApiTask fetchOpenApiTask = new FetchOpenApiTask(context, fetchOpenApiQueue); + Thread.ofVirtual().start(fetchOpenApiTask::run); + + final ListMethodsTask listMethodsTask = new ListMethodsTask(context, listMethodsQueue, fetchWsdlsQueue, + fetchRestQueue, + fetchOpenApiQueue); + Thread.ofVirtual().start(listMethodsTask::run); + + // The ListClientsTask is the main task that starts the whole process and + // gathers information that the other tasks will react on to do work + final ListClientsTask listClientsTask = new ListClientsTask(context, listMethodsQueue, fetchCompaniesQueue, + fetchOrganizationsQueue); + + long collectorInterval = context.getBean(TaskPoolConfiguration.class).getCollectorInterval(); + log.info("Starting up catalog collector with collector interval of {}", collectorInterval); + + scheduler.scheduleWithFixedDelay(listClientsTask::run, 0, collectorInterval, TimeUnit.MINUTES); + } } diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/CatalogSupervisor.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/CatalogSupervisor.java deleted file mode 100644 index e699e205..00000000 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/CatalogSupervisor.java +++ /dev/null @@ -1,125 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.actors; - -import akka.actor.ActorRef; -import akka.actor.OneForOneStrategy; -import akka.routing.SmallestMailboxPool; -import fi.vrk.xroad.catalog.collector.extension.SpringExtension; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; -import scala.concurrent.duration.Duration; - -import static akka.actor.SupervisorStrategy.restart; - -/** - * Supervisor to get list of all clients in system and initiate a ClientActor for each - *

- * A router is configured at startup time, managing a pool of task actors. - */ -@Component("CatalogSupervisor") -@Scope("prototype") -@Slf4j -public class CatalogSupervisor extends XRoadCatalogActor { - - public static final String LIST_CLIENTS_ACTOR_ROUTER = "list-clients-actor-router"; - public static final String LIST_METHODS_ACTOR_ROUTER = "list-methods-actor-router"; - public static final String FETCH_WSDL_ACTOR_ROUTER = "fetch-wsdl-actor-router"; - public static final String FETCH_OPENAPI_ACTOR_ROUTER = "fetch-openapi-actor-router"; - public static final String FETCH_REST_ACTOR_ROUTER = "fetch-rest-actor-router"; - - @Autowired - private SpringExtension springExtension; - - private ActorRef listClientsPoolRouter; - private ActorRef listMethodsPoolRouter; - private ActorRef fetchWsdlPoolRouter; - private ActorRef fetchOpenApiPoolRouter; - private ActorRef fetchRestPoolRouter; - - @Value("${xroad-catalog.list-methods-pool-size}") - private int listMethodsPoolSize; - - @Value("${xroad-catalog.fetch-wsdl-pool-size}") - private int fetchWsdlPoolSize; - - @Value("${xroad-catalog.fetch-openapi-pool-size}") - private int fetchOpenApiPoolSize; - - @Value("${xroad-catalog.fetch-rest-pool-size}") - private int fetchRestPoolSize; - - @Override - public void preStart() throws Exception { - - log.info("Starting up"); - - // prepare each pool. Give ActorRefs to other pools, as needed. - // To make this possible, create pools in correct order - - // for each pool, supervisor strategy restarts each individual actor if it fails. - // default is to restart the whole pool, which is not what we want: - // http://doc.akka.io/docs/akka/current/java/routing.html#Supervision - - fetchWsdlPoolRouter = getContext().actorOf(new SmallestMailboxPool(fetchWsdlPoolSize) - .withSupervisorStrategy(new OneForOneStrategy(-1, - Duration.Inf(), - (Throwable t) -> restart())) - .props(springExtension.props("fetchWsdlActor")), - FETCH_WSDL_ACTOR_ROUTER); - - fetchOpenApiPoolRouter = getContext().actorOf(new SmallestMailboxPool(fetchOpenApiPoolSize) - .withSupervisorStrategy(new OneForOneStrategy(-1, - Duration.Inf(), - (Throwable t) -> restart())) - .props(springExtension.props("fetchOpenApiActor")), - FETCH_OPENAPI_ACTOR_ROUTER); - - fetchRestPoolRouter = getContext().actorOf(new SmallestMailboxPool(fetchRestPoolSize) - .withSupervisorStrategy(new OneForOneStrategy(-1, - Duration.Inf(), - (Throwable t) -> restart())) - .props(springExtension.props("fetchRestActor")), - FETCH_REST_ACTOR_ROUTER); - - listMethodsPoolRouter = getContext().actorOf(new SmallestMailboxPool(listMethodsPoolSize) - .withSupervisorStrategy(new OneForOneStrategy(-1, - Duration.Inf(), - (Throwable t) -> restart())) - .props(springExtension.props("listMethodsActor", fetchWsdlPoolRouter, fetchOpenApiPoolRouter, fetchRestPoolRouter)), - LIST_METHODS_ACTOR_ROUTER); - - listClientsPoolRouter = getContext().actorOf(new SmallestMailboxPool(1) - .withSupervisorStrategy(new OneForOneStrategy(-1, - Duration.Inf(), - (Throwable t) -> restart())) - .props(springExtension.props("listClientsActor", listMethodsPoolRouter)), - LIST_CLIENTS_ACTOR_ROUTER); - - super.preStart(); - } - - @Override - protected boolean handleMessage(Object message) { - - if (START_COLLECTING.equals(message)) { - listClientsPoolRouter.tell(ListClientsActor.START_COLLECTING, getSelf()); - return true; - } else { - return false; - } - } -} diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/FetchOpenApiActor.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/FetchOpenApiActor.java deleted file mode 100644 index 81471e13..00000000 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/FetchOpenApiActor.java +++ /dev/null @@ -1,102 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.actors; - -import fi.vrk.xroad.catalog.collector.util.ClientTypeUtil; -import fi.vrk.xroad.catalog.collector.util.Endpoint; -import fi.vrk.xroad.catalog.collector.util.MethodListUtil; -import fi.vrk.xroad.catalog.collector.util.XRoadClient; -import fi.vrk.xroad.catalog.collector.util.XRoadRestServiceIdentifierType; -import fi.vrk.xroad.catalog.persistence.CatalogService; -import fi.vrk.xroad.catalog.persistence.entity.ServiceId; -import fi.vrk.xroad.catalog.persistence.entity.SubsystemId; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; - -import java.net.URL; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -@Component -@Scope("prototype") -@Slf4j -public class FetchOpenApiActor extends XRoadCatalogActor { - - private static AtomicInteger openApiCounter = new AtomicInteger(0); - - @Value("${xroad-catalog.security-server-host}") - private String xroadSecurityServerHost; - - @Value("${xroad-catalog.xroad-instance}") - private String xroadInstance; - - @Value("${xroad-catalog.member-code}") - private String memberCode; - - @Value("${xroad-catalog.member-class}") - private String memberClass; - - @Value("${xroad-catalog.subsystem-code}") - private String subsystemCode; - - @Value("${xroad-catalog.webservices-endpoint}") - private String webservicesEndpoint; - - @Autowired - protected CatalogService catalogService; - - private XRoadClient xroadClient; - - @Override - public void preStart() throws Exception { - xroadClient = new XRoadClient( - ClientTypeUtil.toSubsystem(xroadInstance, memberClass, memberCode, subsystemCode), - new URL(webservicesEndpoint)); - } - - @Override - protected boolean handleMessage(Object message) { - if (message instanceof XRoadRestServiceIdentifierType service) { - log.info("Fetching openApi [{}] {}", openApiCounter.addAndGet(1), ClientTypeUtil.toString(service)); - String openApi = xroadClient.getOpenApi(service, xroadSecurityServerHost, xroadInstance, memberClass, - memberCode, subsystemCode, catalogService); - catalogService.saveOpenApi(createSubsystemId(service), createServiceId(service), openApi); - List endpointList = MethodListUtil.getEndpointList(service); - catalogService.prepareEndpoints(createSubsystemId(service), createServiceId(service)); - for (Endpoint endpoint : endpointList) { - catalogService.saveEndpoint(createSubsystemId(service), createServiceId(service), endpoint.getMethod(), - endpoint.getPath()); - } - log.info("Saved openApi successfully"); - return true; - } else { - return false; - } - } - - private ServiceId createServiceId(XRoadRestServiceIdentifierType service) { - return new ServiceId(service.getServiceCode(), - service.getServiceVersion()); - } - - private SubsystemId createSubsystemId(XRoadRestServiceIdentifierType service) { - return new SubsystemId(service.getXRoadInstance(), - service.getMemberClass(), - service.getMemberCode(), - service.getSubsystemCode()); - } - -} diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/FetchRestActor.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/FetchRestActor.java deleted file mode 100644 index 0f385ce4..00000000 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/FetchRestActor.java +++ /dev/null @@ -1,103 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.actors; - -import fi.vrk.xroad.catalog.collector.util.ClientTypeUtil; -import fi.vrk.xroad.catalog.collector.util.Endpoint; -import fi.vrk.xroad.catalog.collector.util.MethodListUtil; -import fi.vrk.xroad.catalog.collector.util.XRoadRestServiceIdentifierType; -import fi.vrk.xroad.catalog.persistence.CatalogService; -import fi.vrk.xroad.catalog.persistence.entity.ServiceId; -import fi.vrk.xroad.catalog.persistence.entity.SubsystemId; -import lombok.extern.slf4j.Slf4j; -import org.json.JSONArray; -import org.json.JSONObject; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; - -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -@Component -@Scope("prototype") -@Slf4j -public class FetchRestActor extends XRoadCatalogActor { - - private static AtomicInteger restCounter = new AtomicInteger(0); - - private static final String METHOD = "method"; - - private static final String PATH = "path"; - - @Value("${xroad-catalog.security-server-host}") - private String xroadSecurityServerHost; - - @Value("${xroad-catalog.xroad-instance}") - private String xroadInstance; - - @Value("${xroad-catalog.member-code}") - private String memberCode; - - @Value("${xroad-catalog.member-class}") - private String memberClass; - - @Value("${xroad-catalog.subsystem-code}") - private String subsystemCode; - - @Value("${xroad-catalog.webservices-endpoint}") - private String webservicesEndpoint; - - @Autowired - protected CatalogService catalogService; - - @Override - protected boolean handleMessage(Object message) { - if (message instanceof XRoadRestServiceIdentifierType service) { - log.info("Fetching rest [{}] {}", restCounter.addAndGet(1), ClientTypeUtil.toString(service)); - List endpointList = MethodListUtil.getEndpointList(service); - String endpointData = "{\"endpoint_data\":"; - JSONArray endPointsJSONArray = new JSONArray(); - JSONObject endpointJson; - catalogService.prepareEndpoints(createSubsystemId(service), createServiceId(service)); - for (Endpoint endpoint : endpointList) { - endpointJson = new JSONObject(); - endpointJson.put(METHOD, endpoint.getMethod()); - endpointJson.put(PATH, endpoint.getPath()); - endPointsJSONArray.put(endpointJson); - catalogService.saveEndpoint(createSubsystemId(service), createServiceId(service), endpoint.getMethod(), - endpoint.getPath()); - } - endpointData += endPointsJSONArray + "}"; - catalogService.saveRest(createSubsystemId(service), createServiceId(service), endpointData); - log.info("Saved rest successfully"); - return true; - } else { - return false; - } - } - - private ServiceId createServiceId(XRoadRestServiceIdentifierType service) { - return new ServiceId(service.getServiceCode(), - service.getServiceVersion()); - } - - private SubsystemId createSubsystemId(XRoadRestServiceIdentifierType service) { - return new SubsystemId(service.getXRoadInstance(), - service.getMemberClass(), - service.getMemberCode(), - service.getSubsystemCode()); - } - -} diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/FetchWsdlActor.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/FetchWsdlActor.java deleted file mode 100644 index e3fdb9e3..00000000 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/FetchWsdlActor.java +++ /dev/null @@ -1,90 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.actors; - -import fi.vrk.xroad.catalog.collector.util.ClientTypeUtil; -import fi.vrk.xroad.catalog.collector.util.XRoadClient; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadServiceIdentifierType; -import fi.vrk.xroad.catalog.persistence.CatalogService; -import fi.vrk.xroad.catalog.persistence.entity.ServiceId; -import fi.vrk.xroad.catalog.persistence.entity.SubsystemId; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; - -import java.net.URL; -import java.util.concurrent.atomic.AtomicInteger; - -@Component -@Scope("prototype") -@Slf4j -public class FetchWsdlActor extends XRoadCatalogActor { - - private static AtomicInteger wsdlCounter = new AtomicInteger(0); - - @Value("${xroad-catalog.xroad-instance}") - private String xroadInstance; - - @Value("${xroad-catalog.member-code}") - private String memberCode; - - @Value("${xroad-catalog.member-class}") - private String memberClass; - - @Value("${xroad-catalog.subsystem-code}") - private String subsystemCode; - - @Value("${xroad-catalog.webservices-endpoint}") - private String webservicesEndpoint; - - @Autowired - protected CatalogService catalogService; - - private XRoadClient xroadClient; - - @Override - public void preStart() throws Exception { - xroadClient = new XRoadClient( - ClientTypeUtil.toSubsystem(xroadInstance, memberClass, memberCode, subsystemCode), - new URL(webservicesEndpoint)); - } - - @Override - protected boolean handleMessage(Object message) { - if (message instanceof XRoadServiceIdentifierType) { - XRoadServiceIdentifierType service = (XRoadServiceIdentifierType) message; - log.info("Fetching wsdl [{}] {}", wsdlCounter.addAndGet(1), ClientTypeUtil.toString(service)); - String wsdl = xroadClient.getWsdl(service, catalogService); - catalogService.saveWsdl(createSubsystemId(service), createServiceId(service), wsdl); - log.info("Saved wsdl successfully"); - return true; - } else { - return false; - } - } - - private ServiceId createServiceId(XRoadServiceIdentifierType service) { - return new ServiceId(service.getServiceCode(), - service.getServiceVersion()); - } - - private SubsystemId createSubsystemId(XRoadServiceIdentifierType service) { - return new SubsystemId(service.getXRoadInstance(), - service.getMemberClass(), - service.getMemberCode(), - service.getSubsystemCode()); - } - -} diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/ListClientsActor.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/ListClientsActor.java deleted file mode 100644 index 89eab055..00000000 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/ListClientsActor.java +++ /dev/null @@ -1,137 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.actors; - -import fi.vrk.xroad.catalog.collector.util.ClientListUtil; -import fi.vrk.xroad.catalog.collector.util.ClientTypeUtil; -import fi.vrk.xroad.catalog.collector.wsimport.ClientList; -import fi.vrk.xroad.catalog.collector.wsimport.ClientType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; -import fi.vrk.xroad.catalog.persistence.CatalogService; -import fi.vrk.xroad.catalog.persistence.entity.Member; -import fi.vrk.xroad.catalog.persistence.entity.MemberId; -import fi.vrk.xroad.catalog.persistence.entity.Subsystem; -import akka.actor.ActorRef; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; -import org.springframework.web.client.RestOperations; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.util.HashMap; -import java.util.HashSet; -import java.util.concurrent.atomic.AtomicInteger; - -@Component -@Scope("prototype") -@Slf4j -public class ListClientsActor extends XRoadCatalogActor { - - public static final String START_COLLECTING = "StartCollecting"; - - @Autowired - @Qualifier("listClientsRestOperations") - private RestOperations restOperations; - - @Autowired - protected CatalogService catalogService; - - @Value("${xroad-catalog.list-clients-host}") - private String host; - - @Value("${xroad-catalog.fetch-run-unlimited}") - private Boolean fetchUnlimited; - - @Value("${xroad-catalog.fetch-time-after-hour}") - private Integer fetchTimeAfterHour; - - @Value("${xroad-catalog.fetch-time-before-hour}") - private Integer fetchTimeBeforeHour; - - private static AtomicInteger clientCounter = new AtomicInteger(0); - - // supervisor-created pool of list clients actors - protected ActorRef listMethodsPoolRef; - - public ListClientsActor(ActorRef listMethodsPoolRef) { - this.listMethodsPoolRef = listMethodsPoolRef; - } - - @Override - public void preStart() throws Exception { - log.info("preStart {}", this.hashCode()); - super.preStart(); - } - - @Override - protected boolean handleMessage(Object message) - throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { - if (START_COLLECTING.equals(message)) { - if (Boolean.TRUE.equals(fetchUnlimited) || isTimeBetweenHours(fetchTimeAfterHour, fetchTimeBeforeHour)) { - return fetchClients(); - } - return true; - } else { - return false; - } - } - - private boolean fetchClients() { - String listClientsUrl = host + "/listClients"; - - log.info("Getting client list from {}", listClientsUrl); - ClientList clientList = ClientListUtil.clientListFromResponse(listClientsUrl, catalogService); - HashMap m = populateMapWithMembers(clientList); - catalogService.saveAllMembersAndSubsystems(m.values()); - - for (ClientType clientType : clientList.getMember()) { - listMethodsPoolRef.tell(clientType, getSelf()); - } - - log.info("all clients (" + (clientCounter.get() - 1) + ") sent to actor"); - - return true; - } - - private static boolean isTimeBetweenHours(int fetchHourAfter, int fetchHourBefore) { - LocalDateTime today = LocalDateTime.now(); - LocalDateTime fetchTimeFrom = LocalDate.now().atTime(fetchHourAfter, 0); - LocalDateTime fetchTimeTo = LocalDate.now().atTime(fetchHourBefore, 0); - return (today.isAfter(fetchTimeFrom) && today.isBefore(fetchTimeTo)); - } - - private static HashMap populateMapWithMembers(ClientList clientList) { - HashMap m = new HashMap<>(); - for (ClientType clientType : clientList.getMember()) { - log.info("{} - {}", clientCounter.addAndGet(1), ClientTypeUtil.toString(clientType)); - Member newMember = new Member(clientType.getId().getXRoadInstance(), clientType.getId() - .getMemberClass(), - clientType.getId().getMemberCode(), clientType.getName()); - newMember.setSubsystems(new HashSet<>()); - m.putIfAbsent(newMember.createKey(), newMember); - - if (XRoadObjectType.SUBSYSTEM.equals(clientType.getId().getObjectType())) { - Subsystem newSubsystem = new Subsystem(newMember, clientType.getId().getSubsystemCode()); - m.get(newMember.createKey()).getAllSubsystems().add(newSubsystem); - } - } - - return m; - } -} diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/ListMethodsActor.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/ListMethodsActor.java deleted file mode 100644 index ce5cb1e8..00000000 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/ListMethodsActor.java +++ /dev/null @@ -1,153 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.actors; - -import fi.vrk.xroad.catalog.collector.util.ClientTypeUtil; -import fi.vrk.xroad.catalog.collector.util.MethodListUtil; -import fi.vrk.xroad.catalog.collector.util.XRoadClient; -import fi.vrk.xroad.catalog.collector.util.XRoadRestServiceIdentifierType; -import fi.vrk.xroad.catalog.collector.wsimport.ClientType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadServiceIdentifierType; -import fi.vrk.xroad.catalog.persistence.CatalogService; -import fi.vrk.xroad.catalog.persistence.entity.Member; -import fi.vrk.xroad.catalog.persistence.entity.Service; -import fi.vrk.xroad.catalog.persistence.entity.Subsystem; -import akka.actor.ActorRef; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -@Component -@Scope("prototype") -@Slf4j -public class ListMethodsActor extends XRoadCatalogActor { - - private static AtomicInteger methodCounter = new AtomicInteger(0); - - private static final String SERVICE_TYPE_REST = "REST"; - - @Value("${xroad-catalog.security-server-host}") - private String xroadSecurityServerHost; - - @Value("${xroad-catalog.xroad-instance}") - private String xroadInstance; - - @Value("${xroad-catalog.member-code}") - private String memberCode; - - @Value("${xroad-catalog.member-class}") - private String memberClass; - - @Value("${xroad-catalog.subsystem-code}") - private String subsystemCode; - - @Value("${xroad-catalog.webservices-endpoint}") - private String webservicesEndpoint; - - @Value("${xroad-catalog.error-log-length-in-days}") - private Integer errorLogLengthInDays; - - @Value("${xroad-catalog.flush-log-time-after-hour}") - private Integer flushLogTimeAfterHour; - - @Value("${xroad-catalog.flush-log-time-before-hour}") - private Integer flushLogTimeBeforeHour; - - @Autowired - protected CatalogService catalogService; - - // supervisor-created pool of list methods actors - private ActorRef fetchWsdlPoolRef; - private ActorRef fetchOpenApiPoolRef; - private ActorRef fetchRestPoolRef; - private XRoadClient xroadClient; - - public ListMethodsActor(ActorRef fetchWsdlPoolRef, ActorRef fetchOpenApiPoolRef, ActorRef fetchRestPoolRef) { - this.fetchWsdlPoolRef = fetchWsdlPoolRef; - this.fetchOpenApiPoolRef = fetchOpenApiPoolRef; - this.fetchRestPoolRef = fetchRestPoolRef; - } - - @Override - public void preStart() throws Exception { - xroadClient = new XRoadClient( - ClientTypeUtil.toSubsystem(xroadInstance, memberClass, memberCode, subsystemCode), - new URL(webservicesEndpoint)); - } - - @Override - protected boolean handleMessage(Object message) { - if (message instanceof ClientType) { - log.info("{} onReceive {}", methodCounter.addAndGet(1), this.hashCode()); - ClientType clientType = (ClientType) message; - flushErrorLogs(); - if (XRoadObjectType.SUBSYSTEM.equals(clientType.getId().getObjectType())) { - saveSubsystemsAndServices(clientType); - } - return true; - } else { - return false; - } - } - - private void flushErrorLogs() { - if (MethodListUtil.shouldFlushLogEntries(flushLogTimeAfterHour, flushLogTimeBeforeHour)) { - catalogService.deleteOldErrorLogEntries(errorLogLengthInDays); - } - } - - private void saveSubsystemsAndServices(ClientType clientType) { - Subsystem subsystem = new Subsystem( - new Member(clientType.getId().getXRoadInstance(), clientType.getId().getMemberClass(), - clientType.getId().getMemberCode(), clientType.getName()), - clientType.getId().getSubsystemCode()); - - log.info("{} Handling subsystem {} ", methodCounter, subsystem); - - List restServices = MethodListUtil.methodListFromResponse(clientType, - xroadSecurityServerHost, xroadInstance, memberClass, memberCode, subsystemCode, catalogService); - log.info("Received all REST methods for client {} ", ClientTypeUtil.toString(clientType)); - - List soapServices = xroadClient.getMethods(clientType.getId(), catalogService); - log.info("Received all SOAP methods for client {} ", ClientTypeUtil.toString(clientType)); - - List services = new ArrayList<>(); - for (XRoadRestServiceIdentifierType service : restServices) { - services.add(new Service(subsystem, service.getServiceCode(), service.getServiceVersion())); - } - for (XRoadServiceIdentifierType service : soapServices) { - services.add(new Service(subsystem, service.getServiceCode(), service.getServiceVersion())); - } - - catalogService.saveServices(subsystem.createKey(), services); - - for (XRoadServiceIdentifierType service : soapServices) { - fetchWsdlPoolRef.tell(service, getSender()); - } - - for (XRoadRestServiceIdentifierType service : restServices) { - if (service.getServiceType().equalsIgnoreCase(SERVICE_TYPE_REST)) { - fetchRestPoolRef.tell(service, getSender()); - } else { - fetchOpenApiPoolRef.tell(service, getSender()); - } - } - } -} diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/OrganizationsActor.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/OrganizationsActor.java deleted file mode 100644 index 4930e868..00000000 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/OrganizationsActor.java +++ /dev/null @@ -1,193 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.actors; - -import fi.vrk.xroad.catalog.collector.util.ClientTypeUtil; -import fi.vrk.xroad.catalog.collector.util.MethodListUtil; -import fi.vrk.xroad.catalog.collector.util.XRoadClient; -import fi.vrk.xroad.catalog.collector.util.XRoadRestServiceIdentifierType; -import fi.vrk.xroad.catalog.collector.wsimport.ClientType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadServiceIdentifierType; -import fi.vrk.xroad.catalog.persistence.CatalogService; -import fi.vrk.xroad.catalog.persistence.entity.Member; -import fi.vrk.xroad.catalog.persistence.entity.Service; -import fi.vrk.xroad.catalog.persistence.entity.Subsystem; -import akka.actor.ActorRef; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -@Component -@Scope("prototype") -@Slf4j -public class OrganizationsActor extends XRoadCatalogActor { - - private static AtomicInteger methodCounter = new AtomicInteger(0); - - private boolean organizationsFetched = false; - - private boolean companiesFetched = false; - - private static final String SERVICE_TYPE_REST = "REST"; - - @Value("${xroad-catalog.security-server-host}") - private String xroadSecurityServerHost; - - @Value("${xroad-catalog.xroad-instance}") - private String xroadInstance; - - @Value("${xroad-catalog.member-code}") - private String memberCode; - - @Value("${xroad-catalog.member-class}") - private String memberClass; - - @Value("${xroad-catalog.subsystem-code}") - private String subsystemCode; - - @Value("${xroad-catalog.webservices-endpoint}") - private String webservicesEndpoint; - - @Value("${xroad-catalog.fetch-companies-time-after-hour}") - private Integer fetchCompaniesTimeAfterHour; - - @Value("${xroad-catalog.fetch-companies-time-before-hour}") - private Integer fetchCompaniesTimeBeforeHour; - - @Value("${xroad-catalog.fetch-companies-run-unlimited}") - private Boolean fetchCompaniesUnlimited; - - @Value("${xroad-catalog.error-log-length-in-days}") - private Integer errorLogLengthInDays; - - @Value("${xroad-catalog.flush-log-time-after-hour}") - private Integer flushLogTimeAfterHour; - - @Value("${xroad-catalog.flush-log-time-before-hour}") - private Integer flushLogTimeBeforeHour; - - @Autowired - protected CatalogService catalogService; - - // supervisor-created pool of list methods actors - private ActorRef fetchWsdlPoolRef; - private ActorRef fetchOpenApiPoolRef; - - private ActorRef fetchRestPoolRef; - private ActorRef fetchOrganizationsPoolRef; - private ActorRef fetchCompaniesPoolRef; - private XRoadClient xroadClient; - - public OrganizationsActor(ActorRef fetchWsdlPoolRef, - ActorRef fetchOpenApiPoolRef, - ActorRef fetchRestPoolRef, - ActorRef fetchOrganizationsPoolRef, - ActorRef fetchCompaniesPoolRef) { - this.fetchWsdlPoolRef = fetchWsdlPoolRef; - this.fetchOpenApiPoolRef = fetchOpenApiPoolRef; - this.fetchRestPoolRef = fetchRestPoolRef; - this.fetchOrganizationsPoolRef = fetchOrganizationsPoolRef; - this.fetchCompaniesPoolRef = fetchCompaniesPoolRef; - } - - @Override - public void preStart() throws Exception { - xroadClient = new XRoadClient( - ClientTypeUtil.toSubsystem(xroadInstance, memberClass, memberCode, subsystemCode), - new URL(webservicesEndpoint)); - } - - @Override - protected boolean handleMessage(Object message) { - if (message instanceof ClientType) { - log.info("{} onReceive {}", methodCounter.addAndGet(1), this.hashCode()); - ClientType clientType = (ClientType) message; - fetchOrganizations(clientType); - fetchCompanies(clientType); - flushErrorLogs(); - if (XRoadObjectType.SUBSYSTEM.equals(clientType.getId().getObjectType())) { - saveSubsystemsAndServices(clientType); - } - return true; - } else { - return false; - } - } - - private void fetchOrganizations(ClientType clientType) { - if (Boolean.FALSE.equals(organizationsFetched)) { - fetchOrganizationsPoolRef.tell(clientType, getSelf()); - organizationsFetched = true; - } - } - - private void fetchCompanies(ClientType clientType) { - if (MethodListUtil.shouldFetchCompanies(fetchCompaniesUnlimited, fetchCompaniesTimeAfterHour, - fetchCompaniesTimeBeforeHour)) { - if (Boolean.FALSE.equals(companiesFetched)) { - fetchCompaniesPoolRef.tell(clientType, getSelf()); - companiesFetched = true; - } - } - } - - private void flushErrorLogs() { - if (MethodListUtil.shouldFlushLogEntries(flushLogTimeAfterHour, flushLogTimeBeforeHour)) { - catalogService.deleteOldErrorLogEntries(errorLogLengthInDays); - } - } - - private void saveSubsystemsAndServices(ClientType clientType) { - Subsystem subsystem = new Subsystem( - new Member(clientType.getId().getXRoadInstance(), clientType.getId().getMemberClass(), - clientType.getId().getMemberCode(), clientType.getName()), - clientType.getId().getSubsystemCode()); - log.info("{} Handling subsystem {} ", methodCounter, subsystem); - - List restServices = MethodListUtil.methodListFromResponse(clientType, - xroadSecurityServerHost, xroadInstance, memberClass, memberCode, subsystemCode, catalogService); - log.info("Received all REST methods for client {} ", ClientTypeUtil.toString(clientType)); - - List soapServices = xroadClient.getMethods(clientType.getId(), catalogService); - log.info("Received all SOAP methods for client {} ", ClientTypeUtil.toString(clientType)); - - List services = new ArrayList<>(); - for (XRoadRestServiceIdentifierType service : restServices) { - services.add(new Service(subsystem, service.getServiceCode(), service.getServiceVersion())); - } - for (XRoadServiceIdentifierType service : soapServices) { - services.add(new Service(subsystem, service.getServiceCode(), service.getServiceVersion())); - } - - catalogService.saveServices(subsystem.createKey(), services); - - for (XRoadServiceIdentifierType service : soapServices) { - fetchWsdlPoolRef.tell(service, getSender()); - } - - for (XRoadRestServiceIdentifierType service : restServices) { - if (service.getServiceType().equalsIgnoreCase(SERVICE_TYPE_REST)) { - fetchRestPoolRef.tell(service, getSender()); - } else { - fetchOpenApiPoolRef.tell(service, getSender()); - } - } - } -} diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/OrganizationsSupervisor.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/OrganizationsSupervisor.java deleted file mode 100644 index 96ae6a19..00000000 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/OrganizationsSupervisor.java +++ /dev/null @@ -1,160 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.actors; - -import akka.actor.ActorRef; -import akka.actor.OneForOneStrategy; -import akka.routing.SmallestMailboxPool; -import fi.vrk.xroad.catalog.collector.extension.SpringExtension; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Profile; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; -import scala.concurrent.duration.Duration; - -import static akka.actor.SupervisorStrategy.restart; - -/** - * Supervisor to get list of all clients in system and initiate a ClientActor - * for each - *

- * A router is configured at startup time, managing a pool of task actors. - */ -@Component("OrganizationsSupervisor") -@Scope("prototype") -@Slf4j -@Profile("fi") -public class OrganizationsSupervisor extends XRoadCatalogActor { - - public static final String START_COLLECTING = "StartCollecting"; - - public static final String LIST_CLIENTS_ACTOR_ROUTER = "list-clients-actor-router"; - public static final String ORGANIZATIONS_ACTOR_ROUTER = "organizations-actor-router"; - public static final String FETCH_WSDL_ACTOR_ROUTER = "fetch-wsdl-actor-router"; - public static final String FETCH_OPENAPI_ACTOR_ROUTER = "fetch-openapi-actor-router"; - public static final String FETCH_REST_ACTOR_ROUTER = "fetch-rest-actor-router"; - public static final String FETCH_ORGANIZATIONS_ACTOR_ROUTER = "fetch-organizations-actor-router"; - public static final String FETCH_COMPANIES_ACTOR_ROUTER = "fetch-companies-actor-router"; - - @Autowired - private SpringExtension springExtension; - - private ActorRef listClientsPoolRouter; - - private ActorRef organizationsPoolRouter; - private ActorRef fetchWsdlPoolRouter; - private ActorRef fetchOpenApiPoolRouter; - - private ActorRef fetchRestPoolRouter; - private ActorRef fetchOrganizationsPoolRouter; - private ActorRef fetchCompaniesPoolRouter; - - @Value("${xroad-catalog.list-methods-pool-size}") - private int listMethodsPoolSize; - - @Value("${xroad-catalog.fetch-wsdl-pool-size}") - private int fetchWsdlPoolSize; - - @Value("${xroad-catalog.fetch-openapi-pool-size}") - private int fetchOpenApiPoolSize; - - @Value("${xroad-catalog.fetch-rest-pool-size}") - private int fetchRestPoolSize; - - @Value("${xroad-catalog.fetch-organizations-pool-size}") - private int fetchOrganizationsPoolSize; - - @Value("${xroad-catalog.fetch-companies-pool-size}") - private int fetchCompaniesPoolSize; - - @Override - public void preStart() throws Exception { - - log.info("Starting up"); - - // prepare each pool. Give ActorRefs to other pools, as needed. - // To make this possible, create pools in correct order - - // for each pool, supervisor strategy restarts each individual actor if it - // fails. - // default is to restart the whole pool, which is not what we want: - // http://doc.akka.io/docs/akka/current/java/routing.html#Supervision - - fetchWsdlPoolRouter = getContext().actorOf(new SmallestMailboxPool(fetchWsdlPoolSize) - .withSupervisorStrategy(new OneForOneStrategy(-1, - Duration.Inf(), - (Throwable t) -> restart())) - .props(springExtension.props("fetchWsdlActor")), - FETCH_WSDL_ACTOR_ROUTER); - - fetchOpenApiPoolRouter = getContext().actorOf(new SmallestMailboxPool(fetchOpenApiPoolSize) - .withSupervisorStrategy(new OneForOneStrategy(-1, - Duration.Inf(), - (Throwable t) -> restart())) - .props(springExtension.props("fetchOpenApiActor")), - FETCH_OPENAPI_ACTOR_ROUTER); - - fetchRestPoolRouter = getContext().actorOf(new SmallestMailboxPool(fetchRestPoolSize) - .withSupervisorStrategy(new OneForOneStrategy(-1, - Duration.Inf(), - (Throwable t) -> restart())) - .props(springExtension.props("fetchRestActor")), - FETCH_REST_ACTOR_ROUTER); - - fetchOrganizationsPoolRouter = getContext().actorOf(new SmallestMailboxPool(fetchOrganizationsPoolSize) - .withSupervisorStrategy(new OneForOneStrategy(-1, - Duration.Inf(), - (Throwable t) -> restart())) - .props(springExtension.props("fetchOrganizationsActor")), - FETCH_ORGANIZATIONS_ACTOR_ROUTER); - - fetchCompaniesPoolRouter = getContext().actorOf(new SmallestMailboxPool(fetchCompaniesPoolSize) - .withSupervisorStrategy(new OneForOneStrategy(-1, - Duration.Inf(), - (Throwable t) -> restart())) - .props(springExtension.props("fetchCompaniesActor")), - FETCH_COMPANIES_ACTOR_ROUTER); - - organizationsPoolRouter = getContext().actorOf(new SmallestMailboxPool(listMethodsPoolSize) - .withSupervisorStrategy(new OneForOneStrategy(-1, - Duration.Inf(), - (Throwable t) -> restart())) - .props(springExtension.props("organizationsActor", fetchWsdlPoolRouter, - fetchOpenApiPoolRouter, - fetchRestPoolRouter, fetchOrganizationsPoolRouter, - fetchCompaniesPoolRouter)), - ORGANIZATIONS_ACTOR_ROUTER); - - listClientsPoolRouter = getContext().actorOf(new SmallestMailboxPool(1) - .withSupervisorStrategy(new OneForOneStrategy(-1, - Duration.Inf(), - (Throwable t) -> restart())) - .props(springExtension.props("listClientsActor", organizationsPoolRouter)), - LIST_CLIENTS_ACTOR_ROUTER); - - super.preStart(); - } - - @Override - protected boolean handleMessage(Object message) { - - if (START_COLLECTING.equals(message)) { - listClientsPoolRouter.tell(ListClientsActor.START_COLLECTING, getSelf()); - return true; - } else { - return false; - } - } -} diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/XRoadCatalogActor.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/XRoadCatalogActor.java deleted file mode 100644 index 6d3d0507..00000000 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/XRoadCatalogActor.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.actors; - -import akka.actor.UntypedAbstractActor; -import fi.vrk.xroad.catalog.collector.util.CatalogCollectorRuntimeException; -import akka.actor.Terminated; -import lombok.extern.slf4j.Slf4j; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; - -@Slf4j -public abstract class XRoadCatalogActor extends UntypedAbstractActor { - - public static final String START_COLLECTING = "StartCollecting"; - - protected abstract boolean handleMessage(Object message) throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException; - - @Override - public void onReceive(Object message) throws Exception { - log.info("{} handleXRoadCatalogMessage {}", this.hashCode()); - if (handleMessage(message)) { - return; - } else if (message instanceof Terminated) { - throw new CatalogCollectorRuntimeException("Terminated: " + message); - } else { - log.error("Unable to handle message {}", message); - throw new CatalogCollectorRuntimeException("Unable to handle message"); - } - } - -} diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/configuration/ApplicationConfiguration.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/configuration/ApplicationConfiguration.java index e4c483ce..cfe356a6 100644 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/configuration/ApplicationConfiguration.java +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/configuration/ApplicationConfiguration.java @@ -12,91 +12,15 @@ */ package fi.vrk.xroad.catalog.collector.configuration; -import fi.vrk.xroad.catalog.collector.extension.SpringExtension; -import akka.actor.ActorSystem; -import com.typesafe.config.Config; -import com.typesafe.config.ConfigFactory; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; -import org.springframework.context.ApplicationContext; -import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.ImportResource; import org.springframework.context.annotation.Lazy; -import org.springframework.http.client.SimpleClientHttpRequestFactory; -import org.springframework.web.client.RestOperations; -import org.springframework.web.client.RestTemplate; @Configuration @Lazy @ComponentScan(basePackages = { - "fi.vrk.xroad.catalog.collector.actors", - "fi.vrk.xroad.catalog.collector.extension", - "fi.vrk.xroad.catalog" - + ".persistence" }) -@ImportResource({ "classpath:META-INF/cxf/cxf.xml" }) -@Slf4j -public class ApplicationConfiguration extends SpringBootServletInitializer { - - @Value("${xroad-catalog.collector-interval-min}") - private Long collectorInterval; - - - @Bean - public ActorSystem actorSystem(ApplicationContext applicationContext, - SpringExtension springExtension) { - - ActorSystem system = ActorSystem - .create("AkkaTaskProcessing", akkaConfiguration()); - - // Initialize the application context in the Akka Spring Extension - springExtension.initialize(applicationContext); - return system; - } - - - @Bean - public Config akkaConfiguration() { - return ConfigFactory.load(); - } - - @Bean - @Qualifier("listClientsRestOperations") - public RestOperations getRestOperations() { - return createTimeoutingRestTemplate(); - } - - @Bean - @Qualifier("wsdlRestOperations") - public RestOperations getDynamicWsdlRestOperations() { - log.info("-------------- Configuration"); - return createTimeoutingRestTemplate(); - } - - @Bean - public Long getCollectorInterval() { - return collectorInterval; - } - - private static final int TIMEOUT = 10 * 60 * 1000; // 10 minutes - private RestTemplate createTimeoutingRestTemplate() { - RestTemplate rt = new RestTemplate(); - setTimeout(rt, TIMEOUT); - return rt; - } - - private void setTimeout(RestTemplate restTemplate, int timeout) { - //Explicitly setting ClientHttpRequestFactory instance to - //SimpleClientHttpRequestFactory instance to leverage - //set*Timeout methods - restTemplate.setRequestFactory(new SimpleClientHttpRequestFactory()); - SimpleClientHttpRequestFactory rf = (SimpleClientHttpRequestFactory) restTemplate - .getRequestFactory(); - rf.setReadTimeout(timeout); - rf.setConnectTimeout(timeout); - } + "fi.vrk.xroad.catalog.persistence" +}) +public class ApplicationConfiguration { } diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/configuration/TaskPoolConfiguration.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/configuration/TaskPoolConfiguration.java new file mode 100644 index 00000000..6e13bd23 --- /dev/null +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/configuration/TaskPoolConfiguration.java @@ -0,0 +1,136 @@ +/** + * + * The MIT License + * + * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) + * Copyright (c) 2016-2023 Finnish Digital Agency + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +package fi.vrk.xroad.catalog.collector.configuration; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +import lombok.Getter; + +@Getter +@Configuration +public class TaskPoolConfiguration { + + // X-Road instance parameters + + @Value("${xroad-catalog.xroad-instance}") + private String xroadInstance; + + @Value("${xroad-catalog.member-class}") + private String memberClass; + + @Value("${xroad-catalog.member-code}") + private String memberCode; + + @Value("${xroad-catalog.subsystem-code}") + private String subsystemCode; + + // Security server URLs + + @Value("${xroad-catalog.security-server-host}") + private String securityServerHost; + + @Value("${xroad-catalog.webservices-endpoint}") + private String webservicesEndpoint; + + @Value("${xroad-catalog.list-clients-host}") + private String listClientsHost; + + @Value("${xroad-catalog.fetch-wsdl-host}") + private String fetchWsdlHost; + + @Value("${xroad-catalog.fetch-openapi-host}") + private String fetchOpenapiHost; + + // Parameters related to the "fi" profile + + @Value("${xroad-catalog.fetch-organizations-url}") + private String fetchOrganizationsUrl; + + @Value("${xroad-catalog.fetch-companies-url}") + private String fetchCompaniesUrl; + + @Value("${xroad-catalog.max-organizations-per-request:100}") + private int maxOrganizationsPerRequest; + + @Value("${xroad-catalog.fetch-companies-limit:1000}") + private int fetchCompaniesLimit; + + @Value("${xroad-catalog.fetch-organizations-limit:2000}") + private int fetchOrganizationsLimit; + + @Value("${xroad-catalog.fetch-companies-run-unlimited:false}") + private boolean fetchCompaniesRunUnlimited; + + @Value("${xroad-catalog.fetch-companies-time-after-hour:3}") + private int fetchCompaniesTimeAfterHour; + + @Value("${xroad-catalog.fetch-companies-time-before-hour:4}") + private int fetchCompaniesTimeBeforeHour; + + @Value("${xroad-catalog.flush-log-time-after-hour:3}") + private int flushLogTimeAfterHour; + + @Value("${xroad-catalog.flush-log-time-before-hour:4}") + private int flushLogTimeBeforeHour; + + @Value("${xroad-catalog.error-log-length-in-days:90}") + private int errorLogLengthInDays; + + @Value("${xroad-catalog.fetch-run-unlimited:false}") + private boolean fetchRunUnlimited; + + @Value("${xroad-catalog.fetch-time-after-hour:3}") + private int fetchTimeAfterHour; + + @Value("${xroad-catalog.fetch-time-before-hour:4}") + private int fetchTimeBeforeHour; + + // Collector internal parameters + + @Value("${xroad-catalog.collector-interval-min:20}") + private long collectorInterval; + + @Value("${xroad-catalog.list-methods-pool-size:50}") + private int listMethodsPoolSize; + + @Value("${xroad-catalog.fetch-wsdl-pool-size:10}") + private int fetchWsdlPoolSize; + + @Value("${xroad-catalog.fetch-openapi-pool-size:10}") + private int fetchOpenapiPoolSize; + + @Value("${xroad-catalog.fetch-rest-pool-size:10}") + private int fetchRestPoolSize; + + @Value("${xroad-catalog.fetch-organizations-pool-size:10}") + private int fetchOrganizationsPoolSize; + + @Value("${xroad-catalog.fetch-companies-pool-size:10}") + private int fetchCompaniesPoolSize; + +} diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/extension/SpringActorProducer.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/extension/SpringActorProducer.java deleted file mode 100644 index 902c967a..00000000 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/extension/SpringActorProducer.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.extension; - -import akka.actor.Actor; -import akka.actor.IndirectActorProducer; -import org.springframework.context.ApplicationContext; - -public class SpringActorProducer implements IndirectActorProducer { - - private final ApplicationContext applicationContext; - private final String actorBeanName; - - // arguments to use when creating a bean instance using explicit arguments - private final Object[] args; - - public SpringActorProducer(ApplicationContext applicationContext, - String actorBeanName) { - this.applicationContext = applicationContext; - this.actorBeanName = actorBeanName; - args = null; - } - - public SpringActorProducer(ApplicationContext applicationContext, - String actorBeanName, - Object... args) { - this.applicationContext = applicationContext; - this.actorBeanName = actorBeanName; - this.args = args; - } - - @Override - public Actor produce() { - if (args != null) { - return (Actor) applicationContext.getBean(actorBeanName, args); - } else { - return (Actor) applicationContext.getBean(actorBeanName); - } - } - - @Override - public Class actorClass() { - return (Class) applicationContext.getType(actorBeanName); - } -} - diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/extension/SpringExtension.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/extension/SpringExtension.java deleted file mode 100644 index c48bae4f..00000000 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/extension/SpringExtension.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.extension; - -import akka.actor.Extension; -import akka.actor.Props; -import org.springframework.context.ApplicationContext; -import org.springframework.stereotype.Component; - -@Component -public class SpringExtension implements Extension { - - private ApplicationContext applicationContext; - - @SuppressWarnings("checkstyle:HiddenField") - public void initialize(ApplicationContext applicationContext) { - this.applicationContext = applicationContext; - } - - public Props props(String actorBeanName) { - return Props.create(SpringActorProducer.class, - applicationContext, actorBeanName); - } - - public Props props(String actorBeanName, Object... args) { - return Props.create(SpringActorProducer.class, - applicationContext, actorBeanName, args); - } - -} diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/BaseFetchTask.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/BaseFetchTask.java new file mode 100644 index 00000000..047338bc --- /dev/null +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/BaseFetchTask.java @@ -0,0 +1,111 @@ +/** + * + * The MIT License + * + * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) + * Copyright (c) 2016-2023 Finnish Digital Agency + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +package fi.vrk.xroad.catalog.collector.tasks; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Semaphore; + +import org.springframework.context.ApplicationContext; + +import fi.vrk.xroad.catalog.collector.util.XRoadRestServiceIdentifierType; +import fi.vrk.xroad.catalog.collector.wsimport.XRoadServiceIdentifierType; +import fi.vrk.xroad.catalog.persistence.CatalogService; +import fi.vrk.xroad.catalog.persistence.entity.ServiceId; +import fi.vrk.xroad.catalog.persistence.entity.SubsystemId; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class BaseFetchTask { + + protected final CatalogService catalogService; + + private final BlockingQueue inputQueue; + + private final Semaphore semaphore; + + protected BaseFetchTask(final ApplicationContext applicationContext, final BlockingQueue inputQueue, + final int poolSize) { + this.catalogService = applicationContext.getBean(CatalogService.class); + + this.inputQueue = inputQueue; + + this.semaphore = new Semaphore(poolSize); + } + + public void run() { + log.info("Starting {} with pool size {}", getClass().getSimpleName(), semaphore.availablePermits()); + try { + while (true) { + log.debug("Polling for input ... "); + + // take() blocks until an element becomes available or it gets interrupted + T input = inputQueue.take(); + semaphore.acquire(); + Thread.ofVirtual().start(() -> wrappedFetch(input)); + } + } catch (InterruptedException e) { + log.warn("Interrupted while handling inputs, stopping {}", getClass().getSimpleName(), e); + Thread.currentThread().interrupt(); + } + } + + private void wrappedFetch(final T input) { + try { + fetch(input); + } catch (Exception e) { + log.error("Error fetching data", e); + } finally { + semaphore.release(); + } + } + + protected abstract void fetch(T input); + + protected ServiceId createServiceId(XRoadServiceIdentifierType service) { + return new ServiceId(service.getServiceCode(), + service.getServiceVersion()); + } + + protected ServiceId createServiceId(XRoadRestServiceIdentifierType service) { + return new ServiceId(service.getServiceCode(), + service.getServiceVersion()); + } + + protected SubsystemId createSubsystemId(XRoadServiceIdentifierType service) { + return new SubsystemId(service.getXRoadInstance(), + service.getMemberClass(), + service.getMemberCode(), + service.getSubsystemCode()); + } + + protected SubsystemId createSubsystemId(XRoadRestServiceIdentifierType service) { + return new SubsystemId(service.getXRoadInstance(), + service.getMemberClass(), + service.getMemberCode(), + service.getSubsystemCode()); + } +} diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/FetchCompaniesActor.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchCompaniesTask.java similarity index 69% rename from xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/FetchCompaniesActor.java rename to xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchCompaniesTask.java index 2f468d80..4e594141 100644 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/FetchCompaniesActor.java +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchCompaniesTask.java @@ -1,17 +1,41 @@ /** - * The MIT License * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency + * The MIT License * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) + * Copyright (c) 2016-2023 Finnish Digital Agency * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package fi.vrk.xroad.catalog.collector.actors; +package fi.vrk.xroad.catalog.collector.tasks; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Semaphore; +import org.json.JSONArray; +import org.json.JSONObject; +import org.springframework.context.ApplicationContext; + +import fi.vrk.xroad.catalog.collector.configuration.TaskPoolConfiguration; import fi.vrk.xroad.catalog.collector.util.OrganizationUtil; import fi.vrk.xroad.catalog.collector.wsimport.ClientType; import fi.vrk.xroad.catalog.persistence.CatalogService; @@ -29,42 +53,59 @@ import fi.vrk.xroad.catalog.persistence.entity.RegisteredEntry; import fi.vrk.xroad.catalog.persistence.entity.RegisteredOffice; import lombok.extern.slf4j.Slf4j; -import org.json.JSONArray; -import org.json.JSONObject; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; -import java.util.ArrayList; -import java.util.List; -@Component -@Scope("prototype") @Slf4j -public class FetchCompaniesActor extends XRoadCatalogActor { +public class FetchCompaniesTask { + + private final String fetchCompaniesUrl; + + private final Integer fetchCompaniesLimit; + + private final CatalogService catalogService; + + private final CompanyService companyService; + + private final BlockingQueue fetchCompaniesQueue; - @Value("${xroad-catalog.fetch-companies-url}") - private String fetchCompaniesUrl; + private final TaskPoolConfiguration taskPoolConfiguration; - @Value("${xroad-catalog.fetch-companies-limit}") - private Integer fetchCompaniesLimit; + private final Semaphore semaphore; - @Autowired - protected CatalogService catalogService; + public FetchCompaniesTask(final ApplicationContext applicationContext, + final BlockingQueue fetchCompaniesQueue) { + this.catalogService = applicationContext.getBean(CatalogService.class); + this.companyService = applicationContext.getBean(CompanyService.class); - @Autowired - protected CompanyService companyService; + this.fetchCompaniesQueue = fetchCompaniesQueue; - @Override - public void preStart() throws Exception { - // This method is here just to override the method from the superclass + this.taskPoolConfiguration = applicationContext.getBean(TaskPoolConfiguration.class); + this.fetchCompaniesUrl = taskPoolConfiguration.getFetchCompaniesUrl(); + this.fetchCompaniesLimit = taskPoolConfiguration.getFetchCompaniesLimit(); + + this.semaphore = new Semaphore(taskPoolConfiguration.getFetchCompaniesPoolSize()); + + } + + public void run() { + log.info("Starting {} with pool size {}", getClass().getSimpleName(), semaphore.availablePermits()); + try { + while (true) { + log.debug("Waiting for data ... "); + + // take() blocks until an element becomes available or it gets interrupted + ClientType client = fetchCompaniesQueue.take(); + semaphore.acquire(); + Thread.ofVirtual().start(() -> fetchCompaniesForCLient(client)); + } + } catch (InterruptedException e) { + log.warn("Interrupted while waiting for data, stopping {}", getClass().getSimpleName(), e); + Thread.currentThread().interrupt(); + } } - @Override - protected boolean handleMessage(Object message) { - if (message instanceof ClientType) { - ClientType clientType = (ClientType) message; - JSONObject companiesJson = OrganizationUtil.getCompanies(clientType, fetchCompaniesLimit, fetchCompaniesUrl, + protected void fetchCompaniesForCLient(final ClientType client) { + try { + JSONObject companiesJson = OrganizationUtil.getCompanies(client, fetchCompaniesLimit, fetchCompaniesUrl, catalogService); JSONArray companiesArray = companiesJson.optJSONArray("results"); int numberOfCompanies = companiesArray.length(); @@ -72,14 +113,15 @@ protected boolean handleMessage(Object message) { companiesArray.forEach(item -> { JSONObject company = (JSONObject) item; String businessCode = company.optString("businessId"); - JSONObject companyJson = OrganizationUtil.getCompany(clientType, fetchCompaniesUrl, businessCode, + JSONObject companyJson = OrganizationUtil.getCompany(client, fetchCompaniesUrl, businessCode, catalogService); saveData(companyJson.optJSONArray("results")); }); log.info("Successfully saved data for {} companies", numberOfCompanies); - return true; - } else { - return false; + } catch (Exception e) { + log.error("Error while fetching companies for client {}", client, e); + } finally { + semaphore.release(); } } diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchOpenApiTask.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchOpenApiTask.java new file mode 100644 index 00000000..fe1ac80d --- /dev/null +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchOpenApiTask.java @@ -0,0 +1,97 @@ +/** + * + * The MIT License + * + * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) + * Copyright (c) 2016-2023 Finnish Digital Agency + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +package fi.vrk.xroad.catalog.collector.tasks; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.List; +import java.util.concurrent.BlockingQueue; + +import org.springframework.context.ApplicationContext; + +import fi.vrk.xroad.catalog.collector.configuration.TaskPoolConfiguration; +import fi.vrk.xroad.catalog.collector.util.ClientTypeUtil; +import fi.vrk.xroad.catalog.collector.util.Endpoint; +import fi.vrk.xroad.catalog.collector.util.MethodListUtil; +import fi.vrk.xroad.catalog.collector.util.XRoadClient; +import fi.vrk.xroad.catalog.collector.util.XRoadRestServiceIdentifierType; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class FetchOpenApiTask extends BaseFetchTask { + + private final String xroadSecurityServerHost; + + private final String xroadInstance; + + private final String memberCode; + + private final String memberClass; + + private final String subsystemCode; + + private final String webservicesEndpoint; + + private final XRoadClient xroadClient; + + public FetchOpenApiTask(final ApplicationContext applicationContext, + final BlockingQueue openApiServices) throws URISyntaxException { + super(applicationContext, openApiServices, + applicationContext.getBean(TaskPoolConfiguration.class).getFetchOpenapiPoolSize()); + + TaskPoolConfiguration taskPoolConfiguration = applicationContext.getBean(TaskPoolConfiguration.class); + this.xroadSecurityServerHost = taskPoolConfiguration.getSecurityServerHost(); + this.xroadInstance = taskPoolConfiguration.getXroadInstance(); + this.memberCode = taskPoolConfiguration.getMemberCode(); + this.memberClass = taskPoolConfiguration.getMemberClass(); + this.subsystemCode = taskPoolConfiguration.getSubsystemCode(); + this.webservicesEndpoint = taskPoolConfiguration.getWebservicesEndpoint(); + + this.xroadClient = new XRoadClient( + ClientTypeUtil.toSubsystem(xroadInstance, memberClass, memberCode, subsystemCode), + new URI(webservicesEndpoint)); + } + + @Override + protected void fetch(final XRoadRestServiceIdentifierType service) { + try { + log.info("Fetching OpenApi for {}", ClientTypeUtil.toString(service)); + String openApi = xroadClient.getOpenApi(service, xroadSecurityServerHost, xroadInstance, memberClass, + memberCode, subsystemCode, catalogService); + catalogService.saveOpenApi(createSubsystemId(service), createServiceId(service), openApi); + List endpointList = MethodListUtil.getEndpointList(service); + catalogService.prepareEndpoints(createSubsystemId(service), createServiceId(service)); + for (Endpoint endpoint : endpointList) { + catalogService.saveEndpoint(createSubsystemId(service), createServiceId(service), endpoint.getMethod(), + endpoint.getPath()); + } + log.info("Saved OpenApi for {} successfully", ClientTypeUtil.toString(service)); + } catch (Exception e) { + log.error("Failed to fetch OpenAPI for {}", ClientTypeUtil.toString(service), e); + } + } +} diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/FetchOrganizationsActor.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchOrganizationsTask.java similarity index 78% rename from xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/FetchOrganizationsActor.java rename to xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchOrganizationsTask.java index b934645d..b2a9cbb7 100644 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/actors/FetchOrganizationsActor.java +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchOrganizationsTask.java @@ -1,19 +1,45 @@ /** - * The MIT License * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency + * The MIT License * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) + * Copyright (c) 2016-2023 Finnish Digital Agency * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package fi.vrk.xroad.catalog.collector.actors; +package fi.vrk.xroad.catalog.collector.tasks; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicInteger; +import org.json.JSONArray; +import org.json.JSONObject; +import org.springframework.context.ApplicationContext; + +import fi.vrk.xroad.catalog.collector.configuration.TaskPoolConfiguration; import fi.vrk.xroad.catalog.collector.util.OrganizationUtil; import fi.vrk.xroad.catalog.collector.wsimport.ClientType; +import fi.vrk.xroad.catalog.persistence.CatalogService; import fi.vrk.xroad.catalog.persistence.OrganizationService; import fi.vrk.xroad.catalog.persistence.entity.Address; import fi.vrk.xroad.catalog.persistence.entity.Email; @@ -34,80 +60,87 @@ import fi.vrk.xroad.catalog.persistence.entity.StreetAddressMunicipalityName; import fi.vrk.xroad.catalog.persistence.entity.StreetAddressPostOffice; import fi.vrk.xroad.catalog.persistence.entity.WebPage; -import fi.vrk.xroad.catalog.persistence.CatalogService; import lombok.extern.slf4j.Slf4j; -import org.json.JSONArray; -import org.json.JSONObject; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; -@Component -@Scope("prototype") @Slf4j -public class FetchOrganizationsActor extends XRoadCatalogActor { +public class FetchOrganizationsTask { - @Value("${xroad-catalog.fetch-organizations-url}") private String fetchOrganizationsUrl; - @Value("${xroad-catalog.max-organizations-per-request}") private Integer maxOrganizationsPerRequest; - @Value("${xroad-catalog.fetch-organizations-limit}") private Integer fetchOrganizationsLimit; - @Autowired - protected CatalogService catalogService; + private final CatalogService catalogService; + + private final OrganizationService organizationService; + + private final BlockingQueue fetchOrganizationsQueue; + + private final TaskPoolConfiguration taskPoolConfiguration; + + private final Semaphore semaphore; + + public FetchOrganizationsTask(final ApplicationContext applicationContext, + final BlockingQueue fetchOrganizationsQueue) { + this.catalogService = applicationContext.getBean(CatalogService.class); + this.organizationService = applicationContext.getBean(OrganizationService.class); + + this.fetchOrganizationsQueue = fetchOrganizationsQueue; + + this.taskPoolConfiguration = applicationContext.getBean(TaskPoolConfiguration.class); + this.fetchOrganizationsUrl = taskPoolConfiguration.getFetchOrganizationsUrl(); + this.maxOrganizationsPerRequest = taskPoolConfiguration.getMaxOrganizationsPerRequest(); + this.fetchOrganizationsLimit = taskPoolConfiguration.getFetchOrganizationsLimit(); - @Autowired - protected OrganizationService organizationService; + this.semaphore = new Semaphore(taskPoolConfiguration.getFetchOrganizationsPoolSize()); - @Override - public void preStart() throws Exception { - // This method is here just to override the method from the superclass } - @Override - protected boolean handleMessage(Object message) - throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { - if (message instanceof ClientType) { - List organizationIds = OrganizationUtil.getOrganizationIdsList((ClientType) message, - fetchOrganizationsUrl, + public void run() { + log.info("Starting {} with pool size {}", getClass().getSimpleName(), semaphore.availablePermits()); + try { + while (true) { + log.debug("Waiting for data ... "); + + // take() blocks until an element becomes available or it gets interrupted + ClientType client = fetchOrganizationsQueue.take(); + semaphore.acquire(); + Thread.ofVirtual().start(() -> fetchOrganizationsForClient(client)); + } + } catch (InterruptedException e) { + log.warn("Interrupted while waiting for data, stopping {}", getClass().getSimpleName(), e); + Thread.currentThread().interrupt(); + } + } + + protected void fetchOrganizationsForClient(final ClientType client) { + try { + List organizationIds = OrganizationUtil.getOrganizationIdsList(client, fetchOrganizationsUrl, fetchOrganizationsLimit, catalogService); int numberOfOrganizations = organizationIds.size(); log.info("Fetched {} organization GUIDs from {}", numberOfOrganizations, fetchOrganizationsUrl); AtomicInteger elementCount = new AtomicInteger(); - AtomicInteger batchCount = new AtomicInteger(); List guidsList = new ArrayList<>(); organizationIds.forEach(id -> { guidsList.add(id); elementCount.getAndIncrement(); if (elementCount.get() % maxOrganizationsPerRequest == 0) { - batchCount.getAndIncrement(); - saveBatch(OrganizationUtil.getDataByIds((ClientType) message, guidsList, fetchOrganizationsUrl, - catalogService)); + saveBatch(OrganizationUtil.getDataByIds(client, guidsList, fetchOrganizationsUrl, catalogService)); guidsList.clear(); } - if (elementCount.get() == organizationIds.size()) { - batchCount.getAndIncrement(); - saveBatch(OrganizationUtil.getDataByIds((ClientType) message, guidsList, fetchOrganizationsUrl, - catalogService)); + if (elementCount.get() == organizationIds.size() && !guidsList.isEmpty()) { + saveBatch(OrganizationUtil.getDataByIds(client, guidsList, fetchOrganizationsUrl, catalogService)); } }); - log.info("Saved data of {} organizations successfully", numberOfOrganizations); - - return true; - } else { - return false; + log.info("Processed {} organizations", numberOfOrganizations); + } catch (Exception e) { + log.error("Error while fetching organizations for client {}", client, e); + } finally { + semaphore.release(); } + } private void saveBatch(JSONArray data) { diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchRestTask.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchRestTask.java new file mode 100644 index 00000000..f5434151 --- /dev/null +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchRestTask.java @@ -0,0 +1,80 @@ +/** + * + * The MIT License + * + * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) + * Copyright (c) 2016-2023 Finnish Digital Agency + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +package fi.vrk.xroad.catalog.collector.tasks; + +import java.util.List; +import java.util.concurrent.BlockingQueue; + +import org.json.JSONArray; +import org.json.JSONObject; +import org.springframework.context.ApplicationContext; + +import fi.vrk.xroad.catalog.collector.configuration.TaskPoolConfiguration; +import fi.vrk.xroad.catalog.collector.util.ClientTypeUtil; +import fi.vrk.xroad.catalog.collector.util.Endpoint; +import fi.vrk.xroad.catalog.collector.util.MethodListUtil; +import fi.vrk.xroad.catalog.collector.util.XRoadRestServiceIdentifierType; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class FetchRestTask extends BaseFetchTask { + + private static final String METHOD = "method"; + + private static final String PATH = "path"; + + public FetchRestTask(final ApplicationContext applicationContext, + final BlockingQueue restServices) { + super(applicationContext, restServices, + applicationContext.getBean(TaskPoolConfiguration.class).getFetchRestPoolSize()); + } + + @Override + protected void fetch(final XRoadRestServiceIdentifierType service) { + try { + log.info("Fetching REST for {}", ClientTypeUtil.toString(service)); + List endpointList = MethodListUtil.getEndpointList(service); + String endpointData = "{\"endpoint_data\":"; + JSONArray endPointsJSONArray = new JSONArray(); + JSONObject endpointJson; + catalogService.prepareEndpoints(createSubsystemId(service), createServiceId(service)); + for (Endpoint endpoint : endpointList) { + endpointJson = new JSONObject(); + endpointJson.put(METHOD, endpoint.getMethod()); + endpointJson.put(PATH, endpoint.getPath()); + endPointsJSONArray.put(endpointJson); + catalogService.saveEndpoint(createSubsystemId(service), createServiceId(service), endpoint.getMethod(), + endpoint.getPath()); + } + endpointData += endPointsJSONArray + "}"; + catalogService.saveRest(createSubsystemId(service), createServiceId(service), endpointData); + log.info("Saved REST for {} successfully", ClientTypeUtil.toString(service)); + } catch (Exception e) { + log.error("Failed to fetch REST for {}", ClientTypeUtil.toString(service), e); + } + } +} diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchWsdlsTask.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchWsdlsTask.java new file mode 100644 index 00000000..f938dc05 --- /dev/null +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/FetchWsdlsTask.java @@ -0,0 +1,84 @@ +/** + * + * The MIT License + * + * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) + * Copyright (c) 2016-2023 Finnish Digital Agency + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +package fi.vrk.xroad.catalog.collector.tasks; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.concurrent.BlockingQueue; + +import org.springframework.context.ApplicationContext; + +import fi.vrk.xroad.catalog.collector.configuration.TaskPoolConfiguration; +import fi.vrk.xroad.catalog.collector.util.ClientTypeUtil; +import fi.vrk.xroad.catalog.collector.util.XRoadClient; +import fi.vrk.xroad.catalog.collector.wsimport.XRoadServiceIdentifierType; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class FetchWsdlsTask extends BaseFetchTask { + + private final String xroadInstance; + + private final String memberCode; + + private final String memberClass; + + private final String subsystemCode; + + private final String webservicesEndpoint; + + private final XRoadClient xroadClient; + + public FetchWsdlsTask(final ApplicationContext applicationContext, + final BlockingQueue wsdlServices) throws URISyntaxException { + super(applicationContext, wsdlServices, + applicationContext.getBean(TaskPoolConfiguration.class).getFetchWsdlPoolSize()); + + TaskPoolConfiguration taskPoolConfiguration = applicationContext.getBean(TaskPoolConfiguration.class); + this.xroadInstance = taskPoolConfiguration.getXroadInstance(); + this.memberCode = taskPoolConfiguration.getMemberCode(); + this.memberClass = taskPoolConfiguration.getMemberClass(); + this.subsystemCode = taskPoolConfiguration.getSubsystemCode(); + this.webservicesEndpoint = taskPoolConfiguration.getWebservicesEndpoint(); + + this.xroadClient = new XRoadClient( + ClientTypeUtil.toSubsystem(xroadInstance, memberClass, memberCode, subsystemCode), + new URI(webservicesEndpoint)); + } + + @Override + protected void fetch(XRoadServiceIdentifierType service) { + try { + log.info("Fetching WSDL for {}", ClientTypeUtil.toString(service)); + String wsdl = xroadClient.getWsdl(service, catalogService); + catalogService.saveWsdl(createSubsystemId(service), createServiceId(service), wsdl); + log.info("WSDL for {} saved successfully", ClientTypeUtil.toString(service)); + } catch (Exception e) { + log.error("Failed to fetch WSDL for {}", ClientTypeUtil.toString(service), e); + } + } +} diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/ListClientsTask.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/ListClientsTask.java new file mode 100644 index 00000000..19ef1be7 --- /dev/null +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/ListClientsTask.java @@ -0,0 +1,144 @@ +/** + * + * The MIT License + * + * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) + * Copyright (c) 2016-2023 Finnish Digital Agency + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +package fi.vrk.xroad.catalog.collector.tasks; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Queue; + +import org.springframework.context.ApplicationContext; + +import fi.vrk.xroad.catalog.collector.configuration.TaskPoolConfiguration; +import fi.vrk.xroad.catalog.collector.util.ClientListUtil; +import fi.vrk.xroad.catalog.collector.util.ClientTypeUtil; +import fi.vrk.xroad.catalog.collector.util.CollectorUtils; +import fi.vrk.xroad.catalog.collector.wsimport.ClientList; +import fi.vrk.xroad.catalog.collector.wsimport.ClientType; +import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; +import fi.vrk.xroad.catalog.persistence.CatalogService; +import fi.vrk.xroad.catalog.persistence.entity.ErrorLog; +import fi.vrk.xroad.catalog.persistence.entity.Member; +import fi.vrk.xroad.catalog.persistence.entity.MemberId; +import fi.vrk.xroad.catalog.persistence.entity.Subsystem; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ListClientsTask { + + private final TaskPoolConfiguration taskPoolConfiguration; + private final CatalogService catalogService; + private final Queue listMethodsQueue; + private final Queue fetchCompaniesQueue; + private final Queue fetchOrganizationsQueue; + + public ListClientsTask(ApplicationContext applicationContext, Queue listMethodsQueue, + Queue fetchCompaniesQueue, Queue fetchOrganizationsQueue) { + this.taskPoolConfiguration = applicationContext.getBean(TaskPoolConfiguration.class); + this.catalogService = applicationContext.getBean(CatalogService.class); + this.listMethodsQueue = listMethodsQueue; + this.fetchCompaniesQueue = fetchCompaniesQueue; + this.fetchOrganizationsQueue = fetchOrganizationsQueue; + } + + public void run() { + if (CollectorUtils.isTimeBetweenHours(taskPoolConfiguration.getFlushLogTimeAfterHour(), + taskPoolConfiguration.getFlushLogTimeBeforeHour())) { + catalogService.deleteOldErrorLogEntries(taskPoolConfiguration.getErrorLogLengthInDays()); + } + + if (taskPoolConfiguration.isFetchRunUnlimited() + || CollectorUtils.isTimeBetweenHours(taskPoolConfiguration.getFetchTimeAfterHour(), + taskPoolConfiguration.getFetchTimeBeforeHour())) { + fetchClients(); + } + } + + private void fetchClients() { + String listClientsUrl = taskPoolConfiguration.getListClientsHost() + "/listClients"; + try { + log.info("Getting client list from {}", listClientsUrl); + ClientList clientList = ClientListUtil.clientListFromResponse(listClientsUrl); + HashMap m = populateMapWithMembers(clientList); + catalogService.saveAllMembersAndSubsystems(m.values()); + + // We only fetch WSDL-s and REST services from subsystems + List subsystems = clientList.getMember().stream() + .filter(client -> XRoadObjectType.SUBSYSTEM.equals(client.getId().getObjectType())) + .toList(); + listMethodsQueue.addAll(subsystems); + + log.info("All subsystems ({}) sent to ListMethodsTask", subsystems.size()); + + // The fetchCompaniesQueue and fetchOrganizationsQueue should only be + // initialized if the FI profile is active. The current actor implementation + // only ran these once and the specific client had no effect. This should be + // refactored once we get clarification on how this is expected to work. + if (fetchCompaniesQueue != null + && CollectorUtils.shouldFetchCompanies(taskPoolConfiguration.isFetchCompaniesRunUnlimited(), + taskPoolConfiguration.getFetchCompaniesTimeAfterHour(), + taskPoolConfiguration.getFetchCompaniesTimeBeforeHour())) { + fetchCompaniesQueue.add(clientList.getMember().getFirst()); + log.info("Notice send to the FetchCompaniesTask to do work"); + } + if (fetchOrganizationsQueue != null) { + fetchOrganizationsQueue.add(clientList.getMember().getFirst()); + log.info("Notice sent to the FetchOrganizationsTask to do work"); + } + } catch ( + + Exception e) { + ErrorLog errorLog = CollectorUtils.createErrorLog(null, + "Error when fetching listClients(url: " + listClientsUrl + "): " + e.getMessage(), + "500"); + catalogService.saveErrorLog(errorLog); + log.error("Error when fetching listClients(url: {})", listClientsUrl, e); + } + + } + + private HashMap populateMapWithMembers(ClientList clientList) { + HashMap m = new HashMap<>(); + int clientCounter = 0; + for (ClientType clientType : clientList.getMember()) { + clientCounter++; + log.info("{} - {}", clientCounter, ClientTypeUtil.toString(clientType)); + Member newMember = new Member(clientType.getId().getXRoadInstance(), clientType.getId() + .getMemberClass(), + clientType.getId().getMemberCode(), clientType.getName()); + newMember.setSubsystems(new HashSet<>()); + m.putIfAbsent(newMember.createKey(), newMember); + + if (XRoadObjectType.SUBSYSTEM.equals(clientType.getId().getObjectType())) { + Subsystem newSubsystem = new Subsystem(newMember, clientType.getId().getSubsystemCode()); + m.get(newMember.createKey()).getAllSubsystems().add(newSubsystem); + } + } + + return m; + } +} diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/ListMethodsTask.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/ListMethodsTask.java new file mode 100644 index 00000000..dbe0eaef --- /dev/null +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/tasks/ListMethodsTask.java @@ -0,0 +1,172 @@ +/** + * + * The MIT License + * + * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) + * Copyright (c) 2016-2023 Finnish Digital Agency + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +package fi.vrk.xroad.catalog.collector.tasks; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Semaphore; + +import org.springframework.context.ApplicationContext; + +import fi.vrk.xroad.catalog.collector.configuration.TaskPoolConfiguration; +import fi.vrk.xroad.catalog.collector.util.ClientTypeUtil; +import fi.vrk.xroad.catalog.collector.util.MethodListUtil; +import fi.vrk.xroad.catalog.collector.util.XRoadClient; +import fi.vrk.xroad.catalog.collector.util.XRoadRestServiceIdentifierType; +import fi.vrk.xroad.catalog.collector.wsimport.ClientType; +import fi.vrk.xroad.catalog.collector.wsimport.XRoadServiceIdentifierType; +import fi.vrk.xroad.catalog.persistence.CatalogService; +import fi.vrk.xroad.catalog.persistence.entity.Member; +import fi.vrk.xroad.catalog.persistence.entity.Service; +import fi.vrk.xroad.catalog.persistence.entity.Subsystem; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ListMethodsTask { + + private static final String SERVICE_TYPE_REST = "REST"; + + private String xroadSecurityServerHost; + + private String xroadInstance; + + private String memberCode; + + private String memberClass; + + private String subsystemCode; + + private String webservicesEndpoint; + + private final CatalogService catalogService; + + private final TaskPoolConfiguration taskPoolConfiguration; + + private final XRoadClient xroadClient; + + private final Semaphore semaphore; + + private final BlockingQueue clientsQueue; + + private final Queue wsdlQueue; + + private final Queue openApiQueue; + + private final Queue restQueue; + + public ListMethodsTask(final ApplicationContext applicationContext, final BlockingQueue clientsQueue, + final Queue wsdlQueue, final Queue restQueue, + final Queue openApiQueue) throws URISyntaxException { + this.catalogService = applicationContext.getBean(CatalogService.class); + + this.clientsQueue = clientsQueue; + this.wsdlQueue = wsdlQueue; + this.openApiQueue = openApiQueue; + this.restQueue = restQueue; + + this.taskPoolConfiguration = applicationContext.getBean(TaskPoolConfiguration.class); + this.xroadSecurityServerHost = taskPoolConfiguration.getSecurityServerHost(); + this.xroadInstance = taskPoolConfiguration.getXroadInstance(); + this.memberCode = taskPoolConfiguration.getMemberCode(); + this.memberClass = taskPoolConfiguration.getMemberClass(); + this.subsystemCode = taskPoolConfiguration.getSubsystemCode(); + this.webservicesEndpoint = taskPoolConfiguration.getWebservicesEndpoint(); + + this.semaphore = new Semaphore(taskPoolConfiguration.getListMethodsPoolSize()); + + this.xroadClient = new XRoadClient( + ClientTypeUtil.toSubsystem(xroadInstance, memberClass, memberCode, subsystemCode), + new URI(webservicesEndpoint)); + } + + public void run() { + log.info("Starting ListMethodsTask with pool size {}", taskPoolConfiguration.getListMethodsPoolSize()); + try { + while (true) { + log.debug("Polling for clients ... "); + + // take() blocks until an element becomes available or it gets interrupted + ClientType client = clientsQueue.take(); + semaphore.acquire(); + Thread.ofVirtual().start(() -> saveSubsystemsAndServices(client)); + } + } catch (InterruptedException e) { + log.warn("Interrupted while waiting for clients, stopping ListMethodsTask", e); + Thread.currentThread().interrupt(); + } + } + + private void saveSubsystemsAndServices(final ClientType clientType) { + try { + Subsystem subsystem = new Subsystem( + new Member(clientType.getId().getXRoadInstance(), clientType.getId().getMemberClass(), + clientType.getId().getMemberCode(), clientType.getName()), + clientType.getId().getSubsystemCode()); + + log.debug("Handling subsystem {} ", subsystem); + + List restServices = MethodListUtil.methodListFromResponse(clientType, + xroadSecurityServerHost, xroadInstance, memberClass, memberCode, subsystemCode, catalogService); + log.info("Received {} REST methods for client {} ", restServices.size(), + ClientTypeUtil.toString(clientType)); + + List soapServices = xroadClient.getMethods(clientType.getId(), catalogService); + log.info("Received {} SOAP methods for client {} ", soapServices.size(), + ClientTypeUtil.toString(clientType)); + + List services = new ArrayList<>(); + for (XRoadRestServiceIdentifierType service : restServices) { + services.add(new Service(subsystem, service.getServiceCode(), service.getServiceVersion())); + } + for (XRoadServiceIdentifierType service : soapServices) { + services.add(new Service(subsystem, service.getServiceCode(), service.getServiceVersion())); + } + + catalogService.saveServices(subsystem.createKey(), services); + + this.wsdlQueue.addAll(soapServices); + + for (XRoadRestServiceIdentifierType service : restServices) { + if (service.getServiceType().equalsIgnoreCase(SERVICE_TYPE_REST)) { + this.restQueue.add(service); + } else { + this.openApiQueue.add(service); + } + } + + log.debug("Subsystem {} handled", subsystem); + } catch (Exception e) { + log.error("Error while handling client {}", ClientTypeUtil.toString(clientType), e); + } finally { + semaphore.release(); + } + } +} diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/ClientListUtil.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/ClientListUtil.java index edb80032..c3d62cdc 100644 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/ClientListUtil.java +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/ClientListUtil.java @@ -16,8 +16,6 @@ import fi.vrk.xroad.catalog.collector.wsimport.ClientType; import fi.vrk.xroad.catalog.collector.wsimport.XRoadClientIdentifierType; import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; -import fi.vrk.xroad.catalog.persistence.CatalogService; -import fi.vrk.xroad.catalog.persistence.entity.ErrorLog; import org.json.JSONArray; import org.json.JSONObject; import org.springframework.http.ResponseEntity; @@ -25,23 +23,18 @@ public final class ClientListUtil { + private static final RestTemplate REST_TEMPLATE = new RestTemplate(); + private ClientListUtil() { // Private empty constructor } - public static ClientList clientListFromResponse(String url, CatalogService catalogService) { - RestTemplate restTemplate = new RestTemplate(); + public static ClientList clientListFromResponse(String url) { JSONArray members = new JSONArray(); - try { - ResponseEntity response = restTemplate.getForEntity(url, String.class); - JSONObject json = new JSONObject(response.getBody()); - members = json.getJSONArray("member"); - } catch (Exception e) { - ErrorLog errorLog = MethodListUtil.createErrorLog(null, - "Error when fetching listClients(url: " + url + "): " + e.getMessage(), - "500"); - catalogService.saveErrorLog(errorLog); - } + + ResponseEntity response = REST_TEMPLATE.getForEntity(url, String.class); + JSONObject json = new JSONObject(response.getBody()); + members = json.getJSONArray("member"); ClientList clientList = new ClientList(); for (int i = 0; i < members.length(); i++) { diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/CollectorUtils.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/CollectorUtils.java new file mode 100644 index 00000000..e293403e --- /dev/null +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/CollectorUtils.java @@ -0,0 +1,74 @@ +/** + * + * The MIT License + * + * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) + * Copyright (c) 2016-2023 Finnish Digital Agency + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +package fi.vrk.xroad.catalog.collector.util; + +import java.time.LocalDate; +import java.time.LocalDateTime; + +import fi.vrk.xroad.catalog.collector.wsimport.ClientType; +import fi.vrk.xroad.catalog.persistence.entity.ErrorLog; + +public final class CollectorUtils { + + private CollectorUtils() { + } + + public static boolean shouldFetchCompanies(boolean fetchUnlimited, int fetchHourAfter, int fetchHourBefore) { + if (fetchUnlimited) { + return true; + } + return isTimeBetweenHours(fetchHourAfter, fetchHourBefore); + } + + public static boolean isTimeBetweenHours(int fetchHourAfter, int fetchHourBefore) { + LocalDateTime today = LocalDateTime.now(); + LocalDateTime fetchTimeFrom = LocalDate.now().atTime(fetchHourAfter, 0); + LocalDateTime fetchTimeTo = LocalDate.now().atTime(fetchHourBefore, 0); + return (today.isAfter(fetchTimeFrom) && today.isBefore(fetchTimeTo)); + } + + public static ErrorLog createErrorLog(ClientType clientType, String message, String code) { + if (clientType != null) { + return ErrorLog.builder() + .created(LocalDateTime.now()) + .message(message) + .code(code) + .xRoadInstance(clientType.getId().getXRoadInstance()) + .memberClass(clientType.getId().getMemberClass()) + .memberCode(clientType.getId().getMemberCode()) + .groupCode(clientType.getId().getGroupCode()) + .securityCategoryCode(clientType.getId().getSecurityCategoryCode()) + .serverCode(clientType.getId().getServerCode()) + .serviceCode(clientType.getId().getServiceCode()) + .serviceVersion(clientType.getId().getServiceVersion()) + .subsystemCode(clientType.getId().getSubsystemCode()) + .build(); + } + return ErrorLog.builder().created(LocalDateTime.now()).message(message).code(code).build(); + } + +} diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/MethodListUtil.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/MethodListUtil.java index 3605bcd2..d59e7bdb 100644 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/MethodListUtil.java +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/MethodListUtil.java @@ -25,7 +25,6 @@ import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.client.RestTemplate; -import java.time.LocalDate; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; @@ -33,23 +32,14 @@ @Slf4j public final class MethodListUtil { + private static final RestTemplate REST_TEMPLATE = new RestTemplate(); + private static SecurityServerMetadata securityServerMetadata; private MethodListUtil() { // Private empty constructor } - public static boolean shouldFetchCompanies(boolean fetchUnlimited, int fetchHourAfter, int fetchHourBefore) { - if (fetchUnlimited) { - return true; - } - return isTimeBetweenHours(fetchHourAfter, fetchHourBefore); - } - - public static boolean shouldFlushLogEntries(int fetchHourAfter, int fetchHourBefore) { - return isTimeBetweenHours(fetchHourAfter, fetchHourBefore); - } - public static List methodListFromResponse(ClientType clientType, String host, String xRoadInstance, @@ -131,33 +121,6 @@ public static List getEndpointList return endpointList; } - public static ErrorLog createErrorLog(ClientType clientType, String message, String code) { - if (clientType != null) { - return ErrorLog.builder() - .created(LocalDateTime.now()) - .message(message) - .code(code) - .xRoadInstance(clientType.getId().getXRoadInstance()) - .memberClass(clientType.getId().getMemberClass()) - .memberCode(clientType.getId().getMemberCode()) - .groupCode(clientType.getId().getGroupCode()) - .securityCategoryCode(clientType.getId().getSecurityCategoryCode()) - .serverCode(clientType.getId().getServerCode()) - .serviceCode(clientType.getId().getServiceCode()) - .serviceVersion(clientType.getId().getServiceVersion()) - .subsystemCode(clientType.getId().getSubsystemCode()) - .build(); - } - return ErrorLog.builder().created(LocalDateTime.now()).message(message).code(code).build(); - } - - private static boolean isTimeBetweenHours(int fetchHourAfter, int fetchHourBefore) { - LocalDateTime today = LocalDateTime.now(); - LocalDateTime fetchTimeFrom = LocalDate.now().atTime(fetchHourAfter, 0); - LocalDateTime fetchTimeTo = LocalDate.now().atTime(fetchHourBefore, 0); - return (today.isAfter(fetchTimeFrom) && today.isBefore(fetchTimeTo)); - } - private static String createHeader(String xRoadInstance, String memberClass, String memberCode, String subsystemCode) { return new StringBuilder() @@ -169,7 +132,6 @@ private static String createHeader(String xRoadInstance, String memberClass, Str private static JSONObject getJSON(String url, ClientType clientType, String xRoadClientHeader, CatalogService catalogService) { - RestTemplate restTemplate = new RestTemplate(); HttpHeaders headers = new HttpHeaders(); List mediaTypes = new ArrayList<>(); mediaTypes.add(MediaType.APPLICATION_JSON); @@ -177,7 +139,7 @@ private static JSONObject getJSON(String url, ClientType clientType, String xRoa headers.set("X-Road-Client", xRoadClientHeader); final HttpEntity entity = new HttpEntity<>(headers); try { - ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, entity, + ResponseEntity response = REST_TEMPLATE.exchange(url, HttpMethod.GET, entity, String.class); return new JSONObject(response.getBody()); } catch (Exception e) { diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/OrganizationUtil.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/OrganizationUtil.java index 2dc97a04..67763207 100644 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/OrganizationUtil.java +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/OrganizationUtil.java @@ -99,7 +99,8 @@ private OrganizationUtil() { public static JSONObject getCompanies(ClientType clientType, Integer fetchCompaniesLimit, String url, - CatalogService catalogService) { + CatalogService catalogService) + throws NoSuchAlgorithmException, KeyStoreException, KeyManagementException { final String fetchCompaniesUrl = new StringBuilder() .append(url) .append("?totalResults=").append(TOTAL_RESULTS) @@ -113,31 +114,34 @@ public static JSONObject getCompanies(ClientType clientType, jsonObject = new JSONObject(ret); return jsonObject; } catch (KeyStoreException e) { - ErrorLog errorLog = MethodListUtil.createErrorLog(clientType, + ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, "KeyStoreException occurred when fetching list of companies from url " + url, "500"); catalogService.saveErrorLog(errorLog); log.error("KeyStoreException occurred when fetching companies from url {}", url); + throw e; } catch (NoSuchAlgorithmException e) { - ErrorLog errorLog = MethodListUtil.createErrorLog(clientType, + ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, "NoSuchAlgorithmException occurred when fetching companies from url " + url, "500"); catalogService.saveErrorLog(errorLog); log.error("NoSuchAlgorithmException occurred when fetching companies from url", url); + throw e; } catch (KeyManagementException e) { - ErrorLog errorLog = MethodListUtil.createErrorLog(clientType, + ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, "KeyManagementException occurred when fetching companies from url " + url, "500"); catalogService.saveErrorLog(errorLog); log.error("KeyManagementException occurred when fetching companies from url {}", url); + throw e; } catch (Exception e) { - ErrorLog errorLog = MethodListUtil.createErrorLog(clientType, + ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, "Exception occurred when fetching companies from url " + url, "500"); catalogService.saveErrorLog(errorLog); log.error("Exception occurred when fetching companies from url {}", url); + throw e; } - return jsonObject; } public static JSONObject getCompany(ClientType clientType, String url, String businessCode, @@ -150,7 +154,7 @@ public static JSONObject getCompany(ClientType clientType, String url, String bu jsonObject = new JSONObject(ret); return jsonObject; } catch (KeyStoreException e) { - ErrorLog errorLog = MethodListUtil.createErrorLog(clientType, + ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, "KeyStoreException occurred when fetching companies from url " + url + WITH_BUSINESS_CODE + businessCode, "500"); @@ -158,7 +162,7 @@ public static JSONObject getCompany(ClientType clientType, String url, String bu log.error("KeyStoreException occurred when fetching companies from url {} with businessCode {}", url, businessCode); } catch (NoSuchAlgorithmException e) { - ErrorLog errorLog = MethodListUtil.createErrorLog(clientType, + ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, "NoSuchAlgorithmException occurred when fetching companies from url " + url + WITH_BUSINESS_CODE + businessCode, "500"); @@ -166,7 +170,7 @@ public static JSONObject getCompany(ClientType clientType, String url, String bu log.error("NoSuchAlgorithmException occurred when fetching companies from url {} with businessCode {}", url, businessCode); } catch (KeyManagementException e) { - ErrorLog errorLog = MethodListUtil.createErrorLog(clientType, + ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, "KeyManagementException occurred when fetching companies from url " + url + WITH_BUSINESS_CODE + businessCode, "500"); @@ -174,7 +178,7 @@ public static JSONObject getCompany(ClientType clientType, String url, String bu log.error("KeyManagementException occurred when fetching companies from url {} with businessCode {}", url, businessCode); } catch (Exception e) { - ErrorLog errorLog = MethodListUtil.createErrorLog(clientType, + ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, "Exception occurred when fetching companies from url " + url + WITH_BUSINESS_CODE + businessCode, "500"); @@ -188,8 +192,8 @@ public static JSONObject getCompany(ClientType clientType, String url, String bu public static List getOrganizationIdsList(ClientType clientType, String url, Integer fetchOrganizationsLimit, CatalogService catalogService) throws KeyStoreException, NoSuchAlgorithmException, KeyManagementException { - List idsList = new ArrayList<>(); try { + List idsList = new ArrayList<>(); String response = getResponseBody(url); JSONObject json = new JSONObject(response); JSONArray itemList = json.optJSONArray("itemList"); @@ -201,27 +205,30 @@ public static List getOrganizationIdsList(ClientType clientType, String } return idsList; } catch (KeyStoreException e) { - ErrorLog errorLog = MethodListUtil.createErrorLog(clientType, + ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, "KeyStoreException occurred when fetching organization ids with from url " + url, "500"); catalogService.saveErrorLog(errorLog); log.error("KeyStoreException occurred when fetching organization ids with from url {}", url); + throw e; } catch (NoSuchAlgorithmException e) { - ErrorLog errorLog = MethodListUtil.createErrorLog(clientType, + ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, "NoSuchAlgorithmException occurred when fetching organization ids with from url " + url, "500"); catalogService.saveErrorLog(errorLog); log.error("NoSuchAlgorithmException occurred when fetching organization ids with from url {}", url); + throw e; } catch (KeyManagementException e) { - ErrorLog errorLog = MethodListUtil.createErrorLog(clientType, + ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, "KeyManagementException occurred when fetching organization ids with from url " + url, "500"); catalogService.saveErrorLog(errorLog); log.error("KeyManagementException occurred when fetching organizations with from url {}", url); + throw e; } catch (Exception e) { log.error("Exception occurred when fetching organization ids: " + e.getMessage()); ErrorLog errorLog = ErrorLog.builder() @@ -239,8 +246,8 @@ public static List getOrganizationIdsList(ClientType clientType, String .subsystemCode(clientType.getId().getSubsystemCode()) .build(); catalogService.saveErrorLog(errorLog); + throw e; } - return idsList; } public static Organization createOrganization(JSONObject jsonObject) { @@ -670,6 +677,8 @@ public static JSONArray getDataByIds(ClientType clientType, List guids, final String listOrganizationsUrl = new StringBuilder().append(url) .append("/list?guids=").append(requestGuids).toString(); + log.debug("Fetching organizations with guids: " + listOrganizationsUrl); + JSONArray itemList = new JSONArray(); try { String ret = getResponseBody(listOrganizationsUrl); @@ -677,13 +686,13 @@ public static JSONArray getDataByIds(ClientType clientType, List guids, itemList = json.optJSONArray("items"); return itemList; } catch (KeyStoreException e) { - ErrorLog errorLog = MethodListUtil.createErrorLog(clientType, + ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, "KeyStoreException occurred when fetching organizations with from url " + url, "500"); catalogService.saveErrorLog(errorLog); log.error("KeyStoreException occurred when fetching organizations with from url {}", url); } catch (NoSuchAlgorithmException e) { - ErrorLog errorLog = MethodListUtil.createErrorLog(clientType, + ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, "NoSuchAlgorithmException occurred when fetching organizations with from url " + url, "500"); @@ -691,7 +700,7 @@ public static JSONArray getDataByIds(ClientType clientType, List guids, log.error("NoSuchAlgorithmException occurred when fetching organizations with from url {}", url); } catch (KeyManagementException e) { - ErrorLog errorLog = MethodListUtil.createErrorLog(clientType, + ErrorLog errorLog = CollectorUtils.createErrorLog(clientType, "KeyManagementException occurred when fetching organizations with from url " + url, "500"); diff --git a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/XRoadClient.java b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/XRoadClient.java index b319ae3d..1310b1c8 100644 --- a/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/XRoadClient.java +++ b/xroad-catalog-collector/src/main/java/fi/vrk/xroad/catalog/collector/util/XRoadClient.java @@ -36,7 +36,7 @@ import org.apache.cxf.transport.http.HTTPConduit; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.net.URL; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.time.LocalDateTime; import java.util.ArrayList; @@ -52,19 +52,19 @@ public class XRoadClient { static final int HTTP_CONNECTION_TIMEOUT = 30000; static final int HTTP_RECEIVE_TIMEOUT = 60000; - static final Map META_SERVICE_PORTS = new HashMap<>(); + static final Map META_SERVICE_PORTS = new HashMap<>(); final MetaServicesPort metaServicesPort; final XRoadClientIdentifierType clientId; - public XRoadClient(XRoadClientIdentifierType clientId, URL serverUrl) { + public XRoadClient(XRoadClientIdentifierType clientId, URI serverUrl) { this.metaServicesPort = getMetaServicesPort(serverUrl); final XRoadClientIdentifierType tmp = new XRoadClientIdentifierType(); copyIdentifierType(tmp, clientId); this.clientId = tmp; } - private static synchronized MetaServicesPort getMetaServicesPort(URL serverUrl) { + private static synchronized MetaServicesPort getMetaServicesPort(URI serverUrl) { /** * This is currently a workaround, since the current approach results in all the * dispatchers creating a new port, which in turn causes a failure due to HTTP @@ -72,7 +72,7 @@ private static synchronized MetaServicesPort getMetaServicesPort(URL serverUrl) * * An issue is that CXF ports are not thread-safe, however that is * mostly for the cases of configuring it, which we do in - * {@link}fi.vrk.xroad.catalog.collector.util.XRoadClient#getMetaServicesPort(URL)}. + * {@link}fi.vrk.xroad.catalog.collector.util.XRoadClient#getMetaServicesPort(URI)}. * Actually using the port in multiple threads to do requests as a client should * be safe. * @@ -144,7 +144,7 @@ public List getMethods(XRoadClientIdentifierType mem return response != null ? response.getService() : new ArrayList<>(); } - public String getWsdl(XRoadServiceIdentifierType service, CatalogService catalogService) { + public String getWsdl(XRoadServiceIdentifierType service, CatalogService catalogService) throws Exception { XRoadServiceIdentifierType serviceIdentifierType = new XRoadServiceIdentifierType(); copyIdentifierType(serviceIdentifierType, service); XRoadClientIdentifierType tmpClientId = new XRoadClientIdentifierType(); @@ -186,6 +186,7 @@ public String getWsdl(XRoadServiceIdentifierType service, CatalogService catalog .subsystemCode(service.getSubsystemCode()) .build(); catalogService.saveErrorLog(errorLog); + throw e; } if (!(wsdl.value instanceof byte[])) { @@ -236,11 +237,11 @@ public String getWsdl(XRoadServiceIdentifierType service, CatalogService catalog .subsystemCode(service.getSubsystemCode()) .build(); catalogService.saveErrorLog(errorLog); + throw e; } } else { return new String(wsdl.value, StandardCharsets.UTF_8); } - return null; } public String getOpenApi(XRoadRestServiceIdentifierType service, @@ -284,7 +285,7 @@ private static Holder holder(T value) { return new Holder<>(value); } - private static MetaServicesPort createMetaServicesPort(URL url) { + private static MetaServicesPort createMetaServicesPort(URI url) { ProducerPortService service = new ProducerPortService(); MetaServicesPort port = service.getMetaServicesPortSoap11(); BindingProvider bindingProvider = (BindingProvider) port; diff --git a/xroad-catalog-collector/src/main/resources/application.conf b/xroad-catalog-collector/src/main/resources/application.conf deleted file mode 100644 index 76c6383a..00000000 --- a/xroad-catalog-collector/src/main/resources/application.conf +++ /dev/null @@ -1,20 +0,0 @@ -akka { - - # default dead letter = 10 - log-dead-letters = 1000 - - # Loggers to register at boot time (akka.event.Logging$DefaultLogger logs - # to STDOUT) - loggers = ["akka.event.slf4j.Slf4jLogger"] - - # Log level used by the configured loggers (see "loggers") as soon - # as they have been started; before that, see "stdout-loglevel" - # Options: OFF, ERROR, WARNING, INFO, DEBUG - loglevel = "INFO" - - # Log level for the very basic logger activated during ActorSystem startup. - # This logger prints the log messages to stdout (System.out). - # Options: OFF, ERROR, WARNING, INFO, DEBUG - stdout-loglevel = "ERROR" - -} \ No newline at end of file diff --git a/xroad-catalog-collector/src/main/resources/collector.properties b/xroad-catalog-collector/src/main/resources/collector.properties index be990d2f..7cdc0567 100644 --- a/xroad-catalog-collector/src/main/resources/collector.properties +++ b/xroad-catalog-collector/src/main/resources/collector.properties @@ -36,7 +36,7 @@ xroad-catalog.fetch-time-after-hour=3 xroad-catalog.fetch-time-before-hour=4 xroad-catalog.list-methods-pool-size=5 -# just one actor since the mock structure is not "threadsafe" +# just one thread since the mock structure is not "threadsafe" xroad-catalog.fetch-wsdl-pool-size=1 xroad-catalog.fetch-openapi-pool-size=1 xroad-catalog.fetch-rest-pool-size=1 @@ -45,4 +45,4 @@ xroad-catalog.fetch-companies-pool-size=1 # SSL keystore parameters xroad-catalog.ssl-keystore=/etc/xroad/xroad-catalog/keystore -xroad-catalog.ssl-keystore-password=changeit \ No newline at end of file +xroad-catalog.ssl-keystore-password=changeit diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/CatalogSupervisorTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/CatalogSupervisorTest.java deleted file mode 100644 index cf301065..00000000 --- a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/CatalogSupervisorTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.actors; - -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import akka.testkit.TestActorRef; -import fi.vrk.xroad.catalog.collector.configuration.DevelopmentConfiguration; -import fi.vrk.xroad.catalog.collector.extension.SpringExtension; -import fi.vrk.xroad.catalog.persistence.CatalogService; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.TestPropertySource; - -import static org.junit.jupiter.api.Assertions.assertFalse; - -@SpringBootTest(classes = DevelopmentConfiguration.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@TestPropertySource(properties = {"xroad-catalog.webservices-endpoint=http://localhost:${local.server.port}/metaservices"}) -public class CatalogSupervisorTest { - - @MockBean - CatalogService catalogService; - - @Autowired - ActorSystem actorSystem; - - @Autowired - SpringExtension springExtension; - - - @Test - public void testBasicPlumbing() { - TestActorRef supervisor = TestActorRef.create(actorSystem, springExtension.props("CatalogSupervisor")); - supervisor.tell("StartCollecting", ActorRef.noSender()); - assertFalse(supervisor.isTerminated()); - } - - @Test - public void testBasicPlumbingWithWrongMessageType() { - TestActorRef supervisor = TestActorRef.create(actorSystem, springExtension.props("CatalogSupervisor")); - supervisor.tell("", ActorRef.noSender()); - assertFalse(supervisor.isTerminated()); - } - -} - diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/FetchCompanyActorTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/FetchCompanyActorTest.java deleted file mode 100644 index 49729a12..00000000 --- a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/FetchCompanyActorTest.java +++ /dev/null @@ -1,78 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.actors; - -import fi.vrk.xroad.catalog.collector.configuration.DevelopmentConfiguration; -import fi.vrk.xroad.catalog.collector.extension.SpringExtension; -import fi.vrk.xroad.catalog.collector.wsimport.ClientType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadClientIdentifierType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; -import fi.vrk.xroad.catalog.persistence.CatalogService; -import fi.vrk.xroad.catalog.persistence.CompanyService; -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import akka.testkit.TestActorRef; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.TestPropertySource; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; - -@SpringBootTest(classes = DevelopmentConfiguration.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@TestPropertySource(properties = { - "xroad-catalog.fetch-companies-limit=3", - "xroad-catalog.fetch-companies-run-unlimited=true" -}) -public class FetchCompanyActorTest { - - @MockBean - CompanyService companyService; - - @MockBean - CatalogService catalogService; - - @Autowired - ActorSystem actorSystem; - - @Autowired - SpringExtension springExtension; - - @Test - public void testBasicPlumbing() { - TestActorRef fetchCompanyActor = TestActorRef.create(actorSystem, - springExtension.props("fetchCompaniesActor", null)); - ClientType clientType = new ClientType(); - XRoadClientIdentifierType value = new XRoadClientIdentifierType(); - value.setXRoadInstance("INSTANCE"); - value.setMemberClass("COM"); - value.setMemberCode("1710128-9"); - value.setSubsystemCode("SUBSYSTEM"); - value.setServiceCode("aService"); - value.setServiceVersion("v1"); - value.setObjectType(XRoadObjectType.SERVICE); - clientType.setId(value); - fetchCompanyActor.tell(clientType, ActorRef.noSender()); - verify(companyService, times(3)).saveCompany(any()); - } - - @Test - public void testBasicPlumbingWithInvalidMessageType() { - TestActorRef fetchCompanyActor = TestActorRef.create(actorSystem, springExtension.props("fetchCompaniesActor")); - fetchCompanyActor.tell("", ActorRef.noSender()); - verify(companyService, times(0)).saveCompany(any()); - } - -} diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/FetchOpenApiActorTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/FetchOpenApiActorTest.java deleted file mode 100644 index a630ea75..00000000 --- a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/FetchOpenApiActorTest.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.actors; - -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import akka.testkit.TestActorRef; -import fi.vrk.xroad.catalog.collector.configuration.DevelopmentConfiguration; -import fi.vrk.xroad.catalog.collector.extension.SpringExtension; -import fi.vrk.xroad.catalog.collector.util.Endpoint; -import fi.vrk.xroad.catalog.collector.util.XRoadRestServiceIdentifierType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; -import fi.vrk.xroad.catalog.persistence.CatalogService; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.TestPropertySource; -import java.util.ArrayList; -import java.util.List; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -@SpringBootTest(classes = DevelopmentConfiguration.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@TestPropertySource(properties = { - "xroad-catalog.webservices-endpoint=http://localhost:${local.server.port}/metaservices" -}) -public class FetchOpenApiActorTest { - - @MockBean - CatalogService catalogService; - - @Autowired - ActorSystem actorSystem; - - @Autowired - SpringExtension springExtension; - - @Test - public void testBasicPlumbing() { - TestActorRef fetchOpenApiActor = TestActorRef.create(actorSystem, springExtension.props("fetchOpenApiActor")); - XRoadRestServiceIdentifierType service = new XRoadRestServiceIdentifierType(); - service.setObjectType(XRoadObjectType.SERVICE); - service.setXRoadInstance("INSTANCE"); - service.setMemberClass("CLASS"); - service.setMemberCode("CODE"); - service.setSubsystemCode("SUBSYSTEM"); - service.setServiceCode("aService"); - service.setServiceVersion("v1"); - service.setServiceType("OPENAPI"); - List endpointList = new ArrayList<>(); - Endpoint endpoint = new Endpoint(); - endpoint.setMethod("GET"); - endpoint.setPath("/getServices"); - endpointList.add(endpoint); - service.setEndpoints(endpointList); - fetchOpenApiActor.tell(service, ActorRef.noSender()); - verify(catalogService, times(1)).saveOpenApi(any(), any(), any()); - verify(catalogService, times(1)).saveEndpoint(any(), any(), any(), any()); - } - - @Test - public void testBasicPlumbingWithWrongMessageType() { - TestActorRef fetchOpenApiActor = TestActorRef.create(actorSystem, springExtension.props("fetchOpenApiActor")); - fetchOpenApiActor.tell("Wrong message type object", ActorRef.noSender()); - verify(catalogService, times(0)).saveOpenApi(any(), any(), any()); - verify(catalogService, times(0)).saveEndpoint(any(), any(), any(), any()); - } -} diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/FetchOrganizationActorTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/FetchOrganizationActorTest.java deleted file mode 100644 index 599d6455..00000000 --- a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/FetchOrganizationActorTest.java +++ /dev/null @@ -1,75 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.actors; - -import fi.vrk.xroad.catalog.collector.configuration.DevelopmentConfiguration; -import fi.vrk.xroad.catalog.collector.extension.SpringExtension; -import fi.vrk.xroad.catalog.collector.wsimport.ClientType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadClientIdentifierType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import akka.testkit.TestActorRef; -import fi.vrk.xroad.catalog.persistence.OrganizationService; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.TestPropertySource; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -@SpringBootTest(classes = DevelopmentConfiguration.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@TestPropertySource(properties = { - "xroad-catalog.fetch-organizations-limit=3", - "xroad-catalog.max-organizations-per-request=3" -}) -public class FetchOrganizationActorTest { - - @MockBean - OrganizationService organizationService; - - @Autowired - ActorSystem actorSystem; - - @Autowired - SpringExtension springExtension; - - @Test - public void testBasicPlumbing() { - TestActorRef fetchOrganizationActor = TestActorRef.create(actorSystem, springExtension.props("fetchOrganizationsActor")); - ClientType clientType = new ClientType(); - XRoadClientIdentifierType value = new XRoadClientIdentifierType(); - value.setXRoadInstance("INSTANCE"); - value.setMemberClass("CLASS"); - value.setMemberCode("CODE"); - value.setSubsystemCode("SUBSYSTEM"); - value.setServiceCode("aService"); - value.setServiceVersion("v1"); - value.setObjectType(XRoadObjectType.SERVICE); - clientType.setId(value); - fetchOrganizationActor.tell(clientType, ActorRef.noSender()); - verify(organizationService, times(3)).saveOrganization(any()); - } - - @Test - public void testBasicPlumbingWithWrongMessageType() { - TestActorRef fetchOrganizationActor = TestActorRef.create(actorSystem, springExtension.props("fetchOrganizationsActor")); - fetchOrganizationActor.tell("Wrong message type object", ActorRef.noSender()); - verify(organizationService, times(0)).saveOrganization(any()); - } - -} - diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/FetchRestActorTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/FetchRestActorTest.java deleted file mode 100644 index fe1668fc..00000000 --- a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/FetchRestActorTest.java +++ /dev/null @@ -1,79 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.actors; - -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import akka.testkit.TestActorRef; -import fi.vrk.xroad.catalog.collector.configuration.DevelopmentConfiguration; -import fi.vrk.xroad.catalog.collector.extension.SpringExtension; -import fi.vrk.xroad.catalog.collector.util.Endpoint; -import fi.vrk.xroad.catalog.collector.util.XRoadRestServiceIdentifierType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; -import fi.vrk.xroad.catalog.persistence.CatalogService; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.TestPropertySource; -import java.util.ArrayList; -import java.util.List; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -@SpringBootTest(classes = DevelopmentConfiguration.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@TestPropertySource(properties = { - "xroad-catalog.webservices-endpoint=http://localhost:${local.server.port}/metaservices" -}) -public class FetchRestActorTest { - - @MockBean - CatalogService catalogService; - - @Autowired - ActorSystem actorSystem; - - @Autowired - SpringExtension springExtension; - - @Test - public void testBasicPlumbing() { - TestActorRef fetchRestActor = TestActorRef.create(actorSystem, springExtension.props("fetchRestActor")); - XRoadRestServiceIdentifierType service = new XRoadRestServiceIdentifierType(); - service.setObjectType(XRoadObjectType.SERVICE); - service.setXRoadInstance("INSTANCE"); - service.setMemberClass("CLASS"); - service.setMemberCode("CODE"); - service.setSubsystemCode("SUBSYSTEM"); - service.setServiceCode("aService"); - service.setServiceVersion("v1"); - service.setServiceType("REST"); - List endpointList = new ArrayList<>(); - endpointList.add(Endpoint.builder().method("GET").path("/getServices").build()); - service.setEndpoints(endpointList); - fetchRestActor.tell(service, ActorRef.noSender()); - verify(catalogService, times(1)).saveRest(any(), any(), any()); - verify(catalogService, times(1)).saveEndpoint(any(), any(), any(), any()); - } - - @Test - public void testBasicPlumbingWithWrongMessageType() { - TestActorRef fetchRestActor = TestActorRef.create(actorSystem, springExtension.props("fetchRestActor")); - fetchRestActor.tell("Wrong message type object", ActorRef.noSender()); - verify(catalogService, times(0)).saveRest(any(), any(), any()); - verify(catalogService, times(0)).saveEndpoint(any(), any(), any(), any()); - } -} - diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/FetchWsdlActorTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/FetchWsdlActorTest.java deleted file mode 100644 index 2400df06..00000000 --- a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/FetchWsdlActorTest.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.actors; - -import fi.vrk.xroad.catalog.collector.configuration.DevelopmentConfiguration; -import fi.vrk.xroad.catalog.collector.extension.SpringExtension; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadServiceIdentifierType; -import fi.vrk.xroad.catalog.persistence.CatalogService; -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import akka.testkit.TestActorRef; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.TestPropertySource; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -@SpringBootTest(classes = DevelopmentConfiguration.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@TestPropertySource(properties = { - "xroad-catalog.webservices-endpoint=http://localhost:${local.server.port}/metaservices" -}) -public class FetchWsdlActorTest { - - @MockBean - CatalogService catalogService; - - @Autowired - ActorSystem actorSystem; - - @Autowired - SpringExtension springExtension; - - @Test - public void testBasicPlumbing() { - TestActorRef fetchWsdlActor = TestActorRef.create(actorSystem, springExtension.props("fetchWsdlActor")); - XRoadServiceIdentifierType service = new XRoadServiceIdentifierType(); - service.setObjectType(XRoadObjectType.SERVICE); - service.setXRoadInstance("INSTANCE"); - service.setMemberClass("CLASS"); - service.setMemberCode("CODE"); - service.setSubsystemCode("SUBSYSTEM"); - service.setServiceCode("aService"); - service.setServiceVersion("v1"); - fetchWsdlActor.tell(service, ActorRef.noSender()); - verify(catalogService, times(1)).saveWsdl(any(), any(), any()); - } - - @Test - public void testBasicPlumbingWithWrongMessageType() { - TestActorRef fetchWsdlActor = TestActorRef.create(actorSystem, springExtension.props("fetchWsdlActor")); - fetchWsdlActor.tell("Wrong message type object", ActorRef.noSender()); - verify(catalogService, times(0)).saveWsdl(any(), any(), any()); - } -} - diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/ListClientsActorTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/ListClientsActorTest.java deleted file mode 100644 index 3d7f8eba..00000000 --- a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/ListClientsActorTest.java +++ /dev/null @@ -1,170 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.actors; - -import fi.vrk.xroad.catalog.collector.configuration.DevelopmentConfiguration; -import fi.vrk.xroad.catalog.collector.extension.SpringExtension; -import fi.vrk.xroad.catalog.collector.util.ClientListUtil; -import fi.vrk.xroad.catalog.collector.wsimport.ClientList; -import fi.vrk.xroad.catalog.collector.wsimport.ClientType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadClientIdentifierType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; -import fi.vrk.xroad.catalog.persistence.CatalogService; -import akka.actor.ActorSystem; -import akka.actor.InternalActorRef; -import akka.testkit.javadsl.TestKit; -import akka.testkit.TestActorRef; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.MockedStatic; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.event.annotation.AfterTestClass; -import org.springframework.test.context.event.annotation.BeforeTestClass; -import org.springframework.test.util.ReflectionTestUtils; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.mockStatic; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -@ActiveProfiles("development") -@SpringBootTest(classes = DevelopmentConfiguration.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -public class ListClientsActorTest extends TestKit { - - @MockBean - CatalogService catalogService; - - @Autowired - ActorSystem actorSystem; - - @Autowired - SpringExtension springExtension; - - private InternalActorRef listMethodsPoolRef; - - ListClientsActor listClientsActor; - - static ActorSystem system; - - public ListClientsActorTest() { - super(system); - } - - @BeforeTestClass - public static void setupTest() { - system = ActorSystem.create(); - } - - @AfterTestClass - public static void teardown() { - TestKit.shutdownActorSystem(system); - system = null; - } - - @BeforeEach - public void setup() throws Exception { - listMethodsPoolRef = mock(InternalActorRef.class); - TestActorRef clientsRef = TestActorRef.create(actorSystem, - springExtension.props("listClientsActor", listMethodsPoolRef)); - listClientsActor = clientsRef.underlyingActor(); - ReflectionTestUtils.setField(listClientsActor, "host", "http://localhost"); - ReflectionTestUtils.setField(listClientsActor, "fetchUnlimited", Boolean.TRUE); - MockitoAnnotations.initMocks(this); - } - - @Test - public void testOnReceiveWhenFetchUnlimited() throws Exception { - - try (MockedStatic mocked = Mockito.mockStatic(ClientListUtil.class)) { - ClientList clientList = new ClientList(); - clientList.getMember().add(createClientType(XRoadObjectType.MEMBER, "member1", null)); - clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member1", "sub1")); - clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member1", "sub2")); - clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member1", "sub3")); - clientList.getMember().add(createClientType(XRoadObjectType.MEMBER, "member2", null)); - clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member2", "sssub1")); - clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member2", "sssub2")); - - mocked.when(() -> ClientListUtil.clientListFromResponse(any(String.class), any(CatalogService.class))) - .thenReturn(clientList); - - listClientsActor.onReceive(ListClientsActor.START_COLLECTING); - - verify(catalogService, times(1)).saveAllMembersAndSubsystems(any()); - } - } - - @Test - public void testOnReceiveWhenFetchNotUnlimitedButTimeIsInBetween() throws Exception { - try (MockedStatic mocked = mockStatic(ClientListUtil.class)) { - ReflectionTestUtils.setField(listClientsActor, "host", "http://localhost"); - ReflectionTestUtils.setField(listClientsActor, "fetchUnlimited", Boolean.FALSE); - ReflectionTestUtils.setField(listClientsActor, "fetchTimeAfterHour", 0); - ReflectionTestUtils.setField(listClientsActor, "fetchTimeBeforeHour", 23); - - ClientList clientList = new ClientList(); - clientList.getMember().add(createClientType(XRoadObjectType.MEMBER, "member1", null)); - clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member1", "sub1")); - clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member1", "sub2")); - clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member1", "sub3")); - clientList.getMember().add(createClientType(XRoadObjectType.MEMBER, "member2", null)); - clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member2", "sssub1")); - clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member2", "sssub2")); - - mocked.when(() -> ClientListUtil.clientListFromResponse(any(String.class), any(CatalogService.class))) - .thenReturn(clientList); - - listClientsActor.onReceive(ListClientsActor.START_COLLECTING); - - verify(catalogService, times(1)).saveAllMembersAndSubsystems(any()); - } - } - - @Test - public void testOnReceiveWithEmptyMemberList() throws Exception { - try (MockedStatic mocked = mockStatic(ClientListUtil.class)) { - ClientList clientList = new ClientList(); - mocked.when(() -> ClientListUtil.clientListFromResponse(any(String.class), any(CatalogService.class))) - .thenReturn(clientList); - listClientsActor.onReceive(ListClientsActor.START_COLLECTING); - verify(catalogService, times(1)).saveAllMembersAndSubsystems(any()); - } - } - - @Test - public void testSaveErrorLog() throws Exception { - listClientsActor.onReceive(ListClientsActor.START_COLLECTING); - verify(catalogService, times(1)).saveErrorLog(any()); - } - - protected ClientType createClientType(XRoadObjectType objectType, String memberCode, String subsystemCode) { - ClientType c = new ClientType(); - XRoadClientIdentifierType xrcit = new XRoadClientIdentifierType(); - - xrcit.setXRoadInstance("FI"); - xrcit.setMemberClass("GOV"); - xrcit.setMemberCode(memberCode); - xrcit.setSubsystemCode(subsystemCode); - xrcit.setObjectType(objectType); - c.setId(xrcit); - c.setName(memberCode); - return c; - - } -} diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/ListClientsRestTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/ListClientsRestTest.java deleted file mode 100644 index 94ffa274..00000000 --- a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/ListClientsRestTest.java +++ /dev/null @@ -1,149 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.actors; - -import fi.vrk.xroad.catalog.collector.configuration.DevelopmentConfiguration; -import fi.vrk.xroad.catalog.collector.wsimport.ClientList; -import lombok.extern.slf4j.Slf4j; -import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.http.MediaType; -import org.springframework.test.web.client.MockRestServiceServer; -import org.springframework.web.client.RestTemplate; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.fail; -import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; -import static org.springframework.test.web.client.response.MockRestResponseCreators.withServerError; -import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; - -@Slf4j -@SpringBootTest(classes = DevelopmentConfiguration.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -public class ListClientsRestTest { - - private static final String VALID_LISTCLIENTS_RESPONSE = "\n" - + - "\n" - + - " \n" + - " \n" + - " FI\n" + - " GOV\n" + - " 1710128-9\n" + - " \n" + - " Gofore\n" + - " \n" + - " \n" + - " \n" + - " FI\n" + - " GOV\n" + - " 1710128-9\n" + - " Management\n" + - " \n" + - " Gofore\n" + - " \n" + - " \n" + - " \n" + - " FI\n" + - " GOV\n" + - " 1710128-9\n" + - " SS1\n" + - " \n" + - " Gofore\n" + - " \n" + - "\n"; - - private static final String OUTDATED_GLOBAL_CONF = "Server.ClientProxy.OutdatedGlobalConfGlobal configuration is expiredOutdatedGlobalConf: \n" - + - "\tat ee.ria.xroad.common.conf.globalconf.GlobalConf.verifyValidity(GlobalConf.java:160)\n" + - "\tat ee.ria.xroad.proxy.util.MessageProcessorBase.cacheConfigurationForCurrentThread(MessageProcessorBase.java:64)\n" - + - "\tat ee.ria.xroad.proxy.util.MessageProcessorBase.<init>(MessageProcessorBase.java:55)\n" + - "\tat ee.ria.xroad.proxy.clientproxy.AsicContainerClientRequestProcessor.<init>(AsicContainerClientRequestProcessor.java:108)\n" - + - "\tat ee.ria.xroad.proxy.clientproxy.AsicContainerHandler.createRequestProcessor(AsicContainerHandler.java:63)\n" - + - "\tat ee.ria.xroad.proxy.clientproxy.AbstractClientProxyHandler.handle(AbstractClientProxyHandler.java:80)\n" - + - "\tat ee.ria.xroad.proxy.clientproxy.AsicContainerHandler.handle(AsicContainerHandler.java:39)\n" + - "\tat org.eclipse.jetty.server.handler.HandlerCollection.handle(HandlerCollection.java:154)\n" + - "\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)\n" + - "\tat org.eclipse.jetty.server.Server.handle(Server.java:370)\n" + - "\tat org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:494)\n" - + - "\tat org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:971)\n" - + - "\tat org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1033)\n" - + - "\tat org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:640)\n" + - "\tat org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:231)\n" + - "\tat org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)\n" + - "\tat org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:696)\n" + - "\tat org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:53)\n" + - "\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)\n" + - "\tat org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)\n" + - "\tat java.lang.Thread.run(Thread.java:745)\n" + - ""; - - private static final String TEST_REQUEST_PATH = "/dummy-request-path"; - - @Test - public void testBadMessage() throws Exception { - RestTemplate t = new RestTemplate(); - MockRestServiceServer mockServer = MockRestServiceServer.createServer(t); - - mockServer.expect(requestTo(TEST_REQUEST_PATH)).andRespond( - withSuccess(OUTDATED_GLOBAL_CONF, MediaType.APPLICATION_XML)); - - try { - ClientList clientList = t.getForObject(TEST_REQUEST_PATH, ClientList.class); - fail("should fail since restOperation is not returning valid ClientList"); - } catch (Exception expected) { - log.error("expected exception: ", expected); - } - } - - @Test - public void testOkMessage() throws Exception { - RestTemplate t = new RestTemplate(); - MockRestServiceServer mockServer = MockRestServiceServer.createServer(t); - - mockServer.expect(requestTo(TEST_REQUEST_PATH)).andRespond( - withSuccess(VALID_LISTCLIENTS_RESPONSE, MediaType.APPLICATION_XML)); - - try { - ClientList clientList = t.getForObject(TEST_REQUEST_PATH, ClientList.class); - assertNotNull(clientList); - assertEquals(3, clientList.getMember().size()); - } catch (Exception expected) { - log.error("expected exception: ", expected); - } - - } - - @Test - public void testErrorResponseCode() throws Exception { - RestTemplate t = new RestTemplate(); - MockRestServiceServer mockServer = MockRestServiceServer.createServer(t); - - mockServer.expect(requestTo(TEST_REQUEST_PATH)).andRespond( - withServerError()); - try { - ClientList clientList = t.getForObject(TEST_REQUEST_PATH, ClientList.class); - fail("should fail since not http 200 OK"); - } catch (Exception expected) { - log.error("expected exception: ", expected); - } - } -} diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/ListMethodsActorTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/ListMethodsActorTest.java deleted file mode 100644 index f43cd2b0..00000000 --- a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/ListMethodsActorTest.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.actors; - -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import akka.actor.InternalActorRef; -import akka.testkit.TestActorRef; -import fi.vrk.xroad.catalog.collector.configuration.DevelopmentConfiguration; -import fi.vrk.xroad.catalog.collector.extension.SpringExtension; -import fi.vrk.xroad.catalog.collector.wsimport.ClientType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadClientIdentifierType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; -import fi.vrk.xroad.catalog.persistence.CatalogService; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.TestPropertySource; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -@SpringBootTest(classes = DevelopmentConfiguration.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@TestPropertySource(properties = { - "xroad-catalog.fetch-organizations-limit=3", - "xroad-catalog.fetch-companies-limit=3", - "xroad-catalog.max-organizations-per-request=3" -}) -public class ListMethodsActorTest { - - @MockBean - CatalogService catalogService; - - @Autowired - ActorSystem actorSystem; - - @Autowired - SpringExtension springExtension; - - @Test - public void testBasicPlumbing() { - TestActorRef listMethodsActor = TestActorRef.create(actorSystem, - springExtension.props("listMethodsActor", - mock(InternalActorRef.class), - mock(InternalActorRef.class), - mock(InternalActorRef.class))); - ClientType clientType = new ClientType(); - XRoadClientIdentifierType value = new XRoadClientIdentifierType(); - value.setXRoadInstance("INSTANCE"); - value.setMemberClass("CLASS"); - value.setMemberCode("CODE"); - value.setSubsystemCode("SUBSYSTEM"); - value.setServiceCode("aService"); - value.setServiceVersion("v1"); - value.setObjectType(XRoadObjectType.SUBSYSTEM); - clientType.setId(value); - listMethodsActor.tell(clientType, ActorRef.noSender()); - verify(catalogService, times(1)).saveServices(any(), any()); - } - - @Test - public void testBasicPlumbingWithWrongMessageType() { - TestActorRef listMethodsActor = TestActorRef.create(actorSystem, - springExtension.props("listMethodsActor", - mock(InternalActorRef.class), - mock(InternalActorRef.class), - mock(InternalActorRef.class))); - listMethodsActor.tell("Wrong message type object", ActorRef.noSender()); - verify(catalogService, times(0)).saveServices(any(), any()); - } -} - diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/OrganizationsActorTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/OrganizationsActorTest.java deleted file mode 100644 index 388e532b..00000000 --- a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/OrganizationsActorTest.java +++ /dev/null @@ -1,93 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.actors; - -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import akka.actor.InternalActorRef; -import akka.testkit.TestActorRef; -import fi.vrk.xroad.catalog.collector.configuration.DevelopmentConfiguration; -import fi.vrk.xroad.catalog.collector.extension.SpringExtension; -import fi.vrk.xroad.catalog.collector.wsimport.ClientType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadClientIdentifierType; -import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; -import fi.vrk.xroad.catalog.persistence.CatalogService; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.TestPropertySource; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; - -@SpringBootTest(classes = DevelopmentConfiguration.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@TestPropertySource(properties = { - "xroad-catalog.fetch-organizations-limit=3", - "xroad-catalog.fetch-companies-limit=3", - "xroad-catalog.max-organizations-per-request=3", - "xroad-catalog.fetch-companies-run-unlimited=true", - "xroad-catalog.flush-log-time-after-hour=0", - "xroad-catalog.flush-log-time-before-hour=23" -}) -public class OrganizationsActorTest { - - @MockBean - CatalogService catalogService; - - @Autowired - ActorSystem actorSystem; - - @Autowired - SpringExtension springExtension; - - @Test - public void testBasicPlumbing() { - TestActorRef listMethodsActor = TestActorRef.create(actorSystem, - springExtension.props("organizationsActor", - mock(InternalActorRef.class), - mock(InternalActorRef.class), - mock(InternalActorRef.class), - mock(InternalActorRef.class), - mock(InternalActorRef.class))); - ClientType clientType = new ClientType(); - XRoadClientIdentifierType value = new XRoadClientIdentifierType(); - value.setXRoadInstance("INSTANCE"); - value.setMemberClass("CLASS"); - value.setMemberCode("CODE"); - value.setSubsystemCode("SUBSYSTEM"); - value.setServiceCode("aService"); - value.setServiceVersion("v1"); - value.setObjectType(XRoadObjectType.SUBSYSTEM); - clientType.setId(value); - listMethodsActor.tell(clientType, ActorRef.noSender()); - verify(catalogService, times(1)).saveServices(any(), any()); - } - - @Test - public void testBasicPlumbingWithWrongMessageType() { - TestActorRef listMethodsActor = TestActorRef.create(actorSystem, - springExtension.props("organizationsActor", - mock(InternalActorRef.class), - mock(InternalActorRef.class), - mock(InternalActorRef.class), - mock(InternalActorRef.class), - mock(InternalActorRef.class))); - listMethodsActor.tell("Wrong message type object", ActorRef.noSender()); - verify(catalogService, times(0)).saveServices(any(), any()); - } - -} - diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/OrganizationsSupervisorTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/OrganizationsSupervisorTest.java deleted file mode 100644 index f4ee8fae..00000000 --- a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/actors/OrganizationsSupervisorTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * The MIT License - * - * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) - * Copyright (c) 2016-2023 Finnish Digital Agency - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ -package fi.vrk.xroad.catalog.collector.actors; - -import akka.actor.ActorRef; -import akka.actor.ActorSystem; -import akka.testkit.TestActorRef; -import fi.vrk.xroad.catalog.collector.configuration.DevelopmentConfiguration; -import fi.vrk.xroad.catalog.collector.extension.SpringExtension; -import fi.vrk.xroad.catalog.persistence.CatalogService; -import org.junit.jupiter.api.Test; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.TestPropertySource; - -import static org.junit.jupiter.api.Assertions.assertFalse; - -@SpringBootTest(classes = DevelopmentConfiguration.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) -@TestPropertySource(properties = { - "xroad-catalog.webservices-endpoint=http://localhost:${local.server.port}/metaservices" -}) -@ActiveProfiles("fi") -public class OrganizationsSupervisorTest { - - @MockBean - CatalogService catalogService; - - @Autowired - ActorSystem actorSystem; - - @Autowired - SpringExtension springExtension; - - - @Test - public void testBasicPlumbing() { - TestActorRef supervisor = TestActorRef.create(actorSystem, springExtension.props("OrganizationsSupervisor")); - supervisor.tell("StartCollecting", ActorRef.noSender()); - assertFalse(supervisor.isTerminated()); - } - - @Test - public void testBasicPlumbingWithWrongMessageType() { - TestActorRef supervisor = TestActorRef.create(actorSystem, springExtension.props("OrganizationsSupervisor")); - supervisor.tell("", ActorRef.noSender()); - assertFalse(supervisor.isTerminated()); - } -} - diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchCompaniesTaskTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchCompaniesTaskTest.java new file mode 100644 index 00000000..14174c27 --- /dev/null +++ b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchCompaniesTaskTest.java @@ -0,0 +1,141 @@ +/** + * + * The MIT License + * + * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) + * Copyright (c) 2016-2023 Finnish Digital Agency + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +package fi.vrk.xroad.catalog.collector.tasks; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.Semaphore; + +import org.awaitility.Awaitility; +import org.json.JSONException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.ApplicationContext; +import org.springframework.core.io.Resource; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.util.ReflectionTestUtils; + +import fi.vrk.xroad.catalog.collector.configuration.TaskPoolConfiguration; +import fi.vrk.xroad.catalog.collector.util.OrganizationUtil; +import fi.vrk.xroad.catalog.collector.wsimport.ClientType; +import fi.vrk.xroad.catalog.collector.wsimport.XRoadClientIdentifierType; +import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; +import fi.vrk.xroad.catalog.persistence.CatalogService; +import fi.vrk.xroad.catalog.persistence.CompanyService; + +@SpringBootTest(classes = TaskPoolConfiguration.class) +@TestPropertySource(properties = { "xroad-catalog.fetch-companies-url=" }) +public class FetchCompaniesTaskTest { + + @MockBean + CatalogService catalogService; + + @MockBean + CompanyService companyService; + + @Autowired + private ApplicationContext applicationContext; + + @Value("classpath:mock/companies/getCompanies.json") + private Resource companiesJSON; + + @Value("classpath:mock/companies/company.json") + private Resource companyJSON; + + @Test + public void testBasicNoDeadlock() throws InterruptedException { + /** + * Note that this test will log an error that the operation did not succeed. + * That is ok, because all we want to check here is that the task does not + * deadlock and takes the data from our queue. We interrupt at the end to make + * sure that the task can also be stopped when the program exits. The actual + * fetch logic is mocked and tested below. + */ + BlockingQueue queue = new LinkedBlockingQueue<>(); + FetchCompaniesTask fetchCompaniesTask = new FetchCompaniesTask(applicationContext, queue); + Semaphore semaphore = new Semaphore(1); + ReflectionTestUtils.setField(fetchCompaniesTask, "semaphore", semaphore); + ClientType clientType = new ClientType(); + Thread fetchCompaniesRunner = Thread.ofVirtual().start(fetchCompaniesTask::run); + queue.add(clientType); + + Awaitility.await().atMost(Duration.ofSeconds(2)).until(() -> queue.isEmpty()); + + semaphore.acquire(); + fetchCompaniesRunner.interrupt(); + + assertTrue(queue.isEmpty()); + } + + @Test + public void testFetchCompaniesForClient() throws JSONException, IOException { + try (MockedStatic mock = Mockito.mockStatic(OrganizationUtil.class)) { + FetchCompaniesTask fetchCompaniesTask = new FetchCompaniesTask(applicationContext, null); + + final JSONObject getCompaniesResponse = new JSONObject( + companiesJSON.getContentAsString(StandardCharsets.UTF_8)); + mock.when(() -> OrganizationUtil.getCompanies(any(), any(), any(), any())) + .thenReturn(getCompaniesResponse); + + final JSONObject getCompanyResponse = new JSONObject( + companyJSON.getContentAsString(StandardCharsets.UTF_8)); + mock.when(() -> OrganizationUtil.getCompany(any(), any(), any(), any())) + .thenReturn(getCompanyResponse); + ClientType clientType = new ClientType(); + XRoadClientIdentifierType value = new XRoadClientIdentifierType(); + value.setXRoadInstance("INSTANCE"); + value.setMemberClass("COM"); + value.setMemberCode("1234567-9"); + value.setSubsystemCode("SUBSYSTEM"); + value.setServiceCode("aService"); + value.setServiceVersion("v1"); + value.setObjectType(XRoadObjectType.SERVICE); + clientType.setId(value); + + fetchCompaniesTask.fetchCompaniesForCLient(clientType); + + mock.verify(() -> OrganizationUtil.getCompanies(any(), any(), any(), any()), times(1)); + mock.verify(() -> OrganizationUtil.getCompany(any(), any(), any(), any()), times(4)); + verify(companyService, times(4)).saveCompany(any()); + } + } + +} diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchOpenApiTaskTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchOpenApiTaskTest.java new file mode 100644 index 00000000..1c482179 --- /dev/null +++ b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchOpenApiTaskTest.java @@ -0,0 +1,136 @@ +/** + * + * The MIT License + * + * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) + * Copyright (c) 2016-2023 Finnish Digital Agency + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +package fi.vrk.xroad.catalog.collector.tasks; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.Semaphore; + +import org.awaitility.Awaitility; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.ApplicationContext; +import org.springframework.core.io.Resource; +import org.springframework.test.util.ReflectionTestUtils; + +import fi.vrk.xroad.catalog.collector.configuration.TaskPoolConfiguration; +import fi.vrk.xroad.catalog.collector.util.Endpoint; +import fi.vrk.xroad.catalog.collector.util.MethodListUtil; +import fi.vrk.xroad.catalog.collector.util.XRoadRestServiceIdentifierType; +import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; +import fi.vrk.xroad.catalog.persistence.CatalogService; + +@SpringBootTest(classes = TaskPoolConfiguration.class) +public class FetchOpenApiTaskTest { + + @MockBean + CatalogService catalogService; + + @Autowired + private ApplicationContext applicationContext; + + @Value("classpath:mock/xroad/openapi/openapi.json") + private Resource openApiFile; + + @Test + public void testBasicNoDeadlock() throws InterruptedException, MalformedURLException, URISyntaxException { + /** + * Note that this test will log an error that the operation did not succeed. + * That is ok, because all we want to check here is that the task does not + * deadlock and takes the data from our queue. We interrupt at the end to make + * sure that the task can also be stopped when the program exits. The actual + * fetch logic is mocked and tested below. + */ + BlockingQueue queue = new LinkedBlockingQueue<>(); + FetchOpenApiTask fetchOpenApiTask = new FetchOpenApiTask(applicationContext, queue); + Semaphore semaphore = new Semaphore(1); + ReflectionTestUtils.setField(fetchOpenApiTask, "semaphore", semaphore); + XRoadRestServiceIdentifierType restService = new XRoadRestServiceIdentifierType(); + Thread fetchOpenApiRunner = Thread.ofVirtual().start(fetchOpenApiTask::run); + queue.add(restService); + + Awaitility.await().atMost(Duration.ofSeconds(2)).until(() -> queue.isEmpty()); + + semaphore.acquire(); + fetchOpenApiRunner.interrupt(); + assertTrue(queue.isEmpty()); + } + + @Test + public void testFetch() throws URISyntaxException, InterruptedException, IOException { + try (MockedStatic mock = Mockito.mockStatic(MethodListUtil.class)) { + final String openApiResponse = openApiFile.getContentAsString(StandardCharsets.UTF_8); + mock.when(() -> MethodListUtil.openApiFromResponse(any(), any(), any(), any(), any(), any(), any())) + .thenReturn(openApiResponse); + mock.when(() -> MethodListUtil.getEndpointList(any())).thenCallRealMethod(); + + FetchOpenApiTask fetchOpenApiTask = new FetchOpenApiTask(applicationContext, new LinkedBlockingQueue<>()); + + XRoadRestServiceIdentifierType service = new XRoadRestServiceIdentifierType(); + service.setObjectType(XRoadObjectType.SERVICE); + service.setXRoadInstance("INSTANCE"); + service.setMemberClass("CLASS"); + service.setMemberCode("CODE"); + service.setSubsystemCode("SUBSYSTEM"); + service.setServiceCode("aService"); + service.setServiceVersion("v1"); + service.setServiceType("OPENAPI"); + List endpointList = new ArrayList<>(); + Endpoint endpoint = new Endpoint(); + endpoint.setMethod("GET"); + endpoint.setPath("/getServices"); + endpointList.add(endpoint); + service.setEndpoints(endpointList); + + fetchOpenApiTask.fetch(service); + + mock.verify(() -> MethodListUtil.openApiFromResponse(any(), any(), any(), any(), any(), any(), any()), + times(1)); + verify(catalogService, times(0)).saveErrorLog(any()); + verify(catalogService, times(1)).saveOpenApi(any(), any(), any()); + verify(catalogService, times(1)).saveEndpoint(any(), any(), any(), any()); + } + } + +} diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchOrganizationTaskTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchOrganizationTaskTest.java new file mode 100644 index 00000000..6aa463d3 --- /dev/null +++ b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchOrganizationTaskTest.java @@ -0,0 +1,142 @@ +/** + * + * The MIT License + * + * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) + * Copyright (c) 2016-2023 Finnish Digital Agency + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +package fi.vrk.xroad.catalog.collector.tasks; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.Semaphore; + +import org.awaitility.Awaitility; +import org.json.JSONArray; +import org.json.JSONException; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.ApplicationContext; +import org.springframework.core.io.Resource; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.util.ReflectionTestUtils; + +import fi.vrk.xroad.catalog.collector.configuration.TaskPoolConfiguration; +import fi.vrk.xroad.catalog.collector.util.OrganizationUtil; +import fi.vrk.xroad.catalog.collector.wsimport.ClientType; +import fi.vrk.xroad.catalog.collector.wsimport.XRoadClientIdentifierType; +import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; +import fi.vrk.xroad.catalog.persistence.CatalogService; +import fi.vrk.xroad.catalog.persistence.OrganizationService; + +@SpringBootTest(classes = TaskPoolConfiguration.class) +@TestPropertySource(properties = { + "xroad-catalog.fetch-organizations-limit=3", + "xroad-catalog.max-organizations-per-request=3" +}) +public class FetchOrganizationTaskTest { + + @MockBean + CatalogService catalogService; + + @MockBean + OrganizationService organizationService; + + @Autowired + private ApplicationContext applicationContext; + + @Value("classpath:mock/organizations/organizationsById.json") + private Resource organizationsByIdJSON; + + @Test + public void testBasicNoDeadlock() throws InterruptedException { + /** + * Note that this test will log an error that the operation did not succeed. + * That is ok, because all we want to check here is that the task does not + * deadlock and takes the data from our queue. We interrupt at the end to make + * sure that the task can also be stopped when the program exits. The actual + * fetch logic is mocked and tested below. + */ + BlockingQueue queue = new LinkedBlockingQueue<>(); + FetchOrganizationsTask fetchOrganizationsTask = new FetchOrganizationsTask(applicationContext, queue); + Semaphore semaphore = new Semaphore(1); + ReflectionTestUtils.setField(fetchOrganizationsTask, "semaphore", semaphore); + ClientType clientType = new ClientType(); + Thread fetchOrganizationsRunner = Thread.ofVirtual().start(fetchOrganizationsTask::run); + queue.add(clientType); + Awaitility.await().atMost(Duration.ofSeconds(2)).until(() -> queue.isEmpty()); + semaphore.acquire(); + fetchOrganizationsRunner.interrupt(); + assertTrue(queue.isEmpty()); + } + + @Test + public void testFetchOrganizationsForClient() throws JSONException, IOException { + try (MockedStatic mock = Mockito.mockStatic(OrganizationUtil.class)) { + FetchOrganizationsTask fetchOrganizationsTask = new FetchOrganizationsTask(applicationContext, + null); + + mock.when(() -> OrganizationUtil.getOrganizationIdsList(any(), any(), any(), any())) + .thenReturn(List.of( + "112ea34r3-1k23-412r-9142-1442asd13131", + "112ea34r3-1k23-412r-9142-1442asd13132", + "112ea34r3-1k23-412r-9142-1442asd13133")); + + JSONArray organizationsByIdResponse = new JSONArray( + organizationsByIdJSON.getContentAsString(StandardCharsets.UTF_8)); + mock.when(() -> OrganizationUtil.getDataByIds(any(), any(), any(), any())) + .thenReturn(organizationsByIdResponse); + + ClientType clientType = new ClientType(); + XRoadClientIdentifierType value = new XRoadClientIdentifierType(); + value.setXRoadInstance("INSTANCE"); + value.setMemberClass("CLASS"); + value.setMemberCode("CODE"); + value.setSubsystemCode("SUBSYSTEM"); + value.setServiceCode("aService"); + value.setServiceVersion("v1"); + value.setObjectType(XRoadObjectType.SERVICE); + clientType.setId(value); + + fetchOrganizationsTask.fetchOrganizationsForClient(clientType); + + mock.verify(() -> OrganizationUtil.getOrganizationIdsList(any(), any(), any(), any()), + times(1)); + mock.verify(() -> OrganizationUtil.getDataByIds(any(), any(), any(), any()), times(1)); + verify(organizationService, times(3)).saveOrganization(any()); + } + } +} diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchRestTaskTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchRestTaskTest.java new file mode 100644 index 00000000..c9aef8dc --- /dev/null +++ b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchRestTaskTest.java @@ -0,0 +1,97 @@ +/** + * + * The MIT License + * + * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) + * Copyright (c) 2016-2023 Finnish Digital Agency + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +package fi.vrk.xroad.catalog.collector.tasks; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.Semaphore; + +import org.awaitility.Awaitility; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.ApplicationContext; +import org.springframework.test.util.ReflectionTestUtils; + +import fi.vrk.xroad.catalog.collector.configuration.TaskPoolConfiguration; +import fi.vrk.xroad.catalog.collector.util.Endpoint; +import fi.vrk.xroad.catalog.collector.util.XRoadRestServiceIdentifierType; +import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; +import fi.vrk.xroad.catalog.persistence.CatalogService; + +@SpringBootTest(classes = TaskPoolConfiguration.class) +public class FetchRestTaskTest { + + @MockBean + CatalogService catalogService; + + @Autowired + private ApplicationContext applicationContext; + + @Test + public void testFetchRestTask() throws MalformedURLException, URISyntaxException, InterruptedException { + BlockingQueue restServices = new LinkedBlockingQueue<>(); + FetchRestTask fetchRestTask = new FetchRestTask(applicationContext, restServices); + Semaphore semaphore = new Semaphore(1); + ReflectionTestUtils.setField(fetchRestTask, "semaphore", semaphore); + Thread fetchRestRunner = Thread.ofVirtual().start(fetchRestTask::run); + + XRoadRestServiceIdentifierType service = new XRoadRestServiceIdentifierType(); + service.setObjectType(XRoadObjectType.SERVICE); + service.setXRoadInstance("INSTANCE"); + service.setMemberClass("CLASS"); + service.setMemberCode("CODE"); + service.setSubsystemCode("SUBSYSTEM"); + service.setServiceCode("aService"); + service.setServiceVersion("v1"); + service.setServiceType("REST"); + List endpointList = new ArrayList<>(); + endpointList.add(Endpoint.builder().method("GET").path("/getServices").build()); + service.setEndpoints(endpointList); + + restServices.add(service); + + Awaitility.await().atMost(Duration.ofSeconds(2)).until(() -> restServices.isEmpty()); + + semaphore.acquire(); + fetchRestRunner.interrupt(); + + verify(catalogService, times(0)).saveErrorLog(any()); + verify(catalogService, times(1)).saveRest(any(), any(), any()); + verify(catalogService, times(1)).saveEndpoint(any(), any(), any(), any()); + } +} diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchWsdlsTaskTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchWsdlsTaskTest.java new file mode 100644 index 00000000..d1b63f9c --- /dev/null +++ b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/FetchWsdlsTaskTest.java @@ -0,0 +1,95 @@ +/** + * + * The MIT License + * + * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) + * Copyright (c) 2016-2023 Finnish Digital Agency + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +package fi.vrk.xroad.catalog.collector.tasks; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.time.Duration; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.Semaphore; + +import org.awaitility.Awaitility; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.context.ApplicationContext; +import org.springframework.test.util.ReflectionTestUtils; + +import fi.vrk.xroad.catalog.collector.configuration.DevelopmentConfiguration; +import fi.vrk.xroad.catalog.collector.configuration.TaskPoolConfiguration; +import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; +import fi.vrk.xroad.catalog.collector.wsimport.XRoadServiceIdentifierType; +import fi.vrk.xroad.catalog.persistence.CatalogService; + +@SpringBootTest(classes = { DevelopmentConfiguration.class, + TaskPoolConfiguration.class }, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +public class FetchWsdlsTaskTest { + + @MockBean + CatalogService catalogService; + + @Autowired + private ApplicationContext applicationContext; + + @LocalServerPort + private int port; + + @Test + public void testFetchWsdl() throws MalformedURLException, URISyntaxException, InterruptedException { + TaskPoolConfiguration taskPoolConfiguration = applicationContext.getBean(TaskPoolConfiguration.class); + ReflectionTestUtils.setField(taskPoolConfiguration, "webservicesEndpoint", + "http://localhost:" + port + "/metaservices"); + BlockingQueue wsdlServices = new LinkedBlockingQueue<>(); + FetchWsdlsTask fetchWsdlsTask = new FetchWsdlsTask(applicationContext, wsdlServices); + Semaphore semaphore = new Semaphore(1); + ReflectionTestUtils.setField(fetchWsdlsTask, "semaphore", semaphore); + Thread fetchWsdlsRunner = Thread.ofVirtual().start(fetchWsdlsTask::run); + XRoadServiceIdentifierType service = new XRoadServiceIdentifierType(); + service.setObjectType(XRoadObjectType.SERVICE); + service.setXRoadInstance("INSTANCE"); + service.setMemberClass("CLASS"); + service.setMemberCode("CODE"); + service.setSubsystemCode("SUBSYSTEM"); + service.setServiceCode("aService"); + service.setServiceVersion("v1"); + wsdlServices.add(service); + + Awaitility.await().atMost(Duration.ofSeconds(2)).until(() -> wsdlServices.isEmpty()); + + semaphore.acquire(); + fetchWsdlsRunner.interrupt(); + + verify(catalogService, times(1)).saveWsdl(any(), any(), any()); + } +} diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/ListClientsTaskTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/ListClientsTaskTest.java new file mode 100644 index 00000000..7988a120 --- /dev/null +++ b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/ListClientsTaskTest.java @@ -0,0 +1,236 @@ +/** + * + * The MIT License + * + * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) + * Copyright (c) 2016-2023 Finnish Digital Agency + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +package fi.vrk.xroad.catalog.collector.tasks; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; + +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.context.ApplicationContext; +import org.springframework.test.util.ReflectionTestUtils; + +import fi.vrk.xroad.catalog.collector.configuration.TaskPoolConfiguration; +import fi.vrk.xroad.catalog.collector.util.ClientListUtil; +import fi.vrk.xroad.catalog.collector.wsimport.ClientList; +import fi.vrk.xroad.catalog.collector.wsimport.ClientType; +import fi.vrk.xroad.catalog.collector.wsimport.XRoadClientIdentifierType; +import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; +import fi.vrk.xroad.catalog.persistence.CatalogService; + +@SpringBootTest(classes = TaskPoolConfiguration.class) +public class ListClientsTaskTest { + + @Autowired + private ApplicationContext applicationContext; + + @MockBean + CatalogService catalogService; + + @Test + public void testOnReceiveWhenFetchUnlimited() throws Exception { + + try (MockedStatic mocked = Mockito.mockStatic(ClientListUtil.class)) { + TaskPoolConfiguration conf = applicationContext.getBean(TaskPoolConfiguration.class); + + ReflectionTestUtils.setField(conf, "fetchRunUnlimited", true); + ReflectionTestUtils.setField(conf, "fetchCompaniesRunUnlimited", true); + + ClientList clientList = new ClientList(); + clientList.getMember().add(createClientType(XRoadObjectType.MEMBER, "member1", null)); + clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member1", "sub1")); + clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member1", "sub2")); + clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member1", "sub3")); + clientList.getMember().add(createClientType(XRoadObjectType.MEMBER, "member2", null)); + clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member2", "sssub1")); + clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member2", "sssub2")); + + mocked.when(() -> ClientListUtil.clientListFromResponse(any(String.class))) + .thenReturn(clientList); + + final Queue listMethodsQueue = new ConcurrentLinkedQueue<>(); + final Queue fetchCompaniesQueue = new ConcurrentLinkedQueue<>(); + final Queue fetchOrganisationsQueue = new ConcurrentLinkedQueue<>(); + + ListClientsTask listClientsTask = new ListClientsTask(applicationContext, listMethodsQueue, + fetchCompaniesQueue, fetchOrganisationsQueue); + listClientsTask.run(); + + verify(catalogService, times(1)).saveAllMembersAndSubsystems(any()); + + assertEquals(5, listMethodsQueue.size()); + assertEquals(1, fetchCompaniesQueue.size()); + assertEquals(1, fetchOrganisationsQueue.size()); + } + } + + @Test + public void testWhenFetchNotUnlimitedAndTimeOutsideOfConfiguration() throws Exception { + try (MockedStatic mocked = mockStatic(ClientListUtil.class)) { + TaskPoolConfiguration conf = applicationContext.getBean(TaskPoolConfiguration.class); + + ReflectionTestUtils.setField(conf, "fetchRunUnlimited", false); + ReflectionTestUtils.setField(conf, "fetchTimeAfterHour", 23); + ReflectionTestUtils.setField(conf, "fetchTimeBeforeHour", 23); + + ClientList clientList = new ClientList(); + clientList.getMember().add(createClientType(XRoadObjectType.MEMBER, "member1", null)); + clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member1", "sub1")); + clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member1", "sub2")); + clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member1", "sub3")); + clientList.getMember().add(createClientType(XRoadObjectType.MEMBER, "member2", null)); + clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member2", "sssub1")); + clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member2", "sssub2")); + + mocked.when(() -> ClientListUtil.clientListFromResponse(any())).thenReturn(clientList); + + final Queue listMethodsQueue = new ConcurrentLinkedQueue<>(); + final Queue fetchCompaniesQueue = new ConcurrentLinkedQueue<>(); + final Queue fetchOrganisationsQueue = new ConcurrentLinkedQueue<>(); + + ListClientsTask listClientsTask = new ListClientsTask(applicationContext, listMethodsQueue, + fetchCompaniesQueue, fetchOrganisationsQueue); + listClientsTask.run(); + + verify(catalogService, times(0)).saveAllMembersAndSubsystems(any()); + + assertEquals(0, listMethodsQueue.size()); + assertEquals(0, fetchCompaniesQueue.size()); + assertEquals(0, fetchOrganisationsQueue.size()); + } + } + + @Test + public void testOnReceiveWhenFetchNotUnlimitedButTimeIsInBetween() throws Exception { + try (MockedStatic mocked = mockStatic(ClientListUtil.class)) { + TaskPoolConfiguration conf = applicationContext.getBean(TaskPoolConfiguration.class); + + ReflectionTestUtils.setField(conf, "fetchRunUnlimited", false); + ReflectionTestUtils.setField(conf, "fetchTimeAfterHour", 0); + ReflectionTestUtils.setField(conf, "fetchTimeBeforeHour", 23); + + ReflectionTestUtils.setField(conf, "fetchCompaniesRunUnlimited", false); + ReflectionTestUtils.setField(conf, "fetchCompaniesTimeAfterHour", 0); + ReflectionTestUtils.setField(conf, "fetchCompaniesTimeBeforeHour", 23); + + ClientList clientList = new ClientList(); + clientList.getMember().add(createClientType(XRoadObjectType.MEMBER, "member1", null)); + clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member1", "sub1")); + clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member1", "sub2")); + clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member1", "sub3")); + clientList.getMember().add(createClientType(XRoadObjectType.MEMBER, "member2", null)); + clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member2", "sssub1")); + clientList.getMember().add(createClientType(XRoadObjectType.SUBSYSTEM, "member2", "sssub2")); + + mocked.when(() -> ClientListUtil.clientListFromResponse(any())).thenReturn(clientList); + + final Queue listMethodsQueue = new ConcurrentLinkedQueue<>(); + final Queue fetchCompaniesQueue = new ConcurrentLinkedQueue<>(); + final Queue fetchOrganisationsQueue = new ConcurrentLinkedQueue<>(); + + ListClientsTask listClientsTask = new ListClientsTask(applicationContext, listMethodsQueue, + fetchCompaniesQueue, fetchOrganisationsQueue); + listClientsTask.run(); + + verify(catalogService, times(1)).saveAllMembersAndSubsystems(any()); + + assertEquals(5, listMethodsQueue.size()); + assertEquals(1, fetchCompaniesQueue.size()); + assertEquals(1, fetchOrganisationsQueue.size()); + } + } + + @Test + public void testOnReceiveWithEmptyMemberList() throws Exception { + try (MockedStatic mocked = mockStatic(ClientListUtil.class)) { + TaskPoolConfiguration conf = applicationContext.getBean(TaskPoolConfiguration.class); + + ReflectionTestUtils.setField(conf, "fetchRunUnlimited", true); + + ClientList clientList = new ClientList(); + mocked.when(() -> ClientListUtil.clientListFromResponse(any())).thenReturn(clientList); + + final Queue listMethodsQueue = new ConcurrentLinkedQueue<>(); + final Queue fetchCompaniesQueue = new ConcurrentLinkedQueue<>(); + final Queue fetchOrganisationsQueue = new ConcurrentLinkedQueue<>(); + + ListClientsTask listClientsTask = new ListClientsTask(applicationContext, listMethodsQueue, + fetchCompaniesQueue, fetchOrganisationsQueue); + listClientsTask.run(); + + verify(catalogService, times(1)).saveAllMembersAndSubsystems(any()); + assertEquals(0, listMethodsQueue.size()); + assertEquals(0, fetchCompaniesQueue.size()); + assertEquals(0, fetchOrganisationsQueue.size()); + } + } + + @Test + public void testSaveErrorLog() { + TaskPoolConfiguration conf = applicationContext.getBean(TaskPoolConfiguration.class); + + ReflectionTestUtils.setField(conf, "fetchRunUnlimited", true); + + final Queue listMethodsQueue = new ConcurrentLinkedQueue<>(); + final Queue fetchCompaniesQueue = new ConcurrentLinkedQueue<>(); + final Queue fetchOrganisationsQueue = new ConcurrentLinkedQueue<>(); + + ListClientsTask listClientsTask = new ListClientsTask(applicationContext, listMethodsQueue, + fetchCompaniesQueue, fetchOrganisationsQueue); + listClientsTask.run(); + + verify(catalogService, times(1)).saveErrorLog(any()); + assertEquals(0, listMethodsQueue.size()); + assertEquals(0, fetchCompaniesQueue.size()); + assertEquals(0, fetchOrganisationsQueue.size()); + } + + protected ClientType createClientType(XRoadObjectType objectType, String memberCode, String subsystemCode) { + ClientType c = new ClientType(); + XRoadClientIdentifierType xrcit = new XRoadClientIdentifierType(); + + xrcit.setXRoadInstance("FI"); + xrcit.setMemberClass("GOV"); + xrcit.setMemberCode(memberCode); + xrcit.setSubsystemCode(subsystemCode); + xrcit.setObjectType(objectType); + c.setId(xrcit); + c.setName(memberCode); + return c; + + } +} diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/ListMethodsTaskTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/ListMethodsTaskTest.java new file mode 100644 index 00000000..8b439168 --- /dev/null +++ b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/tasks/ListMethodsTaskTest.java @@ -0,0 +1,115 @@ +/** + * + * The MIT License + * + * Copyright (c) 2023- Nordic Institute for Interoperability Solutions (NIIS) + * Copyright (c) 2016-2023 Finnish Digital Agency + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +package fi.vrk.xroad.catalog.collector.tasks; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.time.Duration; +import java.util.Queue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.Semaphore; + +import org.awaitility.Awaitility; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.boot.test.web.server.LocalServerPort; +import org.springframework.context.ApplicationContext; +import org.springframework.test.util.ReflectionTestUtils; + +import fi.vrk.xroad.catalog.collector.configuration.DevelopmentConfiguration; +import fi.vrk.xroad.catalog.collector.configuration.TaskPoolConfiguration; +import fi.vrk.xroad.catalog.collector.util.XRoadRestServiceIdentifierType; +import fi.vrk.xroad.catalog.collector.wsimport.ClientType; +import fi.vrk.xroad.catalog.collector.wsimport.XRoadClientIdentifierType; +import fi.vrk.xroad.catalog.collector.wsimport.XRoadObjectType; +import fi.vrk.xroad.catalog.collector.wsimport.XRoadServiceIdentifierType; +import fi.vrk.xroad.catalog.persistence.CatalogService; + +@SpringBootTest(classes = { DevelopmentConfiguration.class, + TaskPoolConfiguration.class }, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT) +public class ListMethodsTaskTest { + + @MockBean + CatalogService catalogService; + + @Autowired + private ApplicationContext applicationContext; + + @LocalServerPort + private int port; + + @Test + public void testListMethodsTaskSavesServicesAndGetsDescriptors() + throws MalformedURLException, URISyntaxException, InterruptedException { + TaskPoolConfiguration taskPoolConfiguration = applicationContext.getBean(TaskPoolConfiguration.class); + ReflectionTestUtils.setField(taskPoolConfiguration, "securityServerHost", "http://localhost:" + port); + ReflectionTestUtils.setField(taskPoolConfiguration, "webservicesEndpoint", + "http://localhost:" + port + "/metaservices"); + BlockingQueue listedClients = new LinkedBlockingQueue<>(); + Queue wsdlServices = new LinkedBlockingQueue<>(); + Queue restServices = new LinkedBlockingQueue<>(); + Queue openApiServices = new LinkedBlockingQueue<>(); + ListMethodsTask listMethodsTask = new ListMethodsTask(applicationContext, listedClients, wsdlServices, + restServices, openApiServices); + Semaphore semaphore = new Semaphore(1); + ReflectionTestUtils.setField(listMethodsTask, "semaphore", semaphore); + Thread listMethodsRunner = Thread.ofVirtual().start(listMethodsTask::run); + ClientType clientType = new ClientType(); + XRoadClientIdentifierType value = new XRoadClientIdentifierType(); + value.setXRoadInstance("INSTANCE"); + value.setMemberClass("CLASS"); + value.setMemberCode("CODE"); + value.setSubsystemCode("SUBSYSTEM"); + value.setServiceCode("aService"); + value.setServiceVersion("v1"); + value.setObjectType(XRoadObjectType.SUBSYSTEM); + clientType.setId(value); + listedClients.add(clientType); + + Awaitility.await().atMost(Duration.ofSeconds(2)).until(() -> listedClients.isEmpty()); + + semaphore.acquire(); + listMethodsRunner.interrupt(); + + verify(catalogService, times(1)).saveServices(any(), any()); + + assertEquals(3, wsdlServices.size()); + + assertEquals(0, restServices.size()); + + assertEquals(0, openApiServices.size()); + } + +} diff --git a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/util/XRoadClientTest.java b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/util/XRoadClientTest.java index 2e44a7a2..1740da45 100644 --- a/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/util/XRoadClientTest.java +++ b/xroad-catalog-collector/src/test/java/fi/vrk/xroad/catalog/collector/util/XRoadClientTest.java @@ -15,8 +15,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import java.net.MalformedURLException; -import java.net.URL; +import java.net.URI; +import java.net.URISyntaxException; import java.util.List; import org.junit.jupiter.api.Test; @@ -81,9 +81,9 @@ private XRoadServiceIdentifierType getDefaultService() { } @Test - public void testCallListMethods() throws MalformedURLException { + public void testCallListMethods() throws URISyntaxException { XRoadClientIdentifierType client = getDefaultClient(); - XRoadClient xRoadClient = new XRoadClient(client, new URL(webservicesEndpoint)); + XRoadClient xRoadClient = new XRoadClient(client, new URI(webservicesEndpoint)); assertFalse(xRoadClient == null); XRoadClientIdentifierType client2 = getDefaultClient(); client2.setSubsystemCode(subsystemCode); @@ -92,9 +92,9 @@ public void testCallListMethods() throws MalformedURLException { } @Test - public void testCallGetWsdl() throws MalformedURLException { + public void testCallGetWsdl() throws Exception { XRoadClientIdentifierType client = getDefaultClient(); - XRoadClient xRoadClient = new XRoadClient(client, new URL(webservicesEndpoint)); + XRoadClient xRoadClient = new XRoadClient(client, new URI(webservicesEndpoint)); assertFalse(xRoadClient == null); XRoadServiceIdentifierType service = getDefaultService(); String wsdl = xRoadClient.getWsdl(service, catalogService); diff --git a/xroad-catalog-collector/src/test/resources/mock/companies/company.json b/xroad-catalog-collector/src/test/resources/mock/companies/company.json new file mode 100644 index 00000000..cbde33fc --- /dev/null +++ b/xroad-catalog-collector/src/test/resources/mock/companies/company.json @@ -0,0 +1,166 @@ +{ + "type": "fi.prh.opendata.bis", + "version": "1", + "totalResults": -1, + "resultsFrom": 0, + "previousResultsUri": null, + "nextResultsUri": null, + "exceptionNoticeUri": null, + "results": [ + { + "businessId": "1234567-1", + "name": "Test Company 1", + "registrationDate": "2020-01-01", + "companyForm": "OY", + "detailsUri": null, + "liquidations": [], + "names": [ + { + "order": 0, + "version": 1, + "name": "Test Company 1", + "registrationDate": "2020-01-01", + "endDate": null, + "source": 3 + } + ], + "auxiliaryNames": [], + "addresses": [ + { + "careOf": null, + "street": "Test Street 1", + "postCode": "12345", + "type": 2, + "version": 1, + "city": "Test City", + "country": null, + "registrationDate": "2020-01-01", + "endDate": null, + "language": "FI", + "source": 0 + } + ], + "companyForms": [ + { + "version": 1, + "name": "Limited company", + "type": null, + "registrationDate": "2020-01-01", + "endDate": null, + "language": "EN", + "source": 3 + }, + { + "version": 1, + "name": "Osakeyhtiö", + "type": "OY", + "registrationDate": "2020-01-01", + "endDate": null, + "language": "FI", + "source": 3 + }, + { + "version": 1, + "name": "Aktiebolag", + "type": "AB", + "registrationDate": "2020-01-01", + "endDate": null, + "language": "SE", + "source": 3 + } + ], + "businessLines": [], + "languages": [], + "registedOffices": [ + { + "order": 0, + "version": 1, + "name": "Test City", + "registrationDate": "2020-01-01", + "endDate": null, + "language": "EN", + "source": 0 + }, + { + "order": 0, + "version": 1, + "name": "Test City", + "registrationDate": "2020-01-01", + "endDate": null, + "language": "FI", + "source": 0 + }, + { + "order": 0, + "version": 1, + "name": "Test City", + "registrationDate": "2020-01-01", + "endDate": null, + "language": "SE", + "source": 0 + } + ], + "contactDetails": [ + { + "version": 1, + "value": "+123 123 1234", + "type": "Matkapuhelin", + "registrationDate": "2020-01-01", + "endDate": null, + "language": "FI", + "source": 0 + }, + { + "version": 1, + "value": "+123 123 1234", + "type": "Mobiltelefon", + "registrationDate": "2020-01-01", + "endDate": null, + "language": "SE", + "source": 0 + }, + { + "version": 1, + "value": "+123 123 1234", + "type": "Mobile phone", + "registrationDate": "2020-01-01", + "endDate": null, + "language": "EN", + "source": 0 + } + ], + "registeredEntries": [ + { + "authority": 2, + "register": 1, + "status": 2, + "registrationDate": "2020-01-01", + "endDate": null, + "statusDate": "2020-01-01", + "language": "FI", + "description": "Rekisteröimätön" + }, + { + "authority": 2, + "register": 1, + "status": 2, + "registrationDate": "2020-01-01", + "endDate": null, + "statusDate": "2020-01-01", + "language": "SE", + "description": "Oregistrerad" + }, + { + "authority": 2, + "register": 1, + "status": 2, + "registrationDate": "2020-01-01", + "endDate": null, + "statusDate": "2020-01-01", + "language": "EN", + "description": "Unregistered" + } + ], + "businessIdChanges": [] + } + ]} diff --git a/xroad-catalog-collector/src/test/resources/mock/companies/getCompanies.json b/xroad-catalog-collector/src/test/resources/mock/companies/getCompanies.json new file mode 100644 index 00000000..d222774f --- /dev/null +++ b/xroad-catalog-collector/src/test/resources/mock/companies/getCompanies.json @@ -0,0 +1,39 @@ +{ + "type":"fi.prh.opendata.bis", + "version":"1", + "totalResults":4, + "resultsFrom":0, + "previousResultsUri":null, + "nextResultsUri":"http://localhost/v1?totalResults=true&maxResults=100&resultsFrom=100&companyRegistrationFrom=1970-01-01", + "exceptionNoticeUri":null, + "results":[ + { + "businessId":"1234567-1", + "name":"Test Company 1", + "registrationDate":"2020-01-01", + "companyForm":"OY", + "detailsUri":"http://localhost/v1/1234567-1" + }, + { + "businessId":"1234567-2", + "name":"Test Company 2", + "registrationDate":"2022-02-09", + "companyForm":"OY", + "detailsUri":"http://localhost/v1/1234567-2" + }, + { + "businessId":"1234567-3", + "name":"Test Company 3", + "registrationDate":"2012-06-08", + "companyForm":"OY", + "detailsUri":"http://localhost/v1/1234567-3" + }, + { + "businessId":"1234567-4", + "name":"Test Company 4", + "registrationDate":"2019-05-09", + "companyForm":"OY", + "detailsUri":"http://localhost/v1/1234567-4" + } + ] +} diff --git a/xroad-catalog-collector/src/test/resources/mock/organizations/organizationsById.json b/xroad-catalog-collector/src/test/resources/mock/organizations/organizationsById.json new file mode 100644 index 00000000..18dfc13b --- /dev/null +++ b/xroad-catalog-collector/src/test/resources/mock/organizations/organizationsById.json @@ -0,0 +1,926 @@ +[ + { + "id": "12ea34r3-1k23-412r-9142-1442asd13131", + "sourceId": null, + "oid": null, + "parentOrganizationId": null, + "organizationRootId": "12ea34r3-1k23-412r-9142-1442asd13131", + "municipality": { + "code": "111", + "name": [ + { + "value": "Some Municipality", + "language": "fi" + }, + { + "value": "Some Municipality", + "language": "sv" + }, + { + "value": "Some Municipality", + "language": "en" + } + + ] + }, + "organizationType": "Municipality", + "businessCode": "1234567-1", + "businessName": null, + "organizationNames": [ + { + "language": "fi", + "value": "Some Name", + "type": "Name" + } + ], + "displayNameType": [ + { + "type": "Name", + "language": "fi" + } + ], + "areaType": null, + "areas": null, + "organizationDescriptions": [ + { + "language": "fi", + "value": "Some Description FI", + "type": "Summary" + }, + { + "language": "fi", + "value": "Some Description FI", + "type": "Description" + } + ], + "emails": [ + { + "description": "Some email", + "value": "info@niis.org", + "language": "fi" + }, + { + "description": "Some email", + "value": "info@niis.org", + "language": "fi" + } + ], + "phoneNumbers": [ + { + "additionalInformation": "Additional info field", + "serviceChargeType": "Chargeable", + "chargeDescription": null, + "prefixNumber": "+000", + "isFinnishServiceNumber": false, + "number": "123456789", + "language": "fi" + } + ], + "webPages": [ + { + "url": "https://niis.org", + "value": "Some description", + "language": "fi" + } + ], + "addresses": [ + { + "type": "Visiting", + "subType": "Street", + "postOfficeBoxAddress": null, + "streetAddress": { + "street": [ + { + "value": "Address part", + "language": "fi" + }, + { + "value": "Address part", + "language": "sv" + }, + { + "value": "Address part", + "language": "en" + }, + { + "value": "Address part", + "language": "se" + }, + { + "value": "Address part", + "language": "smn" + }, + { + "value": "Address part", + "language": "sms" + } + ], + "streetNumber": "1", + "postalCode": "12345", + "postOffice": [ + { + "value": "Some post office", + "language": "se" + }, + { + "value": "Some post office", + "language": "smn" + }, + { + "value": "Some post office", + "language": "sv" + }, + { + "value": "Some post office", + "language": "sms" + }, + { + "value": "Some post office", + "language": "en" + }, + { + "value": "Some post office", + "language": "fi" + } + ], + "municipality": { + "code": "123", + "name": [ + { + "value": "Some Municipality", + "language": "fi" + }, + { + "value": "Some Municipality", + "language": "sv" + }, + { + "value": "Some Municipality", + "language": "en" + } + + ] + }, + "additionalInformation": [], + "latitude": "1000001.001", + "longitude": "100001.001", + "coordinateState": "Ok" + }, + "otherAddress": null, + "foreignAddress": null, + "country": "FI" + }, + { + "type": "Postal", + "subType": "Street", + "postOfficeBoxAddress": null, + "streetAddress": { + "street": [ + { + "value": "Address part", + "language": "fi" + }, + { + "value": "Address part", + "language": "sv" + }, + { + "value": "Address part", + "language": "en" + }, + { + "value": "Address part", + "language": "se" + }, + { + "value": "Address part", + "language": "smn" + }, + { + "value": "Address part", + "language": "sms" + } + ], + "streetNumber": "1", + "postalCode": "12314", + "postOffice": [ + { + "value": "Some post office", + "language": "se" + }, + { + "value": "Some post office", + "language": "smn" + }, + { + "value": "Some post office", + "language": "sv" + }, + { + "value": "Some post office", + "language": "sms" + }, + { + "value": "Some post office", + "language": "en" + }, + { + "value": "Some post office", + "language": "fi" + } + ], + "municipality": { + "code": "124", + "name": [ + { + "value": "Some Municipality", + "language": "fi" + }, + { + "value": "Some Municipality", + "language": "sv" + }, + { + "value": "Some Municipality", + "language": "en" + } + + ] + }, + "additionalInformation": [ + { + "value": null, + "language": "fi" + } + ], + "latitude": "1000001.001", + "longitude": "100001.001", + "coordinateState": "Ok" + }, + "otherAddress": null, + "foreignAddress": null, + "country": "FI" + } + ], + "electronicInvoicings": [ + { + "operatorCode": "123142535345", + "electronicInvoicingAddress": "123434523524", + "additionalInformation": [ + { + "value": "Some additional information", + "language": "fi" + } + + ] + }], + "publishingStatus": "Published", + "services": [ + { + "service": { + "id": "13123132-1321-2342-asdf-1321sfasf123", + "name": "Some Service 1" + }, + "roleType": "OtherResponsible", + "provisionType": null, + "additionalInformation": [] + }, + { + "service": { + "id": "1231asd1-sdf1-asfd-1231-afsdfaas1231", + "name": "Some Service 2" + }, + "roleType": "Responsible", + "provisionType": null, + "additionalInformation": [] + }, + { + "service": { + "id": "sdkljlk1-123l-1231-sdfa-1231231asdfa", + "name": "Some Service 3" + }, + "roleType": "Producer", + "provisionType": "SelfProduced", + "additionalInformation": [] + } + ], + "modified": "2020-01-12T12:11:11.131231", + "subOrganizations": null + }, + { + "id": "12ea34r3-1k23-412r-9142-1442asd13132", + "sourceId": null, + "oid": null, + "parentOrganizationId": null, + "organizationRootId": "12ea34r3-1k23-412r-9142-1442asd13132", + "municipality": { + "code": "111", + "name": [ + { + "value": "Some Municipality", + "language": "fi" + }, + { + "value": "Some Municipality", + "language": "sv" + }, + { + "value": "Some Municipality", + "language": "en" + } + + ] + }, + "organizationType": "Municipality", + "businessCode": "1234567-1", + "businessName": null, + "organizationNames": [ + { + "language": "fi", + "value": "Some Name", + "type": "Name" + } + ], + "displayNameType": [ + { + "type": "Name", + "language": "fi" + } + ], + "areaType": null, + "areas": null, + "organizationDescriptions": [ + { + "language": "fi", + "value": "Some Description FI", + "type": "Summary" + }, + { + "language": "fi", + "value": "Some Description FI", + "type": "Description" + } + ], + "emails": [ + { + "description": "Some email", + "value": "info@niis.org", + "language": "fi" + }, + { + "description": "Some email", + "value": "info@niis.org", + "language": "fi" + } + ], + "phoneNumbers": [ + { + "additionalInformation": "Additional info field", + "serviceChargeType": "Chargeable", + "chargeDescription": null, + "prefixNumber": "+000", + "isFinnishServiceNumber": false, + "number": "123456789", + "language": "fi" + } + ], + "webPages": [ + { + "url": "https://niis.org", + "value": "Some description", + "language": "fi" + } + ], + "addresses": [ + { + "type": "Visiting", + "subType": "Street", + "postOfficeBoxAddress": null, + "streetAddress": { + "street": [ + { + "value": "Address part", + "language": "fi" + }, + { + "value": "Address part", + "language": "sv" + }, + { + "value": "Address part", + "language": "en" + }, + { + "value": "Address part", + "language": "se" + }, + { + "value": "Address part", + "language": "smn" + }, + { + "value": "Address part", + "language": "sms" + } + ], + "streetNumber": "1", + "postalCode": "12345", + "postOffice": [ + { + "value": "Some post office", + "language": "se" + }, + { + "value": "Some post office", + "language": "smn" + }, + { + "value": "Some post office", + "language": "sv" + }, + { + "value": "Some post office", + "language": "sms" + }, + { + "value": "Some post office", + "language": "en" + }, + { + "value": "Some post office", + "language": "fi" + } + ], + "municipality": { + "code": "123", + "name": [ + { + "value": "Some Municipality", + "language": "fi" + }, + { + "value": "Some Municipality", + "language": "sv" + }, + { + "value": "Some Municipality", + "language": "en" + } + + ] + }, + "additionalInformation": [], + "latitude": "1000001.001", + "longitude": "100001.001", + "coordinateState": "Ok" + }, + "otherAddress": null, + "foreignAddress": null, + "country": "FI" + }, + { + "type": "Postal", + "subType": "Street", + "postOfficeBoxAddress": null, + "streetAddress": { + "street": [ + { + "value": "Address part", + "language": "fi" + }, + { + "value": "Address part", + "language": "sv" + }, + { + "value": "Address part", + "language": "en" + }, + { + "value": "Address part", + "language": "se" + }, + { + "value": "Address part", + "language": "smn" + }, + { + "value": "Address part", + "language": "sms" + } + ], + "streetNumber": "1", + "postalCode": "12314", + "postOffice": [ + { + "value": "Some post office", + "language": "se" + }, + { + "value": "Some post office", + "language": "smn" + }, + { + "value": "Some post office", + "language": "sv" + }, + { + "value": "Some post office", + "language": "sms" + }, + { + "value": "Some post office", + "language": "en" + }, + { + "value": "Some post office", + "language": "fi" + } + ], + "municipality": { + "code": "124", + "name": [ + { + "value": "Some Municipality", + "language": "fi" + }, + { + "value": "Some Municipality", + "language": "sv" + }, + { + "value": "Some Municipality", + "language": "en" + } + + ] + }, + "additionalInformation": [ + { + "value": null, + "language": "fi" + } + ], + "latitude": "1000001.001", + "longitude": "100001.001", + "coordinateState": "Ok" + }, + "otherAddress": null, + "foreignAddress": null, + "country": "FI" + } + ], + "electronicInvoicings": [ + { + "operatorCode": "123142535345", + "electronicInvoicingAddress": "123434523524", + "additionalInformation": [ + { + "value": "Some additional information", + "language": "fi" + } + + ] + }], + "publishingStatus": "Published", + "services": [ + { + "service": { + "id": "13123132-1321-2342-asdf-1321sfasf123", + "name": "Some Service 1" + }, + "roleType": "OtherResponsible", + "provisionType": null, + "additionalInformation": [] + }, + { + "service": { + "id": "1231asd1-sdf1-asfd-1231-afsdfaas1231", + "name": "Some Service 2" + }, + "roleType": "Responsible", + "provisionType": null, + "additionalInformation": [] + }, + { + "service": { + "id": "sdkljlk1-123l-1231-sdfa-1231231asdfa", + "name": "Some Service 3" + }, + "roleType": "Producer", + "provisionType": "SelfProduced", + "additionalInformation": [] + } + ], + "modified": "2020-01-12T12:11:11.131231", + "subOrganizations": null + }, + { + "id": "12ea34r3-1k23-412r-9142-1442asd13133", + "sourceId": null, + "oid": null, + "parentOrganizationId": null, + "organizationRootId": "12ea34r3-1k23-412r-9142-1442asd13131", + "municipality": { + "code": "111", + "name": [ + { + "value": "Some Municipality", + "language": "fi" + }, + { + "value": "Some Municipality", + "language": "sv" + }, + { + "value": "Some Municipality", + "language": "en" + } + + ] + }, + "organizationType": "Municipality", + "businessCode": "1234567-1", + "businessName": null, + "organizationNames": [ + { + "language": "fi", + "value": "Some Name", + "type": "Name" + } + ], + "displayNameType": [ + { + "type": "Name", + "language": "fi" + } + ], + "areaType": null, + "areas": null, + "organizationDescriptions": [ + { + "language": "fi", + "value": "Some Description FI", + "type": "Summary" + }, + { + "language": "fi", + "value": "Some Description FI", + "type": "Description" + } + ], + "emails": [ + { + "description": "Some email", + "value": "info@niis.org", + "language": "fi" + }, + { + "description": "Some email", + "value": "info@niis.org", + "language": "fi" + } + ], + "phoneNumbers": [ + { + "additionalInformation": "Additional info field", + "serviceChargeType": "Chargeable", + "chargeDescription": null, + "prefixNumber": "+000", + "isFinnishServiceNumber": false, + "number": "123456789", + "language": "fi" + } + ], + "webPages": [ + { + "url": "https://niis.org", + "value": "Some description", + "language": "fi" + } + ], + "addresses": [ + { + "type": "Visiting", + "subType": "Street", + "postOfficeBoxAddress": null, + "streetAddress": { + "street": [ + { + "value": "Address part", + "language": "fi" + }, + { + "value": "Address part", + "language": "sv" + }, + { + "value": "Address part", + "language": "en" + }, + { + "value": "Address part", + "language": "se" + }, + { + "value": "Address part", + "language": "smn" + }, + { + "value": "Address part", + "language": "sms" + } + ], + "streetNumber": "1", + "postalCode": "12345", + "postOffice": [ + { + "value": "Some post office", + "language": "se" + }, + { + "value": "Some post office", + "language": "smn" + }, + { + "value": "Some post office", + "language": "sv" + }, + { + "value": "Some post office", + "language": "sms" + }, + { + "value": "Some post office", + "language": "en" + }, + { + "value": "Some post office", + "language": "fi" + } + ], + "municipality": { + "code": "123", + "name": [ + { + "value": "Some Municipality", + "language": "fi" + }, + { + "value": "Some Municipality", + "language": "sv" + }, + { + "value": "Some Municipality", + "language": "en" + } + + ] + }, + "additionalInformation": [], + "latitude": "1000001.001", + "longitude": "100001.001", + "coordinateState": "Ok" + }, + "otherAddress": null, + "foreignAddress": null, + "country": "FI" + }, + { + "type": "Postal", + "subType": "Street", + "postOfficeBoxAddress": null, + "streetAddress": { + "street": [ + { + "value": "Address part", + "language": "fi" + }, + { + "value": "Address part", + "language": "sv" + }, + { + "value": "Address part", + "language": "en" + }, + { + "value": "Address part", + "language": "se" + }, + { + "value": "Address part", + "language": "smn" + }, + { + "value": "Address part", + "language": "sms" + } + ], + "streetNumber": "1", + "postalCode": "12314", + "postOffice": [ + { + "value": "Some post office", + "language": "se" + }, + { + "value": "Some post office", + "language": "smn" + }, + { + "value": "Some post office", + "language": "sv" + }, + { + "value": "Some post office", + "language": "sms" + }, + { + "value": "Some post office", + "language": "en" + }, + { + "value": "Some post office", + "language": "fi" + } + ], + "municipality": { + "code": "124", + "name": [ + { + "value": "Some Municipality", + "language": "fi" + }, + { + "value": "Some Municipality", + "language": "sv" + }, + { + "value": "Some Municipality", + "language": "en" + } + + ] + }, + "additionalInformation": [ + { + "value": null, + "language": "fi" + } + ], + "latitude": "1000001.001", + "longitude": "100001.001", + "coordinateState": "Ok" + }, + "otherAddress": null, + "foreignAddress": null, + "country": "FI" + } + ], + "electronicInvoicings": [ + { + "operatorCode": "123142535345", + "electronicInvoicingAddress": "123434523524", + "additionalInformation": [ + { + "value": "Some additional information", + "language": "fi" + } + + ] + }], + "publishingStatus": "Published", + "services": [ + { + "service": { + "id": "13123132-1321-2342-asdf-1321sfasf123", + "name": "Some Service 1" + }, + "roleType": "OtherResponsible", + "provisionType": null, + "additionalInformation": [] + }, + { + "service": { + "id": "1231asd1-sdf1-asfd-1231-afsdfaas1231", + "name": "Some Service 2" + }, + "roleType": "Responsible", + "provisionType": null, + "additionalInformation": [] + }, + { + "service": { + "id": "sdkljlk1-123l-1231-sdfa-1231231asdfa", + "name": "Some Service 3" + }, + "roleType": "Producer", + "provisionType": "SelfProduced", + "additionalInformation": [] + } + ], + "modified": "2020-01-12T12:11:11.131231", + "subOrganizations": null + } +] diff --git a/xroad-catalog-collector/src/test/resources/mock/xroad/openapi/openapi.json b/xroad-catalog-collector/src/test/resources/mock/xroad/openapi/openapi.json new file mode 100644 index 00000000..3fbf3bbe --- /dev/null +++ b/xroad-catalog-collector/src/test/resources/mock/xroad/openapi/openapi.json @@ -0,0 +1,68 @@ +{ + "swagger": "2.0", + "info": { + "version": "1.0.0", + "title": "Swagger Petstore", + "description": "A sample API that uses a petstore as an example to demonstrate features in the swagger-2.0 specification", + "termsOfService": "http://swagger.io/terms/", + "contact": { + "name": "Swagger API Team" + }, + "license": { + "name": "MIT" + } + }, + "host": "petstore.swagger.io", + "basePath": "/api", + "schemes": [ + "http" + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "paths": { + "/pets": { + "get": { + "description": "Returns all pets from the system that the user has access to", + "produces": [ + "application/json" + ], + "responses": { + "200": { + "description": "A list of pets.", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/Pet" + } + } + } + } + } + } + }, + "definitions": { + "Pet": { + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "integer", + "format": "int64" + }, + "name": { + "type": "string" + }, + "tag": { + "type": "string" + } + } + } + } +} From d57e186badcdf7bc444630b1242150388d2ae1bf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Jun 2024 06:44:03 +0000 Subject: [PATCH 2/2] Bump com.google.guava:guava in the java-minor-patch group Bumps the java-minor-patch group with 1 update: [com.google.guava:guava](https://github.com/google/guava). Updates `com.google.guava:guava` from 33.2.0-jre to 33.2.1-jre - [Release notes](https://github.com/google/guava/releases) - [Commits](https://github.com/google/guava/commits) --- updated-dependencies: - dependency-name: com.google.guava:guava dependency-type: direct:production update-type: version-update:semver-patch dependency-group: java-minor-patch ... Signed-off-by: dependabot[bot] --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 1ac07659..7f51ecf6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -2,7 +2,7 @@ commons-beanutils = "1.9.4" commons-csv = "1.11.0" cxf = "4.0.4" -guava = "33.2.0-jre" +guava = "33.2.1-jre" h2 = "2.2.224" httpclient = "5.3.1" jackson = "2.17.1"