From 4c5301ffbbde9271990b2926175f4b3cd920587a Mon Sep 17 00:00:00 2001 From: Gavin Date: Mon, 8 Apr 2024 16:29:48 -0700 Subject: [PATCH 1/2] Use jitpack jars and publish shadow jar Javadocs link properly Also remove ServiceChecker, it's completely unused at this point --- build.gradle | 134 +++++++-------- docsource/buildinitclient.rst | 48 +++--- docsource/developers.rst | 2 +- docsource/releasenotes.rst | 1 + .../kbase/common/service/ServiceChecker.java | 121 ------------- .../us/kbase/workspace/WorkspaceServer.java | 34 ++-- .../workspace/kbase/InitWorkspaceServer.java | 16 +- .../kbase/WorkspaceServerMethods.java | 10 +- .../java/us/kbase/test/common/TestCommon.java | 2 +- .../test/service/ServiceCheckerTest.java | 117 ------------- .../test/typedobj/BasicValidationTest.java | 2 +- .../test/typedobj/DetailedValidationTest.java | 2 +- .../kbase/test/typedobj/IdProcessingTest.java | 2 +- .../test/typedobj/MetadataExtractionTest.java | 2 +- .../typedobj/ObjectExtractionByPathTest.java | 2 +- .../typedobj/db/MongoTypeStorageTest.java | 2 +- .../test/typedobj/db/TypeRegisteringTest.java | 2 +- .../controllers/arango/ArangoController.java | 6 +- .../blobstore/BlobstoreController.java | 162 ------------------ .../handle/HandleServiceController.java | 6 +- .../controllers/minio/MinioController.java | 98 ----------- .../sample/SampleServiceController.java | 4 +- .../workspace/WorkspaceController.java | 4 +- .../database/mongo/GridFSBlobStoreTest.java | 4 +- .../database/mongo/MongoInternalsTest.java | 2 +- .../database/mongo/MongoStartUpTest.java | 2 +- .../database/mongo/MongoWorkspaceDBTest.java | 2 +- .../mongo/S3BlobStoreIntegrationTest.java | 4 +- .../database/mongo/SchemaUpdaterTest.java | 2 +- .../HandleAndBytestreamIntegrationTest.java | 19 +- .../workspace/kbase/JSONRPCLayerTest.java | 59 +++---- .../workspace/kbase/JSONRPCLayerTester.java | 28 ++- .../kbase/KBaseWorkspaceConfigTest.java | 2 +- .../test/workspace/kbase/LoggingTest.java | 2 +- .../kbase/SampleServiceIntegrationTest.java | 6 +- .../workspace/kbase/SchemaUpdaterCLITest.java | 2 +- .../workspace/kbase/TypeDelegationTest.java | 2 +- .../workspace/ObjectResolverTest.java | 2 +- .../WorkspaceIntegrationWithGridFSTest.java | 2 +- .../workspace/workspace/WorkspaceTester.java | 6 +- 40 files changed, 201 insertions(+), 724 deletions(-) delete mode 100644 src/main/java/us/kbase/common/service/ServiceChecker.java delete mode 100644 src/test/java/us/kbase/test/common/test/service/ServiceCheckerTest.java delete mode 100644 src/test/java/us/kbase/test/workspace/controllers/blobstore/BlobstoreController.java delete mode 100644 src/test/java/us/kbase/test/workspace/controllers/minio/MinioController.java diff --git a/build.gradle b/build.gradle index f7099d674..904f9f3e1 100644 --- a/build.gradle +++ b/build.gradle @@ -11,10 +11,19 @@ plugins { id 'java' id 'war' id 'jacoco' + id 'maven-publish' id 'org.ajoberstar.grgit' version '4.1.1' id 'com.github.johnrengelman.shadow' version '8.1.1' } +group = 'com.github.kbase' + +var VER_JAVA_COMMON = "0.3.0" +var VER_AUTH2_CLIENT = "0.5.0" +var VER_AUTH2_SHADOW = "0.7.1" +var VER_JACKSON = "2.9.9" + + var DEFAULT_URL = "https://ci.kbase.us/services/ws" var BUILD_DOC_ROOT = "$buildDir/docs/" @@ -32,6 +41,14 @@ var LOC_DOC_HTML = "$rootDir/docshtml" repositories { mavenCentral() + maven { + name = "Jitpack" + url = 'https://jitpack.io' + } + maven { + name = "Clojars" + url = "https://repo.clojars.org/" + } } task buildGitCommitFile { @@ -55,12 +72,6 @@ javadoc { * building the client javadocs. If we ever need the full javadocs make a new task for the * client java docs */ - /* TODO DOCS currently java-common and auth links don't work. ROI for fixing them is low. - * Ideally, in the future make github actions publish docs on release and ref them - * here. - * https://github.com/marketplace/actions/deploy-publish-javadoc - * https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-gradle-registry - */ /* TODO DOCS the current sdk documenation looks like invalid html to javadoc * need to go through and remove * also some spots with < / > that need to be wrapped with {@code } in internal code @@ -68,6 +79,8 @@ javadoc { failOnError = false options { links "https://docs.oracle.com/en/java/javase/11/docs/api/" + links "https://javadoc.jitpack.io/com/github/kbase/auth2_client_java/$VER_AUTH2_CLIENT/javadoc/" + links "https://javadoc.jitpack.io/com/github/kbase/java_common/$VER_JAVA_COMMON/javadoc/" } include "**/workspace/*.java" exclude "**/workspace/WorkspaceServer.java" @@ -163,12 +176,16 @@ shadowJar { // Be careful when updating jars - you may want to set the duplicates strategy to WARN // to see if any of the jars are shadowing the others when building the fat jar, which // has been the case in the past - dependsOn buildDocs + // Can't include the full documentation as jitpack doesn't have sphinx installed + dependsOn javadoc duplicatesStrategy = DuplicatesStrategy.EXCLUDE archiveAppendix = 'test-shadow' from sourceSets.test.output - // we don't want to package the auth shadow jar, so no test runtime classpath - configurations = [project.configurations.runtimeClasspath] + + // don't shadow the shadow jar, don't need it and it's big + dependencies { + exclude(dependency("com.github.kbase:auth2:$VER_AUTH2_SHADOW")) + } enableRelocation true relocationPrefix 'us.kbase.workspace.shadow' @@ -186,6 +203,19 @@ war { from(BUILD_OTHER_DOC_DIR) { into "/WEB-INF/classes/$IN_JAR_DOC_DIR" } } +// This is a filthy hack to get jitpack.io to not try to build the war file. +// Since the war depends on buildDocs which depends on sphinx, the build will fail +// It'd be better if it didn't call assemble and just publishToMavenLocal, +// but no such luck +gradle.taskGraph.whenReady { taskGraph -> + def tasks = taskGraph.getAllTasks() + if (tasks.find {it.name == 'assemble'}) { + tasks.findAll {it.name == 'war' || it.name == "buildDocs"}.each { task -> + task.enabled = false + } + } +} + /* SDK compile notes: * kb-sdk starts a docker container in interactive mode. Gradle's commandLine doesn't allocate * a tty so the command fails. @@ -291,49 +321,38 @@ configurations { testimpl.extendsFrom testImplementation } -def fromURL = { url, name -> - File file = new File("$buildDir/download/${name}.jar") - file.parentFile.mkdirs() - if (!file.exists()) { - new URL(url).withInputStream { downloadStream -> - file.withOutputStream { fileOut -> - fileOut << downloadStream - } +publishing { + publications { + shadow(MavenPublication) { publication -> + project.shadow.component(publication) + artifactId = "workspace_deluxe-test-shadow-all" } } - files(file.absolutePath) } dependencies { // ### General application dependencies ### - implementation fromURL( - 'https://github.com/kbase/jars/raw/master/lib/jars/kbase/handle/AbstractHandleClient-1.0.0.jar', - 'AbstractHandleClient-1.0.0' - ) - implementation fromURL( - 'https://github.com/kbase/jars/raw/master/lib/jars/kbase/auth/kbase-auth-0.4.4.jar', - 'kbase-auth-0.4.4' - ) - implementation fromURL( - 'https://github.com/kbase/jars/raw/master/lib/jars/kbase/kidl/kbase-kidl-parser-1409261812-7863aef.jar', - 'kbase-kidl-parser-1409261812-7863aef' - ) - implementation fromURL( - 'https://github.com/kbase/jars/raw/master/lib/jars/kbase/sample/SampleServiceClient-0.1.1.jar', - 'SampleServiceClient-0.1.1' - ) implementation 'ch.qos.logback:logback-classic:1.1.2' - implementation 'com.google.guava:guava:14.0.1' implementation 'com.github.ben-manes.caffeine:caffeine:2.9.3' + implementation "com.github.kbase:auth2_client_java:$VER_AUTH2_CLIENT" + implementation("com.github.kbase:handle_service2:1.0.6") { + exclude group: 'net.java.dev.jna' // breaks shadow jar + } + implementation "com.github.kbase:java_kidl:0.1.0" + implementation("com.github.kbase:sample_service:0.2.6") { + exclude group: 'net.java.dev.jna' // breaks shadow jar + } + implementation "com.github.kbase:shock_java_client:0.2.0" + implementation 'com.google.guava:guava:14.0.1' implementation 'commons-codec:commons-codec:1.8' implementation 'commons-io:commons-io:2.4' - implementation 'com.fasterxml.jackson.core:jackson-annotations:2.9.9' - implementation 'com.fasterxml.jackson.core:jackson-core:2.9.9' - implementation 'com.fasterxml.jackson.core:jackson-databind:2.9.9' + implementation "com.fasterxml.jackson.core:jackson-annotations:$VER_JACKSON" + implementation "com.fasterxml.jackson.core:jackson-databind:$VER_JACKSON" implementation 'info.picocli:picocli:4.6.1' implementation 'org.apache.commons:commons-lang3:3.1' + implementation 'org.apache.httpcomponents:httpclient:4.5.9' implementation 'org.apache.kafka:kafka-clients:2.1.0' implementation 'org.ini4j:ini4j:0.5.2' implementation 'org.mongodb:bson:4.11.1' @@ -342,34 +361,18 @@ dependencies { implementation 'org.mongodb:mongodb-driver-sync:4.11.1' implementation 'org.slf4j:slf4j-api:1.7.30' - // ### Server dependencies, specifically for java_common JsonServerServlet ### + // ### Server dependencies for java_common JsonServerServlet & WorkspaceServer ### - implementation fromURL( - 'https://github.com/kbase/jars/raw/master/lib/jars/kbase/common/kbase-common-0.2.0.jar', - 'kbase-common-0.2.0' - ) - // Pull from jars repo vs a maven repository to avoid the JNA dependency - // TODO DEPS Need to rework the java common logger to not use syslog4j at all since it's abandonware - // and has a ton of CVEs, even in the newer versions. - implementation fromURL( - 'https://github.com/kbase/jars/raw/master/lib/jars/syslog4j/syslog4j-0.9.46.jar', - 'syslog4j-0.9.46' - ) + implementation("com.github.kbase:java_common:$VER_JAVA_COMMON") { + exclude group: 'net.java.dev.jna' // breaks shadow jar + } + // TODO DEPS Need to rework the java common logger to not use syslog4j at all since it's + // abandonware and has a ton of CVEs, even in the newer versions. + implementation "org.syslog4j:syslog4j:0.9.46" implementation 'javax.annotation:javax.annotation-api:1.3.2' implementation 'javax.servlet:servlet-api:2.5' - // joda-time is required for kbase-common and syslog4j - implementation 'joda-time:joda-time:2.2' // this is OOOOOOLD. But that probably means updating java_common implementation 'org.eclipse.jetty.aggregate:jetty-all:7.0.0.v20091005' - - // ### Blobstore / Shock client and dependencies ### - - implementation fromURL( - 'https://github.com/kbase/jars/raw/master/lib/jars/kbase/shock/shock-client-0.1.0.jar', - 'shock-client-0.1.0' - ) - implementation 'org.apache.httpcomponents:httpclient:4.5.9' - implementation 'org.apache.httpcomponents:httpmime:4.5.8' // ### Amazon S3 ### @@ -381,17 +384,13 @@ dependencies { // ### Test ### - testImplementation fromURL( - 'https://github.com/kbase/jars/raw/master/lib/jars/kbase/auth2/kbase-auth2-test-shadow-all-0.7.0.jar', - 'kbase-auth2-test-shadow-all-0.7.0' - ) testImplementation 'com.arangodb:arangodb-java-driver:6.7.2' + testImplementation 'com.github.kbase:java_test_utilities:0.1.0' + testImplementation "com.github.kbase:auth2:$VER_AUTH2_SHADOW" testImplementation 'com.github.zafarkhaja:java-semver:0.9.0' testImplementation 'junit:junit:4.12' testImplementation 'nl.jqno.equalsverifier:equalsverifier:3.1.10' - testImplementation 'org.hamcrest:hamcrest-core:1.3' testImplementation 'org.mockito:mockito-core:3.0.0' - } task showTestClassPath { @@ -399,4 +398,3 @@ task showTestClassPath { configurations.testimpl.each { println it } } } - diff --git a/docsource/buildinitclient.rst b/docsource/buildinitclient.rst index b29322836..83247756e 100644 --- a/docsource/buildinitclient.rst +++ b/docsource/buildinitclient.rst @@ -48,22 +48,25 @@ Build the client:: The client jar is created in ``build/libs/workspace_deluxe-client.jar``. -For simplicity, copy the required jars into a single directory. You will also need the -`Jackson `_ -annotations, core, and databind jars, which can be downloaded from maven or added to your build -tool:: +For simplicity, copy the required jars into a single directory. You will also need the following +jars, which can be downloaded from a maven repository or https://jitpack.io: + +* The `Jackson `_ annotations, core, and databind jars + (maven) +* The javax annotation api jar (maven) +* The `KBase auth client jar `_ +* The `KBase java_common jar `_ + +:: bareubuntu@bu:~/ws$ mkdir tryjavaclient bareubuntu@bu:~/ws$ cd tryjavaclient/ - bareubuntu@bu:~/ws/tryjavaclient$ cp ../workspace_deluxe/build/libs/workspace_deluxe-client.jar . - bareubuntu@bu:~/ws/tryjavaclient$ cp ../workspace_deluxe/build/download/kbase-auth-0.4.4.jar . - bareubuntu@bu:~/ws/tryjavaclient$ cp ../workspace_deluxe/build/download/kbase-common-0.2.0.jar . bareubuntu@bu:~/ws/tryjavaclient$ ls - jackson-annotations-2.9.9.jar kbase-auth-0.4.4.jar - jackson-core-2.9.9.jar kbase-common-0.2.0.jar - jackson-databind-2.9.9.jar workspace_deluxe-client.jar - + auth2_client_java-0.5.0.jar java_common-0.3.0.jar + jackson-annotations-2.9.9.jar javax.annotation-api-1.3.2.jar + jackson-core-2.9.9.jar workspace_deluxe-client.jar + jackson-databind-2.9.9.jar When creating an application using the WSS it's advisable to use a build tool like ``ant``, ``maven``, or ``gradle`` to organize the required jars. @@ -74,26 +77,23 @@ This simple program initializes and calls a method on the WSS client:: .. code-block:: java + import java.net.URI; import java.net.URL; - import us.kbase.auth.AuthConfig; - import us.kbase.workspace.WorkspaceClient; - import us.kbase.auth.ConfigurableAuthService; import us.kbase.auth.AuthToken; + import us.kbase.auth.client.AuthClient; + import us.kbase.workspace.WorkspaceClient; public class TryWorkspaceClient { public static void main(String[] args) throws Exception { - String authUrl = - "https://ci.kbase.us/services/auth/api/legacy/KBase/Sessions/Login/"; - - ConfigurableAuthService authService = new ConfigurableAuthService( - new AuthConfig().withKBaseAuthServerURL(new URL(authUrl)); + final String authUrl = "https://appdev.kbase.us/services/auth/"; + final AuthClient authcli = AuthClient.from(new URI(authUrl)); - String tokenString = YOUR_AUTH_TOKEN_HERE; - AuthToken token = authService.validateToken(tokenString); + final String tokenString = args[0]; + final AuthToken token = authcli.validateToken(tokenString); - WorkspaceClient client = new WorkspaceClient( - new URL("https://ci.kbase.us/services/ws/"), + final WorkspaceClient client = new WorkspaceClient( + new URL("https://appdev.kbase.us/services/ws/"), token); System.out.println(client.ver()); } @@ -102,7 +102,7 @@ This simple program initializes and calls a method on the WSS client:: Compile and run:: bareubuntu@bu:~/ws/tryjavaclient$ javac -cp "./*" TryWorkspaceClient.java - bareubuntu@bu:~/ws/tryjavaclient$ java -cp "./:./*" TryWorkspaceClient + bareubuntu@bu:~/ws/tryjavaclient$ java -cp "./:./*" TryWorkspaceClient $KBASE_TOKEN 0.14.2 For more client initialization and configuration options, see :ref:`apidocs`. diff --git a/docsource/developers.rst b/docsource/developers.rst index cd3b715cc..b94d23884 100644 --- a/docsource/developers.rst +++ b/docsource/developers.rst @@ -22,7 +22,7 @@ Branches: Recompiling the generated code ------------------------------ -To compile, run ``./gradlew compile``. +To compile the Workspace generated code from ``workspace.spec``, run ``./gradlew compile``. Note that: diff --git a/docsource/releasenotes.rst b/docsource/releasenotes.rst index 4f5c34341..9223d4912 100644 --- a/docsource/releasenotes.rst +++ b/docsource/releasenotes.rst @@ -16,6 +16,7 @@ UPDATES: against Mongo 7. * Gradle has replaced Ant as the build tool. As a consequence, all the built artifacts are now located in the build directory, including the ``update_workspace_database_schema`` script. +* A shadow jar is published on jitpack.io for supporting tests in other repos. VERSION: 0.14.2 (Released 11/9/2023) diff --git a/src/main/java/us/kbase/common/service/ServiceChecker.java b/src/main/java/us/kbase/common/service/ServiceChecker.java deleted file mode 100644 index 6239a1dcf..000000000 --- a/src/main/java/us/kbase/common/service/ServiceChecker.java +++ /dev/null @@ -1,121 +0,0 @@ -package us.kbase.common.service; - -import java.io.IOException; -import java.io.InputStream; -import java.net.HttpURLConnection; -import java.net.URL; - -import org.apache.commons.io.IOUtils; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.databind.ObjectMapper; - - -/** Methods for checking on KBase SDK services. - * @author gaprice@lbl.gov - * - */ -public class ServiceChecker { - - public static final String PY_MSG = "Expecting value: line 1 column 1 (char 0)"; - public static final int PY_CODE = -32700; - public static final String JV_PL_MSG = "HTTP GET not allowed."; - public static final int JV_PL_CODE = -32300; - - /** Checks whether the KBase SDK service at the given url is contactable. - * If not, throws an exception. - * @param serviceURL the url of the service. - * @throws ServiceException if the service cannot be contacted or is not a - * KBase SDK service. - */ - public static void checkService(final URL serviceURL) - throws ServiceException { - try { - final HttpURLConnection conn = - (HttpURLConnection) serviceURL.openConnection(); - conn.setRequestMethod("GET"); - conn.setDoOutput(true); - conn.setUseCaches(false); - - final int responseCode = conn.getResponseCode(); - if (responseCode != 500) { - handleNon500(serviceURL, conn, responseCode); - } else { - final RpcResponse r; - try (final InputStream is = conn.getErrorStream()) { - r = new ObjectMapper() - .readValue(is, RpcResponse.class); - } - conn.disconnect(); - //TODO TEST mock up a test service for the next two checks at some point - if (r.error == null) { - throw new ServiceException(String.format( - "The service at %s is not a KBase SDK service: " + - "no error field in RPC response as expected", - serviceURL)); - } - final String msg = r.error.message; - final int code = r.error.code; - if (!(PY_MSG.equals(msg) && PY_CODE == code) && - !(JV_PL_MSG.equals(msg) && JV_PL_CODE == code)) { - throw new ServiceException(String.format( - "The JSONRPC service at %s is not a KBase SDK " + - "service. Code: %s, message: %s", - serviceURL, code, msg)); - } - // service is ok, nothing to do - } - } catch (IOException e) { - throw new ServiceException(String.format( - "Could not contact the service at URL %s: %s", - serviceURL, e.getMessage()), e); - } - } - - private static void handleNon500( - final URL serviceURL, - final HttpURLConnection conn, - final int responseCode) - throws IOException, ServiceException { - final String error; - try (final InputStream es = conn.getInputStream()) { - error = IOUtils.toString(es, "UTF-8"); - } - conn.disconnect(); - throw new ServiceException(String.format( - "URL %s does not point to a KBase SDK generated " + - "service. Code: %s, message: %s, content: %s", - serviceURL, responseCode, - conn.getResponseMessage(), - error.length() >= 1000 ? error.substring(0, 1000) : - error)); - } - - @JsonIgnoreProperties(ignoreUnknown = true) - private static class RpcError { - - public String message; - public int code; - } - - @JsonIgnoreProperties(ignoreUnknown = true) - private static class RpcResponse { - public RpcError error; - } - - @SuppressWarnings("serial") - public static class ServiceException extends Exception { - - private ServiceException(String message, Throwable cause) { - super(message, cause); - } - - private ServiceException(String message) { - super(message); - } - } - - public static void main(String[] args) throws Exception { - checkService(new URL("http://the-internet.herokuapp.com/status_codes/500")); - } -} diff --git a/src/main/java/us/kbase/workspace/WorkspaceServer.java b/src/main/java/us/kbase/workspace/WorkspaceServer.java index eb4ee4a4f..0b25a11a2 100644 --- a/src/main/java/us/kbase/workspace/WorkspaceServer.java +++ b/src/main/java/us/kbase/workspace/WorkspaceServer.java @@ -16,9 +16,9 @@ import us.kbase.common.service.UObject; //BEGIN_HEADER -import us.kbase.auth.AuthConfig; import us.kbase.auth.AuthException; -import us.kbase.auth.ConfigurableAuthService; +import us.kbase.auth.client.AuthClient; + import static us.kbase.common.utils.ServiceUtils.checkAddlArgs; import static us.kbase.workspace.kbase.ArgUtils.getGlobalWSPerm; import static us.kbase.workspace.kbase.ArgUtils.wsInfoToTuple; @@ -35,9 +35,7 @@ import static us.kbase.workspace.version.WorkspaceVersion.VERSION; import java.io.IOException; -import java.net.MalformedURLException; import java.net.URISyntaxException; -import java.net.URL; import java.nio.file.Paths; import java.util.Arrays; import java.util.HashMap; @@ -143,7 +141,7 @@ public class WorkspaceServer extends JsonServerServlet { private static ThreadLocal _tempSyslog = new ThreadLocal<>(); private static ThreadLocal _tempConfig = new ThreadLocal<>(); private static ThreadLocal _tempStartupFailed = new ThreadLocal<>(); - private static ThreadLocal _tempAuth = new ThreadLocal<>(); + private static ThreadLocal _tempAuth = new ThreadLocal<>(); public WorkspaceServer() { // This deliberately replaces the constructor compiled by kb-sdk. If you recompile the @@ -197,29 +195,17 @@ private static AuthenticationHandler getAuth(final boolean silenceLogs) { if (cfg == null) { return t -> {throw new AuthException("failed to set up auth");}; } - final AuthConfig c = new AuthConfig(); - if (cfg.getAuth2URL().getProtocol().equals("http")) { - c.withAllowInsecureURLs(true); - sysLogger.logInfo("Warning - the Auth Service MKII url uses insecure http. " + - "https is recommended."); - } try { - final URL globusURL = cfg.getAuth2URL().toURI().resolve("api/legacy/globus").toURL(); - final URL kbaseURL = cfg.getAuth2URL().toURI() - .resolve("api/legacy/KBase/Sessions/Login").toURL(); - c.withGlobusAuthURL(globusURL).withKBaseAuthServerURL(kbaseURL); - } catch (URISyntaxException | MalformedURLException e) { + final AuthClient cli = AuthClient.from(cfg.getAuth2URL().toURI()); + _tempAuth.set(cli); + return token -> cli.validateToken(token); + } catch (URISyntaxException e) { sysLogger.logErr("Invalid Auth Service url: " + cfg.getAuth2URL()); _tempStartupFailed.set(true); return t -> {throw new AuthException("failed to set up auth");}; - } - try { - final ConfigurableAuthService cas = new ConfigurableAuthService(c); - _tempAuth.set(cas); - return token -> cas.validateToken(token); - } catch (IOException e) { + } catch (AuthException | IOException e) { sysLogger.logErr("Couldn't connect to authorization service at " + - c.getAuthServerURL() + " : " + e.getLocalizedMessage()); + cfg.getAuth2URL() + " : " + e.getLocalizedMessage()); sysLogger.logErr(e); _tempStartupFailed.set(true); return t -> {throw new AuthException("failed to set up auth");}; @@ -308,7 +294,7 @@ private static JsonServerSyslog getUserLogger(final boolean silenceLogs) { private WorkspaceInitResults initWorkspace( final KBaseWorkspaceConfig cfg, - final ConfigurableAuthService auth, + final AuthClient auth, final JsonServerSyslog logger) { setUpLogger(); setMaxRPCPackageSize(MAX_RPC_PACKAGE_SIZE); diff --git a/src/main/java/us/kbase/workspace/kbase/InitWorkspaceServer.java b/src/main/java/us/kbase/workspace/kbase/InitWorkspaceServer.java index 6d4f66cdb..a1121bfd4 100644 --- a/src/main/java/us/kbase/workspace/kbase/InitWorkspaceServer.java +++ b/src/main/java/us/kbase/workspace/kbase/InitWorkspaceServer.java @@ -33,7 +33,7 @@ import us.kbase.abstracthandle.AbstractHandleClient; import us.kbase.auth.AuthException; import us.kbase.auth.AuthToken; -import us.kbase.auth.ConfigurableAuthService; +import us.kbase.auth.client.AuthClient; import us.kbase.common.service.JsonClientException; import us.kbase.common.service.UnauthorizedException; import us.kbase.sampleservice.SampleServiceClient; @@ -142,7 +142,7 @@ public static void setMaximumUniqueIdCountForTests(final int count) { public static WorkspaceInitResults initWorkspaceServer( final KBaseWorkspaceConfig cfg, - final ConfigurableAuthService auth, + final AuthClient auth, final InitReporter rep) { final TempFilesManager tfm = initTempFilesManager(cfg.getTempDir(), rep); @@ -220,7 +220,7 @@ private static AdministratorHandler getAdminHandler( private static WorkspaceInitResults buildWorkspace( final KBaseWorkspaceConfig cfg, - final ConfigurableAuthService auth, + final AuthClient auth, final AbstractHandleClient hsc, final TempFilesManager tfm, final InitReporter rep) // DO NOT use the rep to report failures. Throw instead. @@ -349,7 +349,7 @@ public WorkspaceClient getClient(final URL workspaceURL) private static BytestreamIdHandlerFactory getShockIdHandlerFactory( final KBaseWorkspaceConfig cfg, - final ConfigurableAuthService auth) + final AuthClient auth) throws WorkspaceInitException { if (cfg.getBytestreamURL() == null) { return new BytestreamIdHandlerFactory(null, null); @@ -376,7 +376,7 @@ public BasicShockClient clone(final BasicShockClient source) private static SampleIdHandlerFactory getSampleIdHandlerFactory( final KBaseWorkspaceConfig cfg, - final ConfigurableAuthService auth, + final AuthClient auth, final InitReporter rep) // DO NOT use the rep to report failures. Throw instead. throws WorkspaceInitException { if (cfg.getSampleServiceURL() == null) { @@ -479,7 +479,7 @@ private static AuthToken getKBaseToken( final String user, final String token, final String source, - final ConfigurableAuthService auth) + final AuthClient auth) throws WorkspaceInitException { final AuthToken t = getToken(token, auth, source); if (!t.getUserName().equals(user)) { @@ -493,7 +493,7 @@ private static AuthToken getKBaseToken( private static AuthToken getToken( final String token, - final ConfigurableAuthService auth, + final AuthClient auth, final String source) throws WorkspaceInitException { if (token == null) { @@ -568,7 +568,7 @@ private static TempFilesManager initTempFilesManager( private static AuthToken getHandleToken( final KBaseWorkspaceConfig cfg, final InitReporter rep, - final ConfigurableAuthService auth) { + final AuthClient auth) { try { return auth.validateToken(cfg.getHandleServiceToken()); } catch (AuthException e) { diff --git a/src/main/java/us/kbase/workspace/kbase/WorkspaceServerMethods.java b/src/main/java/us/kbase/workspace/kbase/WorkspaceServerMethods.java index 52d7d31a3..aaa008f63 100644 --- a/src/main/java/us/kbase/workspace/kbase/WorkspaceServerMethods.java +++ b/src/main/java/us/kbase/workspace/kbase/WorkspaceServerMethods.java @@ -36,7 +36,7 @@ import us.kbase.auth.AuthException; import us.kbase.auth.AuthToken; -import us.kbase.auth.ConfigurableAuthService; +import us.kbase.auth.client.AuthClient; import us.kbase.common.service.Tuple11; import us.kbase.common.service.Tuple9; import us.kbase.typedobj.core.TypeDefId; @@ -102,13 +102,13 @@ public class WorkspaceServerMethods { // TODO JAVADOC private final Workspace ws; - private final ConfigurableAuthService auth; + private final AuthClient auth; private final IdReferenceHandlerSetFactoryBuilder idFacBuilder; public WorkspaceServerMethods( final Workspace ws, final IdReferenceHandlerSetFactoryBuilder idFacBuilder, - final ConfigurableAuthService auth) { + final AuthClient auth) { this.ws = ws; this.idFacBuilder = idFacBuilder; this.auth = auth; @@ -134,7 +134,7 @@ public List getDependencyStatus() { return ret; } - public ConfigurableAuthService getAuth() { + public AuthClient getAuth() { return auth; } @@ -266,7 +266,7 @@ private List validateUsers(final List users, final AuthTo final List wsusers = convertUsers(users); final Map userok; try { - userok = auth.isValidUserName(users, token); + userok = auth.isValidUserName(users, token.getToken()); } catch (UnknownHostException uhe) { //message from UHE is only the host name throw new AuthException( diff --git a/src/test/java/us/kbase/test/common/TestCommon.java b/src/test/java/us/kbase/test/common/TestCommon.java index ccde18bb4..5cf4df005 100644 --- a/src/test/java/us/kbase/test/common/TestCommon.java +++ b/src/test/java/us/kbase/test/common/TestCommon.java @@ -37,7 +37,7 @@ import ch.qos.logback.classic.spi.ILoggingEvent; import ch.qos.logback.classic.spi.IThrowableProxy; import ch.qos.logback.core.AppenderBase; -import us.kbase.common.test.TestException; +import us.kbase.testutils.TestException; import us.kbase.typedobj.core.TempFilesManager; public class TestCommon { diff --git a/src/test/java/us/kbase/test/common/test/service/ServiceCheckerTest.java b/src/test/java/us/kbase/test/common/test/service/ServiceCheckerTest.java deleted file mode 100644 index 9a1a63d55..000000000 --- a/src/test/java/us/kbase/test/common/test/service/ServiceCheckerTest.java +++ /dev/null @@ -1,117 +0,0 @@ -package us.kbase.test.common.test.service; - -import static org.hamcrest.CoreMatchers.is; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.fail; - -import java.net.MalformedURLException; -import java.net.URL; -import java.util.HashMap; -import java.util.Map; - -import org.junit.BeforeClass; -import org.junit.Test; - -import us.kbase.common.service.ServiceChecker; -import us.kbase.common.service.ServiceChecker.ServiceException; - -public class ServiceCheckerTest { - - private static final String KBASE_ENV = "https://ci.kbase.us"; // "https://kbase.us/" - - public static class TestSpec { - public final String err; - public final URL url; - private TestSpec(String err, String url) throws MalformedURLException { - super(); - this.err = err; - this.url = new URL(url); - } - } - - public static Map TESTS = new HashMap<>(); - - @BeforeClass - public static void beforeClass() throws Exception { - TESTS.put("Perl", new TestSpec( - null, KBASE_ENV + "/services/handle_service")); - TESTS.put("Python", new TestSpec( - null, KBASE_ENV + "/services/catalog")); - TESTS.put("Java", new TestSpec( - null, KBASE_ENV + "/services/ws")); - TESTS.put("HTTP", new TestSpec( - null, "http://dev03.berkeley.kbase.us:7058")); - TESTS.put("500", new TestSpec( - "Could not contact the service at URL http://the-internet." + - "herokuapp.com/status_codes/500: Unexpected character ('<' " + - "(code 60)):", - "http://the-internet.herokuapp.com/status_codes/500")); - TESTS.put("200", new TestSpec( - "URL http://google.com does not point to a KBase SDK " + - "generated service. Code: 200, message: OK, content: " + - " adminRoles) - throws Exception { - checkExe(blobstoreExe, "blobstore"); - this.tempDir = makeTempDirs(rootTempDir, "BlobstoreController-", Collections.emptyList()); - this.port = findFreePort(); - - final File bsIniFile = createBlobstoreDeployCfg( - mongohost, - mongoDBname, - s3Host, - s3Bucket, - s3AccessKey, - s3AccessSecret, - s3Region, - kbaseAuthURL, - adminRoles - ); - - this.logfile = tempDir.resolve("blobstore.log"); - - ProcessBuilder servpb = new ProcessBuilder(blobstoreExe, "--conf", bsIniFile.toString()) - .redirectErrorStream(true) - .redirectOutput(logfile.toFile()); - - this.blobstore = servpb.start(); - Thread.sleep(1000); //wait for server to start - } - - private File createBlobstoreDeployCfg( - final String mongohost, - final String mongoDBname, - final String s3Host, - final String s3Bucket, - final String s3AccessKey, - final String s3AccessSecret, - final String s3Region, - final URL kbaseAuthURL, - final List adminRoles) - throws IOException { - final File iniFile = tempDir.resolve("blobstore.cfg").toFile(); - final Ini ini = new Ini(); - final Section ss = ini.add(BLOBSTORE_CFG_SECTION); - ss.add("host", "localhost:" + port); - - ss.add("kbase-auth-url", kbaseAuthURL.toString()); - ss.add("kbase-auth-admin-roles", String.join(", ", adminRoles)); - - ss.add("mongodb-host", mongohost); - ss.add("mongodb-database", mongoDBname); - - ss.add("s3-host", s3Host); - ss.add("s3-bucket", s3Bucket); - ss.add("s3-access-key", s3AccessKey); - ss.add("s3-access-secret", s3AccessSecret); - ss.add("s3-region", s3Region); - ss.add("s3-disable-ssl", "true"); - - ini.store(iniFile); - return iniFile; - } - - /** Get the blobstore port. - * @return the port. - */ - public int getPort() { - return port; - } - - /** Get the directory in which the blobstore is storing temporary files. - * @return the temporary directory. - */ - public Path getTempDir() { - return tempDir; - } - - /** Shut down the blob store. - * @param deleteTempFiles true to delete any temporary files. - * @throws IOException if an IO error occurs. - */ - public void destroy(final boolean deleteTempFiles) throws IOException { - destroy(deleteTempFiles, false); - } - - /** Shut down the blob store. - * @param deleteTempFiles true to delete any temporary files. - * @param dumpLogToStdOut print any blobstore logs to standard out. - * @throws IOException if an IO error occurs. - */ - public void destroy(final boolean deleteTempFiles, final boolean dumpLogToStdOut) - throws IOException { - if (blobstore != null) { - blobstore.destroy(); - } - if (dumpLogToStdOut) { - try (final BufferedReader is = Files.newBufferedReader(logfile)) { - is.lines().forEach(l -> System.out.println(l)); - } - } - if (tempDir != null && deleteTempFiles) { - FileUtils.deleteDirectory(tempDir.toFile()); - } - } -} \ No newline at end of file diff --git a/src/test/java/us/kbase/test/workspace/controllers/handle/HandleServiceController.java b/src/test/java/us/kbase/test/workspace/controllers/handle/HandleServiceController.java index 963d94163..87fbd1ead 100644 --- a/src/test/java/us/kbase/test/workspace/controllers/handle/HandleServiceController.java +++ b/src/test/java/us/kbase/test/workspace/controllers/handle/HandleServiceController.java @@ -1,7 +1,7 @@ package us.kbase.test.workspace.controllers.handle; -import static us.kbase.common.test.controllers.ControllerCommon.findFreePort; -import static us.kbase.common.test.controllers.ControllerCommon.makeTempDirs; +import static us.kbase.testutils.controllers.ControllerCommon.findFreePort; +import static us.kbase.testutils.controllers.ControllerCommon.makeTempDirs; import java.io.BufferedReader; import java.io.File; @@ -16,7 +16,7 @@ import org.ini4j.Profile.Section; import us.kbase.auth.AuthToken; -import us.kbase.common.test.controllers.mongo.MongoController; +import us.kbase.testutils.controllers.mongo.MongoController; /** Q&D Utility to run the Handle Service for the purposes of testing from Java. diff --git a/src/test/java/us/kbase/test/workspace/controllers/minio/MinioController.java b/src/test/java/us/kbase/test/workspace/controllers/minio/MinioController.java deleted file mode 100644 index f8bfe04be..000000000 --- a/src/test/java/us/kbase/test/workspace/controllers/minio/MinioController.java +++ /dev/null @@ -1,98 +0,0 @@ -package us.kbase.test.workspace.controllers.minio; - -import static us.kbase.common.test.controllers.ControllerCommon.checkExe; -import static us.kbase.common.test.controllers.ControllerCommon.findFreePort; -import static us.kbase.common.test.controllers.ControllerCommon.makeTempDirs; - -import java.io.BufferedReader; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Scanner; - -import org.apache.commons.io.FileUtils; - -/** Q&D Utility to run a Minio server for the purposes of testing from - * Java. Always run with the --compat flag. - * @author gaprice@lbl.gov - * - */ -public class MinioController { - - private final Path tempDir; - - private final Process minio; - private final int port; - private final Path logfile; - - public MinioController( - final String minioExe, - final String s3AccessKey, - final String s3AccessSecret, - final Path rootTempDir) - throws Exception { - tempDir = makeTempDirs(rootTempDir, "MinioController-", Arrays.asList("data")); - port = findFreePort(); - - checkExe(minioExe, "minio server"); - - logfile = tempDir.resolve("minio_server.log"); - ProcessBuilder servpb = new ProcessBuilder( - minioExe, - "server", - "--compat", - "--address", "localhost:" + port, - tempDir.resolve("data").toString()) - .redirectErrorStream(true) - .redirectOutput(logfile.toFile()); - - servpb.environment().put("MINIO_ACCESS_KEY", s3AccessKey); - servpb.environment().put("MINIO_SECRET_KEY", s3AccessSecret); - minio = servpb.start(); - Thread.sleep(1000); //wait for server to start - } - - public int getServerPort() { - return port; - } - - public Path getTempDir() { - return tempDir; - } - - public void destroy(boolean deleteTempFiles) throws IOException { - destroy(deleteTempFiles, false); - } - - public void destroy(boolean deleteTempFiles, boolean dumpLogToStdOut) throws IOException { - if (minio != null) { - minio.destroy(); - } - if (dumpLogToStdOut) { - try (final BufferedReader is = Files.newBufferedReader(logfile)) { - is.lines().forEach(l -> System.out.println(l)); - } - } - if (tempDir != null && deleteTempFiles) { - FileUtils.deleteDirectory(tempDir.toFile()); - } - } - - public static void main(String[] args) throws Exception { - MinioController ac = new MinioController( - "Minio", - "foobar", - "Wheewhoowhump", - Paths.get("minio_temp_dir")); - System.out.println(ac.getServerPort()); - Scanner reader = new Scanner(System.in); - System.out.println("any char to shut down"); - //get user input for a - reader.next(); - ac.destroy(false); - reader.close(); - } - -} diff --git a/src/test/java/us/kbase/test/workspace/controllers/sample/SampleServiceController.java b/src/test/java/us/kbase/test/workspace/controllers/sample/SampleServiceController.java index ea568e189..f49928386 100644 --- a/src/test/java/us/kbase/test/workspace/controllers/sample/SampleServiceController.java +++ b/src/test/java/us/kbase/test/workspace/controllers/sample/SampleServiceController.java @@ -1,7 +1,7 @@ package us.kbase.test.workspace.controllers.sample; -import static us.kbase.common.test.controllers.ControllerCommon.findFreePort; -import static us.kbase.common.test.controllers.ControllerCommon.makeTempDirs; +import static us.kbase.testutils.controllers.ControllerCommon.findFreePort; +import static us.kbase.testutils.controllers.ControllerCommon.makeTempDirs; import static us.kbase.workspace.database.Util.checkString; import java.io.BufferedReader; diff --git a/src/test/java/us/kbase/test/workspace/controllers/workspace/WorkspaceController.java b/src/test/java/us/kbase/test/workspace/controllers/workspace/WorkspaceController.java index aae6691b6..6e9423a04 100644 --- a/src/test/java/us/kbase/test/workspace/controllers/workspace/WorkspaceController.java +++ b/src/test/java/us/kbase/test/workspace/controllers/workspace/WorkspaceController.java @@ -1,7 +1,7 @@ package us.kbase.test.workspace.controllers.workspace; -import static us.kbase.common.test.controllers.ControllerCommon.findFreePort; -import static us.kbase.common.test.controllers.ControllerCommon.makeTempDirs; +import static us.kbase.testutils.controllers.ControllerCommon.findFreePort; +import static us.kbase.testutils.controllers.ControllerCommon.makeTempDirs; import java.io.File; import java.io.IOException; diff --git a/src/test/java/us/kbase/test/workspace/database/mongo/GridFSBlobStoreTest.java b/src/test/java/us/kbase/test/workspace/database/mongo/GridFSBlobStoreTest.java index 104f3a86c..c7f63c3fa 100644 --- a/src/test/java/us/kbase/test/workspace/database/mongo/GridFSBlobStoreTest.java +++ b/src/test/java/us/kbase/test/workspace/database/mongo/GridFSBlobStoreTest.java @@ -5,7 +5,7 @@ import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static us.kbase.test.common.TestCommon.assertExceptionCorrect; +import static us.kbase.testutils.TestCommon.assertExceptionCorrect; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -33,7 +33,7 @@ import com.mongodb.client.gridfs.model.GridFSUploadOptions; import us.kbase.test.common.TestCommon; -import us.kbase.common.test.controllers.mongo.MongoController; +import us.kbase.testutils.controllers.mongo.MongoController; import us.kbase.typedobj.core.MD5; import us.kbase.typedobj.core.Restreamable; import us.kbase.workspace.database.ByteArrayFileCacheManager; diff --git a/src/test/java/us/kbase/test/workspace/database/mongo/MongoInternalsTest.java b/src/test/java/us/kbase/test/workspace/database/mongo/MongoInternalsTest.java index f087c740e..3fcf134b1 100644 --- a/src/test/java/us/kbase/test/workspace/database/mongo/MongoInternalsTest.java +++ b/src/test/java/us/kbase/test/workspace/database/mongo/MongoInternalsTest.java @@ -38,7 +38,7 @@ import org.junit.Test; import us.kbase.common.service.UObject; -import us.kbase.common.test.controllers.mongo.MongoController; +import us.kbase.testutils.controllers.mongo.MongoController; import us.kbase.common.utils.sortjson.UTF8JsonSorterFactory; import us.kbase.test.common.TestCommon; import us.kbase.test.typedobj.DummyValidatedTypedObject; diff --git a/src/test/java/us/kbase/test/workspace/database/mongo/MongoStartUpTest.java b/src/test/java/us/kbase/test/workspace/database/mongo/MongoStartUpTest.java index a06dfef2f..ee4a2acfd 100644 --- a/src/test/java/us/kbase/test/workspace/database/mongo/MongoStartUpTest.java +++ b/src/test/java/us/kbase/test/workspace/database/mongo/MongoStartUpTest.java @@ -30,7 +30,7 @@ import com.mongodb.client.MongoDatabase; import us.kbase.test.common.TestCommon; -import us.kbase.common.test.controllers.mongo.MongoController; +import us.kbase.testutils.controllers.mongo.MongoController; import us.kbase.typedobj.core.TypeDefId; import us.kbase.typedobj.core.TypeDefName; import us.kbase.workspace.database.WorkspaceInformation; diff --git a/src/test/java/us/kbase/test/workspace/database/mongo/MongoWorkspaceDBTest.java b/src/test/java/us/kbase/test/workspace/database/mongo/MongoWorkspaceDBTest.java index a3c2bcc99..b83657177 100644 --- a/src/test/java/us/kbase/test/workspace/database/mongo/MongoWorkspaceDBTest.java +++ b/src/test/java/us/kbase/test/workspace/database/mongo/MongoWorkspaceDBTest.java @@ -44,7 +44,7 @@ import com.mongodb.client.MongoDatabase; import us.kbase.common.service.UObject; -import us.kbase.common.test.controllers.mongo.MongoController; +import us.kbase.testutils.controllers.mongo.MongoController; import us.kbase.common.utils.sortjson.UTF8JsonSorterFactory; import us.kbase.test.common.TestCommon; import us.kbase.test.typedobj.DummyValidatedTypedObject; diff --git a/src/test/java/us/kbase/test/workspace/database/mongo/S3BlobStoreIntegrationTest.java b/src/test/java/us/kbase/test/workspace/database/mongo/S3BlobStoreIntegrationTest.java index 2def675bd..fd2b108fb 100644 --- a/src/test/java/us/kbase/test/workspace/database/mongo/S3BlobStoreIntegrationTest.java +++ b/src/test/java/us/kbase/test/workspace/database/mongo/S3BlobStoreIntegrationTest.java @@ -24,9 +24,9 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.model.DeleteBucketRequest; import software.amazon.awssdk.services.s3.model.PutObjectRequest; -import us.kbase.common.test.controllers.mongo.MongoController; +import us.kbase.testutils.controllers.minio.MinioController; +import us.kbase.testutils.controllers.mongo.MongoController; import us.kbase.test.common.TestCommon; -import us.kbase.test.workspace.controllers.minio.MinioController; import us.kbase.typedobj.core.MD5; import us.kbase.typedobj.core.Restreamable; import us.kbase.workspace.database.ByteArrayFileCacheManager; diff --git a/src/test/java/us/kbase/test/workspace/database/mongo/SchemaUpdaterTest.java b/src/test/java/us/kbase/test/workspace/database/mongo/SchemaUpdaterTest.java index a7351de66..f48a531ef 100644 --- a/src/test/java/us/kbase/test/workspace/database/mongo/SchemaUpdaterTest.java +++ b/src/test/java/us/kbase/test/workspace/database/mongo/SchemaUpdaterTest.java @@ -29,7 +29,7 @@ import com.mongodb.client.MongoDatabase; import us.kbase.test.common.TestCommon; -import us.kbase.common.test.controllers.mongo.MongoController; +import us.kbase.testutils.controllers.mongo.MongoController; import us.kbase.workspace.database.ResolvedWorkspaceID; import us.kbase.workspace.database.WorkspaceUser; import us.kbase.workspace.database.WorkspaceUserMetadata; diff --git a/src/test/java/us/kbase/test/workspace/kbase/HandleAndBytestreamIntegrationTest.java b/src/test/java/us/kbase/test/workspace/kbase/HandleAndBytestreamIntegrationTest.java index 95a989639..c91d221cb 100644 --- a/src/test/java/us/kbase/test/workspace/kbase/HandleAndBytestreamIntegrationTest.java +++ b/src/test/java/us/kbase/test/workspace/kbase/HandleAndBytestreamIntegrationTest.java @@ -39,11 +39,12 @@ import us.kbase.abstracthandle.AbstractHandleClient; import us.kbase.abstracthandle.Handle; import us.kbase.auth.AuthToken; -import us.kbase.common.mongo.exceptions.InvalidHostException; import us.kbase.common.service.ServerException; import us.kbase.common.service.UObject; -import us.kbase.common.test.TestException; -import us.kbase.common.test.controllers.mongo.MongoController; +import us.kbase.testutils.TestException; +import us.kbase.testutils.controllers.blobstore.BlobstoreController; +import us.kbase.testutils.controllers.minio.MinioController; +import us.kbase.testutils.controllers.mongo.MongoController; import us.kbase.shock.client.BasicShockClient; import us.kbase.shock.client.ShockACLType; import us.kbase.shock.client.ShockFileInformation; @@ -53,9 +54,7 @@ import us.kbase.test.auth2.authcontroller.AuthController; import us.kbase.test.common.TestCommon; import us.kbase.test.workspace.WorkspaceServerThread; -import us.kbase.test.workspace.controllers.blobstore.BlobstoreController; import us.kbase.test.workspace.controllers.handle.HandleServiceController; -import us.kbase.test.workspace.controllers.minio.MinioController; import us.kbase.typedobj.idref.IdReference; import us.kbase.typedobj.idref.IdReferenceHandlerSet; import us.kbase.typedobj.idref.IdReferenceHandlerSet.TooManyIdsException; @@ -295,7 +294,7 @@ private static WorkspaceServer startupWorkspaceServer( final String miniohost, final String minioUser, final String minioKey) - throws InvalidHostException, UnknownHostException, IOException, + throws UnknownHostException, IOException, NoSuchFieldException, IllegalAccessException, Exception, InterruptedException { @@ -804,12 +803,10 @@ public void saveAndGetWithBytestreamIDs() throws Exception { is(set(n1.getId().getId(), n2.getId().getId()))); // check nodes have the same contents - // TODO BLOBSTORE update tests when https://github.com/kbase/shock_java_client/issues/26 - // is fixed checkNode(WS_OWNED_BLOB, n1.getId(), ImmutableMap.of("foo", "bar"), - "contents", "fname", null); // "text"); + "contents", "fname", "text"); checkNode(WS_OWNED_BLOB, n2.getId(), ImmutableMap.of("foo", "bar2"), - "contents2", "fname2", null); //"text2"); + "contents2", "fname2", "text2"); checkPublicRead(n1, false); checkPublicRead(n2, false); @@ -835,7 +832,7 @@ private void checkNode( final ShockNode sn = cli.getNode(id); final ShockFileInformation fi = sn.getFileInformation(); assertThat("incorrect filename", fi.getName(), is(filename)); - assertThat("incorrect format", fi.getFormat(), is(format)); + assertThat("incorrect format", sn.getFormat(), is(format)); assertThat("incorrect file", IOUtils.toString(sn.getFile()), is(file)); } diff --git a/src/test/java/us/kbase/test/workspace/kbase/JSONRPCLayerTest.java b/src/test/java/us/kbase/test/workspace/kbase/JSONRPCLayerTest.java index bc5f12422..6fff90bc5 100644 --- a/src/test/java/us/kbase/test/workspace/kbase/JSONRPCLayerTest.java +++ b/src/test/java/us/kbase/test/workspace/kbase/JSONRPCLayerTest.java @@ -34,7 +34,6 @@ import org.apache.commons.codec.digest.DigestUtils; import org.junit.Test; -import us.kbase.auth.AuthUser; import us.kbase.common.service.JsonTokenStream; import us.kbase.common.service.ServerException; import us.kbase.common.service.Tuple11; @@ -1444,7 +1443,7 @@ public void deprecatedMethods() throws Exception { .withId(wsid), "depsave", USER1, wsinfo.getE4(), 0, "a", "n", wsid); checkDepWSMeta(new us.kbase.workspace.GetWorkspacemetaParams() - .withWorkspace("depsave").withAuth(AUTH_USER2.getTokenString()), + .withWorkspace("depsave").withAuth(TOKEN2), "depsave", USER1, wsinfo.getE4(), 0, "w", "n", wsid); Tuple7 wsmeta = @@ -1486,7 +1485,7 @@ public void deprecatedMethods() throws Exception { Tuple12, Long> obj3 = CLIENT1.saveObject(new us.kbase.workspace.SaveObjectParams().withId("obj3") .withMetadata(meta2).withType(SAFE_TYPE).withWorkspace("depsave") - .withData(new UObject(data)).withAuth(AUTH_USER2.getTokenString())); + .withData(new UObject(data)).withAuth(TOKEN2)); checkDeprecatedSaveInfo(obj1, 1, "obj1", SAFE_TYPE, 1, USER1, wsid, "depsave", "36c4f68f2c98971b9736839232eb08f4", meta); checkDeprecatedSaveInfo(obj2, 2, "obj2", anotherType, 1, USER1, wsid, "depsave", "3c59f762140806c36ab48a152f28e840", meta2); @@ -1495,33 +1494,27 @@ public void deprecatedMethods() throws Exception { checkSavedObjectDep(new ObjectIdentity().withWorkspace("depsave").withName("obj1"), new ObjectIdentity().withWsid(wsid).withObjid(1L), 1, "obj1", SAFE_TYPE, 1, USER1, wsid, "depsave", "36c4f68f2c98971b9736839232eb08f4", - 23, meta, data, AUTH_USER2); + 23, meta, data, TOKEN2); checkSavedObjectDep(new ObjectIdentity().withWorkspace("depsave").withName("obj2"), new ObjectIdentity().withWsid(wsid).withObjid(2L), 2, "obj2", anotherType, 1, USER1, wsid, "depsave", "3c59f762140806c36ab48a152f28e840", - 24, meta2, data2, AUTH_USER2); + 24, meta2, data2, TOKEN2); checkSavedObjectDep(new ObjectIdentity().withWorkspace("depsave").withName("obj3"), new ObjectIdentity().withWsid(wsid).withObjid(3L), 3, "obj3", SAFE_TYPE, 1, USER2, wsid, "depsave", "36c4f68f2c98971b9736839232eb08f4", - 23, meta2, data, AUTH_USER2); + 23, meta2, data, TOKEN2); checkListObjectsDep("depsave", null, null, null, Arrays.asList(obj1, obj2, obj3)); checkListObjectsDep("depsave", anotherType, null, null, Arrays.asList(obj2)); CLIENT1.deleteObjects(Arrays.asList(new ObjectIdentity().withName("obj2").withWorkspace("depsave"))); checkListObjectsDep("depsave", null, 0L, null, Arrays.asList(obj1, obj3)); checkListObjectsDep("depsave", null, 1L, null, Arrays.asList(obj1, obj2, obj3)); - checkListObjectsDep("depsave", null, null, AUTH_USER2.getTokenString(), Arrays.asList(obj1, obj3)); + checkListObjectsDep("depsave", null, null, TOKEN2, Arrays.asList(obj1, obj3)); - String invalidToken = AUTH_USER2.getTokenString() + "a"; + String invalidToken = TOKEN2 + "a"; String badFormatToken = "borkborkbork"; - // old auth service -// String invalidTokenExp = -// "Login failed! Server responded with code 401 UNAUTHORIZED"; -// String badFormatTokenExp = "Login failed! Invalid token"; - // new auth service String invalidTokenExp = - "Login failed! Server responded with code 401 Unauthorized"; - String badFormatTokenExp = invalidTokenExp; + "Auth service returned an error: 10020 Invalid token"; failDepGetWSmeta(new us.kbase.workspace.GetWorkspacemetaParams() @@ -1529,12 +1522,12 @@ public void deprecatedMethods() throws Exception { invalidTokenExp); failDepGetWSmeta(new us.kbase.workspace.GetWorkspacemetaParams() .withWorkspace("depsave").withAuth(badFormatToken), - badFormatTokenExp); + invalidTokenExp); failDepListWs(new us.kbase.workspace.ListWorkspacesParams() .withAuth(invalidToken), invalidTokenExp); failDepListWs(new us.kbase.workspace.ListWorkspacesParams() - .withAuth(badFormatToken), badFormatTokenExp); + .withAuth(badFormatToken), invalidTokenExp); failDepSaveObject(new us.kbase.workspace.SaveObjectParams().withId("obj3") .withMetadata(meta2).withType(SAFE_TYPE).withWorkspace("depsave") @@ -1543,21 +1536,21 @@ public void deprecatedMethods() throws Exception { failDepSaveObject(new us.kbase.workspace.SaveObjectParams().withId("obj3") .withMetadata(meta2).withType(SAFE_TYPE).withWorkspace("depsave") .withData(new UObject(data)).withAuth(badFormatToken), - badFormatTokenExp); + invalidTokenExp); failDepGetObject(new us.kbase.workspace.GetObjectParams() .withWorkspace("depsave").withId("obj3").withAuth(invalidToken), invalidTokenExp); failDepGetObject(new us.kbase.workspace.GetObjectParams() .withWorkspace("depsave").withId("obj3").withAuth(badFormatToken), - badFormatTokenExp); + invalidTokenExp); failDepGetObjectmeta(new us.kbase.workspace.GetObjectmetaParams() .withWorkspace("depsave").withId("obj3").withAuth(invalidToken), invalidTokenExp); failDepGetObjectmeta(new us.kbase.workspace.GetObjectmetaParams() .withWorkspace("depsave").withId("obj3").withAuth(badFormatToken), - badFormatTokenExp); + invalidTokenExp); failDepListObjects(new us.kbase.workspace.ListWorkspaceObjectsParams() .withWorkspace("depsave").withType("thisisabadtype"), @@ -1567,7 +1560,7 @@ public void deprecatedMethods() throws Exception { invalidTokenExp); failDepListObjects(new us.kbase.workspace.ListWorkspaceObjectsParams() .withWorkspace("depsave").withAuth(badFormatToken), - badFormatTokenExp); + invalidTokenExp); } @SuppressWarnings("deprecation") @@ -1750,10 +1743,18 @@ private void failDepGetObject(us.kbase.workspace.GetObjectParams gop, String exp @SuppressWarnings("deprecation") private void checkSavedObjectDep(ObjectIdentity objnames, ObjectIdentity objids, - long id, - String name, String type, int ver, String user, long wsid, - String wsname, String chksum, int size, Map meta, - Map data, AuthUser auth) + final long id, + final String name, + final String type, + final int ver, + final String user, + final long wsid, + final String wsname, + final String chksum, + final int size, + final Map meta, + final Map data, + final String token) throws Exception { us.kbase.workspace.GetObjectOutput goo = CLIENT1.getObject(new us.kbase.workspace.GetObjectParams() .withId(objnames.getName()).withWorkspace(objnames.getWorkspace()) @@ -1765,7 +1766,7 @@ private void checkSavedObjectDep(ObjectIdentity objnames, ObjectIdentity objids, goo = CLIENT1.getObject(new us.kbase.workspace.GetObjectParams() .withId(objnames.getName()).withWorkspace(objnames.getWorkspace()) .withInstance(objnames.getVer()) - .withAuth(auth.getTokenString())); + .withAuth(token)); checkDeprecatedSaveInfo(goo.getMetadata(), id, name, type, ver, user, wsid, wsname, chksum, meta); assertThat("object data is correct", goo.getData().asClassInstance(Object.class), @@ -1781,7 +1782,7 @@ private void checkSavedObjectDep(ObjectIdentity objnames, ObjectIdentity objids, CLIENT1.getObjectmeta(new us.kbase.workspace.GetObjectmetaParams() .withWorkspace(objnames.getWorkspace()) .withId(objnames.getName()).withInstance(objnames.getVer()) - .withAuth(AUTH_USER2.getTokenString())); + .withAuth(TOKEN2)); checkDeprecatedSaveInfo(objmeta, id, name, type, ver, user, wsid, wsname, chksum, meta); @@ -4686,7 +4687,7 @@ public void moduleOwnerShipAndSpecRegistration() throws Exception { fail("registered typespec to module with no perms"); } catch (ServerException ex) { assertThat("got correct exception message", ex.getMessage(), - is("User " + AUTH_USER1.getUserId() + " is not in list of owners of module TestModule2")); + is("User " + USER1 + " is not in list of owners of module TestModule2")); } CLIENT2.administer(new UObject(createData( @@ -4709,7 +4710,7 @@ public void moduleOwnerShipAndSpecRegistration() throws Exception { fail("registered typespec to module with no perms"); } catch (ServerException ex) { assertThat("got correct exception message", ex.getMessage(), - is("User " + AUTH_USER1.getUserId() + " is not in list of owners of module TestModule2")); + is("User " + USER1 + " is not in list of owners of module TestModule2")); } } diff --git a/src/test/java/us/kbase/test/workspace/kbase/JSONRPCLayerTester.java b/src/test/java/us/kbase/test/workspace/kbase/JSONRPCLayerTester.java index 3a5bad2f9..b51b13831 100644 --- a/src/test/java/us/kbase/test/workspace/kbase/JSONRPCLayerTester.java +++ b/src/test/java/us/kbase/test/workspace/kbase/JSONRPCLayerTester.java @@ -38,16 +38,13 @@ import org.junit.Before; import org.junit.BeforeClass; -import us.kbase.auth.AuthConfig; import us.kbase.auth.AuthToken; -import us.kbase.auth.AuthUser; -import us.kbase.auth.ConfigurableAuthService; import us.kbase.common.service.JsonClientException; import us.kbase.common.service.ServerException; import us.kbase.common.service.Tuple11; import us.kbase.common.service.Tuple9; import us.kbase.common.service.UObject; -import us.kbase.common.test.controllers.mongo.MongoController; +import us.kbase.testutils.controllers.mongo.MongoController; import us.kbase.test.auth2.authcontroller.AuthController; import us.kbase.test.common.TestCommon; import us.kbase.test.workspace.JsonTokenStreamOCStat; @@ -116,9 +113,10 @@ public class JSONRPCLayerTester { protected static final String USER1 = "user1"; protected static final String USER2 = "user2"; protected static final String USER3 = "user3"; + protected static String TOKEN1; + protected static String TOKEN2; + protected static String TOKEN3; protected static final String STARUSER = "*"; - protected static AuthUser AUTH_USER1 = null; - protected static AuthUser AUTH_USER2 = null; protected static WorkspaceServer SERVER2 = null; protected static WorkspaceClient CLIENT_FOR_SRV2 = null; // This client connects to SERVER2 protected static WorkspaceClient CLIENT_NO_AUTH = null; @@ -194,14 +192,14 @@ public static void setUpClass() throws Exception { final URL authURL = new URL("http://localhost:" + authc.getServerPort() + "/testmode"); System.out.println("started auth server at " + authURL); TestCommon.createAuthUser(authURL, USER1, "display1"); - final String token1 = TestCommon.createLoginToken(authURL, USER1); + TOKEN1 = TestCommon.createLoginToken(authURL, USER1); TestCommon.createAuthUser(authURL, USER2, "display2"); - final String token2 = TestCommon.createLoginToken(authURL, USER2); + TOKEN2 = TestCommon.createLoginToken(authURL, USER2); TestCommon.createAuthUser(authURL, USER3, "display3"); - final String token3 = TestCommon.createLoginToken(authURL, USER3); - final AuthToken t1 = new AuthToken(token1, USER1); - final AuthToken t2 = new AuthToken(token2, USER2); - final AuthToken t3 = new AuthToken(token3, USER3); + TOKEN3 = TestCommon.createLoginToken(authURL, USER3); + final AuthToken t1 = new AuthToken(TOKEN1, USER1); + final AuthToken t2 = new AuthToken(TOKEN2, USER2); + final AuthToken t3 = new AuthToken(TOKEN3, USER3); TestCommon.createCustomRole(authURL, AUTH_ROLE_READ1, "read 1"); TestCommon.createCustomRole(authURL, AUTH_ROLE_READ2, "read 2"); TestCommon.createCustomRole(authURL, AUTH_ROLE_FULL, "full"); @@ -221,12 +219,6 @@ public static void setUpClass() throws Exception { CLIENT3.setIsInsecureHttpConnectionAllowed(true); CLIENT_NO_AUTH = new WorkspaceClient(wsurl); CLIENT_NO_AUTH.setIsInsecureHttpConnectionAllowed(true); - final ConfigurableAuthService auth = new ConfigurableAuthService( - new AuthConfig().withKBaseAuthServerURL(new URL("http://localhost:" + - authc.getServerPort() + "/testmode/api/legacy/KBase")) - .withAllowInsecureURLs(true)); - AUTH_USER1 = auth.getUserFromToken(t1); - AUTH_USER2 = auth.getUserFromToken(t2); SERVER_AUTH_ADMINS = startupWorkspaceServer( mongohost, DB_WS_NAME_AUTH2_ADMINS, DB_TYPE_NAME_AUTH2_ADMINS, true); diff --git a/src/test/java/us/kbase/test/workspace/kbase/KBaseWorkspaceConfigTest.java b/src/test/java/us/kbase/test/workspace/kbase/KBaseWorkspaceConfigTest.java index be6b86966..e90c10b3d 100644 --- a/src/test/java/us/kbase/test/workspace/kbase/KBaseWorkspaceConfigTest.java +++ b/src/test/java/us/kbase/test/workspace/kbase/KBaseWorkspaceConfigTest.java @@ -4,7 +4,7 @@ import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; import static us.kbase.test.common.TestCommon.set; -import static us.kbase.common.test.controllers.ControllerCommon.makeTempDirs; +import static us.kbase.testutils.controllers.ControllerCommon.makeTempDirs; import java.io.BufferedWriter; import java.io.IOException; diff --git a/src/test/java/us/kbase/test/workspace/kbase/LoggingTest.java b/src/test/java/us/kbase/test/workspace/kbase/LoggingTest.java index ca5b7a8fc..9fbfbd42e 100644 --- a/src/test/java/us/kbase/test/workspace/kbase/LoggingTest.java +++ b/src/test/java/us/kbase/test/workspace/kbase/LoggingTest.java @@ -31,7 +31,7 @@ import us.kbase.common.service.UObject; import us.kbase.common.service.JsonServerSyslog.SyslogOutput; import us.kbase.common.service.Tuple11; -import us.kbase.common.test.controllers.mongo.MongoController; +import us.kbase.testutils.controllers.mongo.MongoController; import us.kbase.test.auth2.authcontroller.AuthController; import us.kbase.test.common.TestCommon; import us.kbase.test.workspace.WorkspaceServerThread; diff --git a/src/test/java/us/kbase/test/workspace/kbase/SampleServiceIntegrationTest.java b/src/test/java/us/kbase/test/workspace/kbase/SampleServiceIntegrationTest.java index f17ad8746..beafbb75c 100644 --- a/src/test/java/us/kbase/test/workspace/kbase/SampleServiceIntegrationTest.java +++ b/src/test/java/us/kbase/test/workspace/kbase/SampleServiceIntegrationTest.java @@ -3,7 +3,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertThat; import static org.junit.Assert.fail; -import static us.kbase.common.test.controllers.ControllerCommon.findFreePort; +import static us.kbase.testutils.controllers.ControllerCommon.findFreePort; import static us.kbase.test.common.TestCommon.set; import static us.kbase.test.workspace.kbase.JSONRPCLayerTester.administerCommand; @@ -39,8 +39,8 @@ import us.kbase.common.service.JsonClientException; import us.kbase.common.service.ServerException; import us.kbase.common.service.UObject; -import us.kbase.common.test.TestException; -import us.kbase.common.test.controllers.mongo.MongoController; +import us.kbase.testutils.TestException; +import us.kbase.testutils.controllers.mongo.MongoController; import us.kbase.sampleservice.CreateSampleParams; import us.kbase.sampleservice.GetSampleACLsParams; import us.kbase.sampleservice.Sample; diff --git a/src/test/java/us/kbase/test/workspace/kbase/SchemaUpdaterCLITest.java b/src/test/java/us/kbase/test/workspace/kbase/SchemaUpdaterCLITest.java index 7d90cf1cf..cc8338009 100644 --- a/src/test/java/us/kbase/test/workspace/kbase/SchemaUpdaterCLITest.java +++ b/src/test/java/us/kbase/test/workspace/kbase/SchemaUpdaterCLITest.java @@ -7,7 +7,7 @@ import static org.mockito.Mockito.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static us.kbase.common.test.controllers.ControllerCommon.makeTempDirs; +import static us.kbase.testutils.controllers.ControllerCommon.makeTempDirs; import java.io.ByteArrayOutputStream; import java.nio.file.Path; diff --git a/src/test/java/us/kbase/test/workspace/kbase/TypeDelegationTest.java b/src/test/java/us/kbase/test/workspace/kbase/TypeDelegationTest.java index b397ca70d..cf27740eb 100644 --- a/src/test/java/us/kbase/test/workspace/kbase/TypeDelegationTest.java +++ b/src/test/java/us/kbase/test/workspace/kbase/TypeDelegationTest.java @@ -30,7 +30,7 @@ import us.kbase.auth.AuthToken; import us.kbase.common.service.ServerException; import us.kbase.common.service.UObject; -import us.kbase.common.test.controllers.mongo.MongoController; +import us.kbase.testutils.controllers.mongo.MongoController; import us.kbase.test.auth2.authcontroller.AuthController; import us.kbase.test.common.MapBuilder; import us.kbase.test.common.TestCommon; diff --git a/src/test/java/us/kbase/test/workspace/workspace/ObjectResolverTest.java b/src/test/java/us/kbase/test/workspace/workspace/ObjectResolverTest.java index 9c1ea670b..03b96ed26 100644 --- a/src/test/java/us/kbase/test/workspace/workspace/ObjectResolverTest.java +++ b/src/test/java/us/kbase/test/workspace/workspace/ObjectResolverTest.java @@ -18,7 +18,7 @@ import com.google.common.collect.ImmutableMap; import us.kbase.test.common.TestCommon; -import us.kbase.common.test.TestException; +import us.kbase.testutils.TestException; import us.kbase.workspace.database.AllUsers; import us.kbase.workspace.database.ObjectIDResolvedWS; import us.kbase.workspace.database.ObjectIdentifier; diff --git a/src/test/java/us/kbase/test/workspace/workspace/WorkspaceIntegrationWithGridFSTest.java b/src/test/java/us/kbase/test/workspace/workspace/WorkspaceIntegrationWithGridFSTest.java index b806f6778..c948c07f5 100644 --- a/src/test/java/us/kbase/test/workspace/workspace/WorkspaceIntegrationWithGridFSTest.java +++ b/src/test/java/us/kbase/test/workspace/workspace/WorkspaceIntegrationWithGridFSTest.java @@ -38,7 +38,7 @@ import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; -import us.kbase.common.test.controllers.mongo.MongoController; +import us.kbase.testutils.controllers.mongo.MongoController; import us.kbase.test.common.TestCommon; import us.kbase.test.workspace.WorkspaceTestCommon; import us.kbase.typedobj.core.AbsoluteTypeDefId; diff --git a/src/test/java/us/kbase/test/workspace/workspace/WorkspaceTester.java b/src/test/java/us/kbase/test/workspace/workspace/WorkspaceTester.java index c45001416..98b18ad1f 100644 --- a/src/test/java/us/kbase/test/workspace/workspace/WorkspaceTester.java +++ b/src/test/java/us/kbase/test/workspace/workspace/WorkspaceTester.java @@ -44,12 +44,12 @@ import org.junit.runners.Parameterized.Parameters; import org.slf4j.LoggerFactory; -import us.kbase.common.test.TestException; -import us.kbase.common.test.controllers.mongo.MongoController; +import us.kbase.testutils.TestException; +import us.kbase.testutils.controllers.minio.MinioController; +import us.kbase.testutils.controllers.mongo.MongoController; import us.kbase.test.common.TestCommon; import us.kbase.test.workspace.JsonTokenStreamOCStat; import us.kbase.test.workspace.WorkspaceTestCommon; -import us.kbase.test.workspace.controllers.minio.MinioController; import us.kbase.typedobj.core.LocalTypeProvider; import us.kbase.typedobj.core.TempFilesManager; import us.kbase.typedobj.core.TypeDefId; From 00969f824dc7eadcf0b8ede87cc5a0fd49672ca3 Mon Sep 17 00:00:00 2001 From: Gavin Date: Mon, 29 Apr 2024 10:06:50 -0700 Subject: [PATCH 2/2] Improve release notes re shadow jar --- build.gradle | 2 +- docsource/releasenotes.rst | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 904f9f3e1..57e496661 100644 --- a/build.gradle +++ b/build.gradle @@ -72,7 +72,7 @@ javadoc { * building the client javadocs. If we ever need the full javadocs make a new task for the * client java docs */ - /* TODO DOCS the current sdk documenation looks like invalid html to javadoc + /* TODO DOCS the current sdk documentation looks like invalid html to javadoc * need to go through and remove * also some spots with < / > that need to be wrapped with {@code } in internal code */ diff --git a/docsource/releasenotes.rst b/docsource/releasenotes.rst index 9223d4912..bd2b07177 100644 --- a/docsource/releasenotes.rst +++ b/docsource/releasenotes.rst @@ -16,7 +16,10 @@ UPDATES: against Mongo 7. * Gradle has replaced Ant as the build tool. As a consequence, all the built artifacts are now located in the build directory, including the ``update_workspace_database_schema`` script. -* A shadow jar is published on jitpack.io for supporting tests in other repos. +* A shadow jar has been published on jitpack.io for supporting tests in other repos. + This allows for starting the workspace service from a single jar in other applications. + All dependencies are included in the jar in shadow namespaces so they don't conflict with + different versions of the dependencies in the application under test. VERSION: 0.14.2 (Released 11/9/2023)