From d831105773a62ebd82dbbd6361ea360085c2553e Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Fri, 28 Jul 2023 18:27:26 +0200 Subject: [PATCH 01/18] [ignore] Code cleanup --- .../java/org/exist/interpreter/Context.java | 4 +-- .../java/org/exist/xquery/ModuleContext.java | 1 - .../java/org/exist/xquery/XQueryContext.java | 25 ++----------------- .../exquery/restxq/impl/IntegrationTest.java | 8 +++--- 4 files changed, 7 insertions(+), 31 deletions(-) diff --git a/exist-core/src/main/java/org/exist/interpreter/Context.java b/exist-core/src/main/java/org/exist/interpreter/Context.java index e07ff231740..cdf28b6af61 100644 --- a/exist-core/src/main/java/org/exist/interpreter/Context.java +++ b/exist-core/src/main/java/org/exist/interpreter/Context.java @@ -760,8 +760,6 @@ public interface Context { */ boolean tailRecursiveCall(FunctionSignature signature); - void mapModule(String namespace, XmldbURI uri); - /** * Import one or more library modules into the function signatures and in-scope variables of the importing module. * @@ -783,7 +781,7 @@ public interface Context { * XQST0070 * XQST0088 */ - Module[] importModule(@Nullable String namespaceURI, @Nullable String prefix, @Nullable AnyURIValue[] locationHints) throws XPathException; + @Nullable Module[] importModule(@Nullable String namespaceURI, @Nullable String prefix, @Nullable AnyURIValue[] locationHints) throws XPathException; /** * Returns the static location mapped to an XQuery source module, if known. diff --git a/exist-core/src/main/java/org/exist/xquery/ModuleContext.java b/exist-core/src/main/java/org/exist/xquery/ModuleContext.java index 5b18c962e94..2dadf69737a 100644 --- a/exist-core/src/main/java/org/exist/xquery/ModuleContext.java +++ b/exist-core/src/main/java/org/exist/xquery/ModuleContext.java @@ -29,7 +29,6 @@ import org.exist.dom.memtree.MemTreeBuilder; import org.exist.security.Subject; import org.exist.storage.UpdateListener; -import org.exist.util.Configuration; import org.exist.util.FileUtils; import org.exist.xmldb.XmldbURI; import org.exist.xquery.value.AnyURIValue; diff --git a/exist-core/src/main/java/org/exist/xquery/XQueryContext.java b/exist-core/src/main/java/org/exist/xquery/XQueryContext.java index 6953d748945..f16fb7040ca 100644 --- a/exist-core/src/main/java/org/exist/xquery/XQueryContext.java +++ b/exist-core/src/main/java/org/exist/xquery/XQueryContext.java @@ -164,8 +164,6 @@ public class XQueryContext implements BinaryValueManager, Context { // Inherited prefix/namespace mappings in the current context private Map inheritedInScopePrefixes = new HashMap<>(); - private Map mappedModules = new HashMap<>(); - private boolean preserveNamespaces = true; private boolean inheritNamespaces = true; @@ -654,7 +652,6 @@ public void updateContext(final XQueryContext from) { this.updateListener = from.updateListener; this.modules = from.modules; this.allModules = from.allModules; - this.mappedModules = from.mappedModules; this.dynamicOptions = from.dynamicOptions; this.staticOptions = from.staticOptions; this.db = from.db; @@ -716,7 +713,6 @@ protected void copyFields(final XQueryContext ctx) { ctx.lastVar = this.lastVar; ctx.variableStackSize = getCurrentStackSize(); ctx.contextStack = this.contextStack; - ctx.mappedModules = new HashMap<>(this.mappedModules); ctx.staticNamespaces = new HashMap<>(this.staticNamespaces); ctx.staticPrefixes = new HashMap<>(this.staticPrefixes); @@ -1432,10 +1428,6 @@ public void reset(final boolean keepGlobals) { } } - if (!keepGlobals) { - mappedModules.clear(); - } - savedState.restore(); attributes.clear(); @@ -2418,13 +2410,7 @@ public boolean tailRecursiveCall(final FunctionSignature signature) { } @Override - public void mapModule(final String namespace, final XmldbURI uri) { - mappedModules.put(namespace, uri); - } - - @Override - public Module[] importModule(@Nullable String namespaceURI, @Nullable String prefix, @Nullable AnyURIValue[] locationHints) - throws XPathException { + public @Nullable Module[] importModule(@Nullable String namespaceURI, @Nullable String prefix, @Nullable AnyURIValue[] locationHints) throws XPathException { if (XML_NS_PREFIX.equals(prefix) || XMLNS_ATTRIBUTE.equals(prefix)) { throw new XPathException(rootExpression, ErrorCodes.XQST0070, @@ -2500,16 +2486,9 @@ public Module[] importModule(@Nullable String namespaceURI, @Nullable String pre return modules; } - private Module importModuleFromLocation( - final String namespaceURI, @Nullable final String prefix, final AnyURIValue locationHint - ) throws XPathException { + protected @Nullable Module importModuleFromLocation(final String namespaceURI, @Nullable final String prefix, final AnyURIValue locationHint) throws XPathException { String location = locationHint.toString(); - //Is the module's namespace mapped to a URL ? - if (mappedModules.containsKey(location)) { - location = mappedModules.get(location).toString(); - } - // is it a Java module? if (location.startsWith(JAVA_URI_START)) { location = location.substring(JAVA_URI_START.length()); diff --git a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java index a18d5907f3f..b1cd08bb717 100644 --- a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java +++ b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java @@ -62,7 +62,7 @@ public class IntegrationTest { private static String TEST_COLLECTION = "/db/restxq/integration-test"; - private static ContentType XQUERY_CONTENT_TYPE = ContentType.create("application/xquery", "UTF-8"); + private static ContentType XQUERY_CONTENT_TYPE = ContentType.create("application/xquery", UTF_8); private static String XQUERY1 = "xquery version \"3.0\";\n" + "\n" + @@ -106,7 +106,7 @@ public static void storeResourceFunctions() throws IOException { response = executor.execute(Request .Put(getRestUri() + "/db/system/config" + TEST_COLLECTION + "/" + CollectionConfiguration.DEFAULT_COLLECTION_CONFIG_FILE) - .bodyString(COLLECTION_CONFIG, ContentType.APPLICATION_XML) + .bodyString(COLLECTION_CONFIG, ContentType.APPLICATION_XML.withCharset(UTF_8)) ).returnResponse(); assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); @@ -128,7 +128,7 @@ public static void storeResourceFunctions() throws IOException { public void mediaTypeJson1() throws IOException { final HttpResponse response = executor.execute(Request .Get(getRestXqUri() + "/media-type-json1") - .addHeader(new BasicHeader("Accept", "application/json")) + .addHeader(new BasicHeader("Accept", ContentType.APPLICATION_JSON.toString())) ).returnResponse(); assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); @@ -137,7 +137,7 @@ public void mediaTypeJson1() throws IOException { private static String asString(final InputStream inputStream) throws IOException { final StringBuilder builder = new StringBuilder(); - try(final Reader reader = new InputStreamReader(inputStream, UTF_8)) { + try (final Reader reader = new InputStreamReader(inputStream, UTF_8)) { final char cbuf[] = new char[4096]; int read = -1; while((read = reader.read(cbuf)) > -1) { From 5e092b9beb0c0557d5a6750597e0510514f823cd Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Fri, 28 Jul 2023 18:28:16 +0200 Subject: [PATCH 02/18] [bugfix] No need to call addModule here as the caller calls it also --- exist-core/src/main/java/org/exist/xquery/XQueryContext.java | 1 - 1 file changed, 1 deletion(-) diff --git a/exist-core/src/main/java/org/exist/xquery/XQueryContext.java b/exist-core/src/main/java/org/exist/xquery/XQueryContext.java index f16fb7040ca..ebcf3911027 100644 --- a/exist-core/src/main/java/org/exist/xquery/XQueryContext.java +++ b/exist-core/src/main/java/org/exist/xquery/XQueryContext.java @@ -2641,7 +2641,6 @@ private ExternalModule compileOrBorrowModule(final String prefix, final String n } final ExternalModuleImpl modExternal = new ExternalModuleImpl(namespaceURI, prefix); - addModule(namespaceURI, modExternal); final XQueryContext modContext = new ModuleContext(this, prefix, namespaceURI, location); modExternal.setContext(modContext); final XQueryLexer lexer = new XQueryLexer(modContext, reader); From 824127b152163b613ffcf458e158260d7bd7d31a Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Fri, 28 Jul 2023 18:29:22 +0200 Subject: [PATCH 03/18] [bugfix] Prevent a Library Module from importing itself Closes https://github.com/eXist-db/exist/issues/1010 --- .../java/org/exist/xquery/ModuleContext.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/exist-core/src/main/java/org/exist/xquery/ModuleContext.java b/exist-core/src/main/java/org/exist/xquery/ModuleContext.java index 2dadf69737a..4ae437712b2 100644 --- a/exist-core/src/main/java/org/exist/xquery/ModuleContext.java +++ b/exist-core/src/main/java/org/exist/xquery/ModuleContext.java @@ -97,6 +97,27 @@ public void setModuleNamespace(final String prefix, final String namespaceURI) { this.moduleNamespace = namespaceURI; } + @Override + protected @Nullable Module importModuleFromLocation(final String namespaceURI, @Nullable final String prefix, final AnyURIValue locationHint) throws XPathException { + // guard against self-recursive import - see: https://github.com/eXist-db/exist/issues/3448 + if (moduleNamespace.equals(namespaceURI) && location.equals(locationHint.toString())) { + final StringBuilder builder = new StringBuilder("The XQuery Library Module '"); + builder.append(namespaceURI); + builder.append("'"); + if (locationHint != null) { + builder.append(" at '"); + builder.append(location); + builder.append("'"); + } + builder.append(" has invalidly attempted to import itself; this will be skipped!"); + LOG.warn(builder.toString()); + + return null; + } + + return super.importModuleFromLocation(namespaceURI, prefix, locationHint); + } + @Override protected void setModulesChanged() { parentContext.setModulesChanged(); From 0df83fd3b4828e156cfd7a8449bd0f713e514309 Mon Sep 17 00:00:00 2001 From: Alan Paxton Date: Thu, 11 May 2023 15:55:41 +0100 Subject: [PATCH 04/18] [test] jetty dependencies and system properties Needed for running jetty in the extensions project, in order to test RESTXQ. [test] webapp config --- extensions/exquery/restxq/pom.xml | 26 ++++ .../exquery/restxq/impl/IntegrationTest.java | 2 +- .../src/test/resources-filtered/conf.xml | 2 +- .../WEB-INF/controller-config.xml | 47 ++++++ .../standalone-webapp/WEB-INF/web.xml | 140 ++++++++++++++++++ 5 files changed, 215 insertions(+), 2 deletions(-) create mode 100644 extensions/exquery/restxq/src/test/resources/standalone-webapp/WEB-INF/controller-config.xml create mode 100644 extensions/exquery/restxq/src/test/resources/standalone-webapp/WEB-INF/web.xml diff --git a/extensions/exquery/restxq/pom.xml b/extensions/exquery/restxq/pom.xml index e1ab8b81232..a36ab2fdab2 100644 --- a/extensions/exquery/restxq/pom.xml +++ b/extensions/exquery/restxq/pom.xml @@ -200,6 +200,18 @@ test + + + org.eclipse.jetty + jetty-deploy + test + + + org.eclipse.jetty + jetty-jmx + test + + @@ -247,11 +259,25 @@ ${project.groupId}:exist-expath:jar:${project.version} org.junit.vintage:junit-vintage-engine:jar:${junit.vintage.version} + org.eclipse.jetty:jetty-deploy:jar:${jetty.version} + org.eclipse.jetty:jetty-jmx:jar:${jetty.version} + + org.apache.maven.plugins + maven-surefire-plugin + + + ${project.basedir}/../../../exist-jetty-config/target/classes/org/exist/jetty + ${project.build.testOutputDirectory}/conf.xml + ${project.build.testOutputDirectory}/standalone-webapp + ${project.build.testOutputDirectory}/log4j2.xml + + + diff --git a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java index b1cd08bb717..fbfc5d44c56 100644 --- a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java +++ b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java @@ -108,7 +108,7 @@ public static void storeResourceFunctions() throws IOException { .Put(getRestUri() + "/db/system/config" + TEST_COLLECTION + "/" + CollectionConfiguration.DEFAULT_COLLECTION_CONFIG_FILE) .bodyString(COLLECTION_CONFIG, ContentType.APPLICATION_XML.withCharset(UTF_8)) ).returnResponse(); - assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); + //assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); response = executor.execute(Request .Put(getRestUri() + TEST_COLLECTION + "/" + XQUERY1_FILENAME) diff --git a/extensions/exquery/restxq/src/test/resources-filtered/conf.xml b/extensions/exquery/restxq/src/test/resources-filtered/conf.xml index 7600f855254..8a65424bb48 100644 --- a/extensions/exquery/restxq/src/test/resources-filtered/conf.xml +++ b/extensions/exquery/restxq/src/test/resources-filtered/conf.xml @@ -721,7 +721,7 @@ - + diff --git a/extensions/exquery/restxq/src/test/resources/standalone-webapp/WEB-INF/controller-config.xml b/extensions/exquery/restxq/src/test/resources/standalone-webapp/WEB-INF/controller-config.xml new file mode 100644 index 00000000000..6e10f7cdf43 --- /dev/null +++ b/extensions/exquery/restxq/src/test/resources/standalone-webapp/WEB-INF/controller-config.xml @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/extensions/exquery/restxq/src/test/resources/standalone-webapp/WEB-INF/web.xml b/extensions/exquery/restxq/src/test/resources/standalone-webapp/WEB-INF/web.xml new file mode 100644 index 00000000000..cda73403067 --- /dev/null +++ b/extensions/exquery/restxq/src/test/resources/standalone-webapp/WEB-INF/web.xml @@ -0,0 +1,140 @@ + + + + + eXist-db – Open Source Native XML Database + eXist-db XML Database + + + EXistServlet + org.exist.http.servlets.EXistServlet + + configuration + conf.xml + + + basedir + WEB-INF/ + + + start + true + + + hidden + false + + + xquery-submission + enabled + + + xupdate-submission + enabled + + 2 + + + + XQueryURLRewrite + org.exist.http.urlrewrite.XQueryURLRewrite + + config + WEB-INF/controller-config.xml + + + send-challenge + true + + + + + RestXqServlet + org.exist.extensions.exquery.restxq.impl.RestXqServlet + + + + + + + XQueryURLRewrite + /* + + + + + css + text/css + + + xml + application/xml + + + xsl + application/xml+xslt + + + xconf + application/xml + + + xmap + application/xml + + + ent + text/plain + + + grm + text/plain + + From 452d9fa07821efbf2e671bda4b6ca4ce128a577d Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Fri, 28 Jul 2023 19:49:59 +0200 Subject: [PATCH 05/18] [bugfix] Use the correct Namespace for XQuery Serialization spec --- .../exist/extensions/exquery/restxq/impl/IntegrationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java index fbfc5d44c56..7fb3a794292 100644 --- a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java +++ b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java @@ -68,7 +68,7 @@ public class IntegrationTest { "\n" + "module namespace mod1 = \"http://mod1\";\n" + "\n" + - "declare namespace output = \"https://www.w3.org/2010/xslt-xquery-serialization\";\n" + + "declare namespace output = \"http://www.w3.org/2010/xslt-xquery-serialization\";\n" + "\n" + "declare\n" + " %rest:GET\n" + From 24d99eed1039fee4317f8dc0ff5c10c11fe46402 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Fri, 28 Jul 2023 20:00:44 +0200 Subject: [PATCH 06/18] [bugfix] Reinstate RESTXQ test for JSON Media Type --- .../exquery/restxq/impl/IntegrationTest.java | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java index 7fb3a794292..59e3addab2d 100644 --- a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java +++ b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java @@ -26,6 +26,7 @@ */ package org.exist.extensions.exquery.restxq.impl; +import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; @@ -38,7 +39,6 @@ import org.exist.test.ExistWebServer; import org.junit.BeforeClass; import org.junit.ClassRule; -import org.junit.Ignore; import org.junit.Test; import java.io.IOException; @@ -76,7 +76,7 @@ public class IntegrationTest { " %output:media-type(\"application/json\")\n" + " %output:method(\"json\")\n" + "function mod1:media-type-json1() {\n" + - " \n" + + " AdamRetter\n" + "};"; private static String XQUERY1_FILENAME = "restxq-tests1.xqm"; private static Executor executor = null; @@ -108,7 +108,7 @@ public static void storeResourceFunctions() throws IOException { .Put(getRestUri() + "/db/system/config" + TEST_COLLECTION + "/" + CollectionConfiguration.DEFAULT_COLLECTION_CONFIG_FILE) .bodyString(COLLECTION_CONFIG, ContentType.APPLICATION_XML.withCharset(UTF_8)) ).returnResponse(); - //assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); + assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); response = executor.execute(Request .Put(getRestUri() + TEST_COLLECTION + "/" + XQUERY1_FILENAME) @@ -123,7 +123,6 @@ public static void storeResourceFunctions() throws IOException { assertNotNull(response.getEntity().getContent()); } - @Ignore("TODO(AR) need to figure out how to access the RESTXQ API from {@link ExistWebServer}") @Test public void mediaTypeJson1() throws IOException { final HttpResponse response = executor.execute(Request @@ -132,7 +131,15 @@ public void mediaTypeJson1() throws IOException { ).returnResponse(); assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); - assertEquals("", response.getEntity().toString()); + + final HttpEntity responseEntity = response.getEntity(); + assertEquals("application/json; charset=utf-8", responseEntity.getContentType().getValue()); + + final String responseBody; + try (final InputStream is = responseEntity.getContent()) { + responseBody = asString(is); + } + assertEquals("{ \"firstName\" : \"Adam\", \"lastName\" : \"Retter\" }", responseBody); } private static String asString(final InputStream inputStream) throws IOException { From bb7b9d2a0161b5cc9599d158749e42ebc7ef7ed2 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Fri, 28 Jul 2023 20:11:53 +0200 Subject: [PATCH 07/18] [test] Add further RESTXQ Media Type acceptance tests --- .../exquery/restxq/impl/IntegrationTest.java | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java index 59e3addab2d..743906914df 100644 --- a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java +++ b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java @@ -70,14 +70,26 @@ public class IntegrationTest { "\n" + "declare namespace output = \"http://www.w3.org/2010/xslt-xquery-serialization\";\n" + "\n" + + "declare %private variable $mod1:data := document { AdamRetter } ;\n" + + "\n" + "declare\n" + " %rest:GET\n" + " %rest:path(\"/media-type-json1\")\n" + " %output:media-type(\"application/json\")\n" + " %output:method(\"json\")\n" + "function mod1:media-type-json1() {\n" + - " AdamRetter\n" + - "};"; + " $mod1:data/person\n" + + "};\n" + + "\n" + + "declare\n" + + " %rest:GET\n" + + " %rest:path(\"/media-type-xml1\")\n" + + " %output:media-type(\"application/xml\")\n" + + " %output:method(\"xml\")\n" + + " %output:indent(\"no\")\n" + + "function mod1:media-type-xml1() {\n" + + " $mod1:data/person\n" + + "};\n"; private static String XQUERY1_FILENAME = "restxq-tests1.xqm"; private static Executor executor = null; @@ -142,6 +154,25 @@ public void mediaTypeJson1() throws IOException { assertEquals("{ \"firstName\" : \"Adam\", \"lastName\" : \"Retter\" }", responseBody); } + @Test + public void mediaTypeXml1() throws IOException { + final HttpResponse response = executor.execute(Request + .Get(getRestXqUri() + "/media-type-xml1") + .addHeader(new BasicHeader("Accept", ContentType.APPLICATION_XML.withCharset(UTF_8).toString())) + ).returnResponse(); + + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + final HttpEntity responseEntity = response.getEntity(); + assertEquals(ContentType.APPLICATION_XML.withCharset(UTF_8).toString(), responseEntity.getContentType().getValue()); + + final String responseBody; + try (final InputStream is = responseEntity.getContent()) { + responseBody = asString(is); + } + assertEquals("AdamRetter", responseBody); + } + private static String asString(final InputStream inputStream) throws IOException { final StringBuilder builder = new StringBuilder(); try (final Reader reader = new InputStreamReader(inputStream, UTF_8)) { From bffd3e799df394e408239fea5822facf92b3acb4 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Fri, 28 Jul 2023 20:20:22 +0200 Subject: [PATCH 08/18] [test] Add further tests for RESTXQ Media Type serialization --- .../exquery/restxq/impl/IntegrationTest.java | 63 +++++++++++++------ 1 file changed, 45 insertions(+), 18 deletions(-) diff --git a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java index 743906914df..de9fbffae4d 100644 --- a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java +++ b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java @@ -78,6 +78,15 @@ public class IntegrationTest { " %output:media-type(\"application/json\")\n" + " %output:method(\"json\")\n" + "function mod1:media-type-json1() {\n" + + " $mod1:data\n" + + "};\n" + + "\n" + + "declare\n" + + " %rest:GET\n" + + " %rest:path(\"/media-type-json2\")\n" + + " %output:media-type(\"application/json\")\n" + + " %output:method(\"json\")\n" + + "function mod1:media-type-json2() {\n" + " $mod1:data/person\n" + "};\n" + "\n" + @@ -88,6 +97,16 @@ public class IntegrationTest { " %output:method(\"xml\")\n" + " %output:indent(\"no\")\n" + "function mod1:media-type-xml1() {\n" + + " $mod1:data\n" + + "};\n" + + "\n" + + "declare\n" + + " %rest:GET\n" + + " %rest:path(\"/media-type-xml2\")\n" + + " %output:media-type(\"application/xml\")\n" + + " %output:method(\"xml\")\n" + + " %output:indent(\"no\")\n" + + "function mod1:media-type-xml2() {\n" + " $mod1:data/person\n" + "};\n"; private static String XQUERY1_FILENAME = "restxq-tests1.xqm"; @@ -137,40 +156,48 @@ public static void storeResourceFunctions() throws IOException { @Test public void mediaTypeJson1() throws IOException { - final HttpResponse response = executor.execute(Request - .Get(getRestXqUri() + "/media-type-json1") - .addHeader(new BasicHeader("Accept", ContentType.APPLICATION_JSON.toString())) - ).returnResponse(); - - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); - - final HttpEntity responseEntity = response.getEntity(); - assertEquals("application/json; charset=utf-8", responseEntity.getContentType().getValue()); + assertMediaTypeResponse("/media-type-json1", ContentType.APPLICATION_JSON, + "application/json; charset=utf-8", + "{ \"firstName\" : \"Adam\", \"lastName\" : \"Retter\" }"); + } - final String responseBody; - try (final InputStream is = responseEntity.getContent()) { - responseBody = asString(is); - } - assertEquals("{ \"firstName\" : \"Adam\", \"lastName\" : \"Retter\" }", responseBody); + @Test + public void mediaTypeJson2() throws IOException { + assertMediaTypeResponse("/media-type-json2", ContentType.APPLICATION_JSON, + "application/json; charset=utf-8", + "{ \"firstName\" : \"Adam\", \"lastName\" : \"Retter\" }"); } @Test public void mediaTypeXml1() throws IOException { + assertMediaTypeResponse("/media-type-xml1", ContentType.APPLICATION_XML.withCharset(UTF_8), + ContentType.APPLICATION_XML.withCharset(UTF_8).toString(), + "AdamRetter"); + } + + @Test + public void mediaTypeXml2() throws IOException { + assertMediaTypeResponse("/media-type-xml2", ContentType.APPLICATION_XML.withCharset(UTF_8), + ContentType.APPLICATION_XML.withCharset(UTF_8).toString(), + "AdamRetter"); + } + + private void assertMediaTypeResponse(final String uriEndpoint, final ContentType acceptContentType, final String expectedResponseContentType, final String expectedResponseBody) throws IOException { final HttpResponse response = executor.execute(Request - .Get(getRestXqUri() + "/media-type-xml1") - .addHeader(new BasicHeader("Accept", ContentType.APPLICATION_XML.withCharset(UTF_8).toString())) + .Get(getRestXqUri() + uriEndpoint) + .addHeader(new BasicHeader("Accept", acceptContentType.toString())) ).returnResponse(); assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); final HttpEntity responseEntity = response.getEntity(); - assertEquals(ContentType.APPLICATION_XML.withCharset(UTF_8).toString(), responseEntity.getContentType().getValue()); + assertEquals(expectedResponseContentType, responseEntity.getContentType().getValue()); final String responseBody; try (final InputStream is = responseEntity.getContent()) { responseBody = asString(is); } - assertEquals("AdamRetter", responseBody); + assertEquals(expectedResponseBody, responseBody); } private static String asString(final InputStream inputStream) throws IOException { From 74ef58eba0225c696cc5254e52bfbef61aeebfd1 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Fri, 28 Jul 2023 21:57:45 +0200 Subject: [PATCH 09/18] [bugfix] JSON Serialization should also handle Document nodes --- .../main/java/org/exist/util/serializer/XQuerySerializer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exist-core/src/main/java/org/exist/util/serializer/XQuerySerializer.java b/exist-core/src/main/java/org/exist/util/serializer/XQuerySerializer.java index 0db36aaa40c..366e3866cbc 100644 --- a/exist-core/src/main/java/org/exist/util/serializer/XQuerySerializer.java +++ b/exist-core/src/main/java/org/exist/util/serializer/XQuerySerializer.java @@ -105,8 +105,8 @@ private void serializeXML(final Sequence sequence, final int start, final int ho private void serializeJSON(final Sequence sequence, final long compilationTime, final long executionTime) throws SAXException, XPathException { // backwards compatibility: if the sequence contains a single element, we assume // it should be transformed to JSON following the rules of the old JSON writer - if (sequence.hasOne() && Type.subTypeOf(sequence.getItemType(), Type.ELEMENT)) { - serializeXML(sequence, 1, sequence.getItemCount(), false, false, compilationTime, executionTime); + if (sequence.hasOne() && (Type.subTypeOf(sequence.getItemType(), Type.DOCUMENT) || Type.subTypeOf(sequence.getItemType(), Type.ELEMENT))) { + serializeXML(sequence, 1, 1, false, false, compilationTime, executionTime); } else { JSONSerializer serializer = new JSONSerializer(broker, outputProperties); serializer.serialize(sequence, writer); From b2239b5cfca4adffe34aec1f29c45fa76c9252be Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Fri, 28 Jul 2023 21:58:36 +0200 Subject: [PATCH 10/18] [refactor] Rename RESTXQ Media Type tests --- .../{IntegrationTest.java => MediaTypeIntegrationTest.java} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/{IntegrationTest.java => MediaTypeIntegrationTest.java} (98%) diff --git a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/MediaTypeIntegrationTest.java similarity index 98% rename from extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java rename to extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/MediaTypeIntegrationTest.java index de9fbffae4d..f82fe1f7dbb 100644 --- a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/IntegrationTest.java +++ b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/MediaTypeIntegrationTest.java @@ -50,7 +50,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertNotNull; -public class IntegrationTest { +public class MediaTypeIntegrationTest { private static String COLLECTION_CONFIG = "\n" + @@ -60,7 +60,7 @@ public class IntegrationTest { " \n" + ""; - private static String TEST_COLLECTION = "/db/restxq/integration-test"; + private static String TEST_COLLECTION = "/db/restxq/media-type-integration-test"; private static ContentType XQUERY_CONTENT_TYPE = ContentType.create("application/xquery", UTF_8); private static String XQUERY1 = From 3776e2c84886c604eae6a0283c747ef653d7bcd6 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Fri, 28 Jul 2023 22:50:51 +0200 Subject: [PATCH 11/18] [test] Test for Self Import Circular Dependency. See https://github.com/eXist-db/exist/issues/3448#issue-640018884 --- .../restxq/impl/AbstractIntegrationTest.java | 163 ++++++++++++++++++ .../restxq/impl/MediaTypeIntegrationTest.java | 75 +------- ...portCircularDependencyIntegrationTest.java | 85 +++++++++ 3 files changed, 252 insertions(+), 71 deletions(-) create mode 100644 extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/AbstractIntegrationTest.java create mode 100644 extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/SelfImportCircularDependencyIntegrationTest.java diff --git a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/AbstractIntegrationTest.java b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/AbstractIntegrationTest.java new file mode 100644 index 00000000000..6415b4b05c8 --- /dev/null +++ b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/AbstractIntegrationTest.java @@ -0,0 +1,163 @@ +/* + * Copyright © 2001, Adam Retter + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.exist.extensions.exquery.restxq.impl; + +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.fluent.Executor; +import org.apache.http.client.fluent.Request; +import org.apache.http.entity.ContentType; +import org.exist.TestUtils; +import org.exist.collections.CollectionConfiguration; +import org.exist.dom.memtree.SAXAdapter; +import org.exist.test.ExistWebServer; +import org.exist.util.ExistSAXParserFactory; +import org.exquery.restxq.Namespace; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +import org.xml.sax.XMLReader; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.Reader; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +public class AbstractIntegrationTest { + + @ClassRule + public static ExistWebServer existWebServer = new ExistWebServer(true, false, true, true); + + protected static Executor executor = null; + + private static String COLLECTION_CONFIG = + "\n" + + "\n" + + " \n" + + " \n" + + " \n" + + ""; + + private static ContentType XQUERY_CONTENT_TYPE = ContentType.create("application/xquery", UTF_8); + + protected static String getServerUri() { + return "http://localhost:" + existWebServer.getPort(); + } + + protected static String getRestUri() { + return getServerUri() + "/rest"; + } + + protected static String getRestXqUri() { + return getServerUri() + "/restxq"; + } + + @BeforeClass + public static void setupExecutor() { + executor = Executor.newInstance() + .auth(TestUtils.ADMIN_DB_USER, TestUtils.ADMIN_DB_PWD) + .authPreemptive(new HttpHost("localhost", existWebServer.getPort())); + } + + protected static void enableRestXqTrigger(final String collectionPath) throws IOException { + final HttpResponse response = executor.execute(Request + .Put(getRestUri() + "/db/system/config" + collectionPath + "/" + CollectionConfiguration.DEFAULT_COLLECTION_CONFIG_FILE) + .bodyString(COLLECTION_CONFIG, ContentType.APPLICATION_XML.withCharset(UTF_8)) + ).returnResponse(); + assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); + } + + protected static void storeXquery(final String collectionPath, final String xqueryFilename, final String xquery) throws IOException { + final HttpResponse response = executor.execute(Request + .Put(getRestUri() + collectionPath + "/" + xqueryFilename) + .bodyString(xquery, XQUERY_CONTENT_TYPE) + ).returnResponse(); + assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); + } + + protected static void assertRestXqResourceFunctionsCount(final int expectedCount) throws IOException { + final HttpResponse response = executor.execute(Request + .Get(getRestUri() + "/db/?_query=rest:resource-functions()") + ).returnResponse(); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + + final Document doc; + try (final InputStream is = response.getEntity().getContent()) { + assertNotNull(is); + doc = parseXml(is); + } + assertNotNull(doc); + + final Element docElem = doc.getDocumentElement(); + assertEquals("exist:result", docElem.getNodeName()); + final NodeList resourceFunctionsList = docElem.getElementsByTagNameNS(Namespace.ANNOTATION_NS, "resource-functions"); + assertEquals(1, resourceFunctionsList.getLength()); + + final Element resourceFunctionsElem = (Element) resourceFunctionsList.item(0); + final NodeList resourceFunctionList = resourceFunctionsElem.getElementsByTagNameNS(Namespace.ANNOTATION_NS, "resource-function"); + assertEquals(expectedCount, resourceFunctionList.getLength()); + } + + protected static Document parseXml(final InputStream inputStream) throws IOException { + final SAXParserFactory saxParserFactory = ExistSAXParserFactory.getSAXParserFactory(); + saxParserFactory.setNamespaceAware(true); + try { + final SAXParser saxParser = saxParserFactory.newSAXParser(); + final XMLReader reader = saxParser.getXMLReader(); + final SAXAdapter adapter = new SAXAdapter(); + reader.setContentHandler(adapter); + reader.parse(new InputSource(inputStream)); + return adapter.getDocument(); + } catch (final SAXException | ParserConfigurationException e) { + throw new IOException(e.getMessage(), e); + } + } + + protected static String asString(final InputStream inputStream) throws IOException { + final StringBuilder builder = new StringBuilder(); + try (final Reader reader = new InputStreamReader(inputStream, UTF_8)) { + final char cbuf[] = new char[4096]; + int read = -1; + while((read = reader.read(cbuf)) > -1) { + builder.append(cbuf, 0, read); + } + } + return builder.toString(); + } +} diff --git a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/MediaTypeIntegrationTest.java b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/MediaTypeIntegrationTest.java index f82fe1f7dbb..f0a7127ada3 100644 --- a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/MediaTypeIntegrationTest.java +++ b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/MediaTypeIntegrationTest.java @@ -27,42 +27,24 @@ package org.exist.extensions.exquery.restxq.impl; import org.apache.http.HttpEntity; -import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; -import org.apache.http.client.fluent.Executor; import org.apache.http.client.fluent.Request; import org.apache.http.entity.ContentType; import org.apache.http.message.BasicHeader; -import org.exist.TestUtils; -import org.exist.collections.CollectionConfiguration; -import org.exist.test.ExistWebServer; import org.junit.BeforeClass; -import org.junit.ClassRule; import org.junit.Test; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; import static org.junit.Assert.assertEquals; import static java.nio.charset.StandardCharsets.UTF_8; -import static org.junit.Assert.assertNotNull; -public class MediaTypeIntegrationTest { - - private static String COLLECTION_CONFIG = - "\n" + - "\n" + - " \n" + - " \n" + - " \n" + - ""; +public class MediaTypeIntegrationTest extends AbstractIntegrationTest { private static String TEST_COLLECTION = "/db/restxq/media-type-integration-test"; - private static ContentType XQUERY_CONTENT_TYPE = ContentType.create("application/xquery", UTF_8); private static String XQUERY1 = "xquery version \"3.0\";\n" + "\n" + @@ -110,48 +92,12 @@ public class MediaTypeIntegrationTest { " $mod1:data/person\n" + "};\n"; private static String XQUERY1_FILENAME = "restxq-tests1.xqm"; - private static Executor executor = null; - - @ClassRule - public static ExistWebServer existWebServer = new ExistWebServer(true, false, true, true); - - private static String getServerUri() { - return "http://localhost:" + existWebServer.getPort(); - } - - private static String getRestUri() { - return getServerUri() + "/rest"; - } - - private static String getRestXqUri() { - return getServerUri() + "/restxq"; - } @BeforeClass public static void storeResourceFunctions() throws IOException { - executor = Executor.newInstance() - .auth(TestUtils.ADMIN_DB_USER, TestUtils.ADMIN_DB_PWD) - .authPreemptive(new HttpHost("localhost", existWebServer.getPort())); - - HttpResponse response = null; - - response = executor.execute(Request - .Put(getRestUri() + "/db/system/config" + TEST_COLLECTION + "/" + CollectionConfiguration.DEFAULT_COLLECTION_CONFIG_FILE) - .bodyString(COLLECTION_CONFIG, ContentType.APPLICATION_XML.withCharset(UTF_8)) - ).returnResponse(); - assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); - - response = executor.execute(Request - .Put(getRestUri() + TEST_COLLECTION + "/" + XQUERY1_FILENAME) - .bodyString(XQUERY1, XQUERY_CONTENT_TYPE) - ).returnResponse(); - assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); - - response = executor.execute(Request - .Get(getRestUri() + "/db/?_query=rest:resource-functions()") - ).returnResponse(); - assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); - assertNotNull(response.getEntity().getContent()); + enableRestXqTrigger(TEST_COLLECTION); + storeXquery(TEST_COLLECTION, XQUERY1_FILENAME, XQUERY1); + assertRestXqResourceFunctionsCount(4); } @Test @@ -199,17 +145,4 @@ private void assertMediaTypeResponse(final String uriEndpoint, final ContentType } assertEquals(expectedResponseBody, responseBody); } - - private static String asString(final InputStream inputStream) throws IOException { - final StringBuilder builder = new StringBuilder(); - try (final Reader reader = new InputStreamReader(inputStream, UTF_8)) { - final char cbuf[] = new char[4096]; - int read = -1; - while((read = reader.read(cbuf)) > -1) { - builder.append(cbuf, 0, read); - } - } - return builder.toString(); - } - } diff --git a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/SelfImportCircularDependencyIntegrationTest.java b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/SelfImportCircularDependencyIntegrationTest.java new file mode 100644 index 00000000000..100686894b4 --- /dev/null +++ b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/SelfImportCircularDependencyIntegrationTest.java @@ -0,0 +1,85 @@ +/* + * Copyright © 2001, Adam Retter + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.exist.extensions.exquery.restxq.impl; + +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; + +/** + * Test for XQuery Library Modules that import themselves, i.e. a self circular dependency. + * See issue #3448. + */ +public class SelfImportCircularDependencyIntegrationTest extends AbstractIntegrationTest { + + private static String COLLECTION_CONFIG = + "\n" + + "\n" + + " \n" + + " \n" + + " \n" + + ""; + + private static String TEST_COLLECTION = "/db/restxq/self-import-circular-dependency-integration-test"; + + private static String XQUERY_FILENAME = "mod1.xqm"; + + private static String STAGE1_XQUERY = + "xquery version \"3.1\";\n" + + "\n" + + "module namespace test = \"test\";\n" + + "\n" + + "declare function test:f1() {\n" + + " \n" + + "};"; + + private static String STAGE2_XQUERY = + "xquery version \"3.1\";\n" + + "\n" + + "module namespace test = \"test\";\n" + + "\n" + + "import module namespace test = \"test\" at \"" + XQUERY_FILENAME + "\";\n" + + "\n" + + "declare function test:f1() {\n" + + " \n" + + "};"; + + private static String STAGE3_XQUERY = STAGE1_XQUERY; + + @BeforeClass + public static void storeResourceFunctions() throws IOException { + enableRestXqTrigger(TEST_COLLECTION); + } + + @Test + public void storeSelfDependentXqueryLibraryModule() throws IOException { + storeXquery(TEST_COLLECTION, XQUERY_FILENAME, STAGE1_XQUERY); + storeXquery(TEST_COLLECTION, XQUERY_FILENAME, STAGE2_XQUERY); + storeXquery(TEST_COLLECTION, XQUERY_FILENAME, STAGE3_XQUERY); + } +} From 5eb6d971c82b990e8ddddf94a4a9423f6f2991d8 Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Sat, 29 Jul 2023 16:38:57 +0200 Subject: [PATCH 12/18] [test] Add XQuery 1.0 and XQuery 3.1 tests for cyclic library module imports --- .../org/exist/xquery/ImportModuleTest.java | 230 +++++++++++++++--- 1 file changed, 198 insertions(+), 32 deletions(-) diff --git a/exist-core/src/test/java/org/exist/xquery/ImportModuleTest.java b/exist-core/src/test/java/org/exist/xquery/ImportModuleTest.java index 254a8a2c247..f224803f45f 100644 --- a/exist-core/src/test/java/org/exist/xquery/ImportModuleTest.java +++ b/exist-core/src/test/java/org/exist/xquery/ImportModuleTest.java @@ -51,7 +51,6 @@ import org.exist.util.StringInputSource; import org.exist.xmldb.XmldbURI; import org.exist.xquery.value.Sequence; -import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.w3c.dom.Element; @@ -1277,37 +1276,39 @@ public void variablesBetweenModules() throws EXistException, PermissionDeniedExc } } - /** - * Checks that XQST0093 is raised if there exists a sequence of modules M1 ... Mi ... M1. + * Check that for XQuery 1.0. eXist-db raises XQTS0093 error when there exists a sequence of library modules MM -> LM1 -> LM2 -> LM1. + * + * See issue #4996. + * See the XQuery 1.0 spec. section: 4.11 Module Import */ - @Ignore("eXist-db does not have cyclic import checks, but it should!") @Test - public void cyclic1() throws EXistException, IOException, PermissionDeniedException, LockException, SAXException { + public void xq10CyclicTwoLibraryModules() throws EXistException, IOException, PermissionDeniedException, LockException, SAXException { final String module1 = "xquery version \"1.0\";\n" + - "module namespace impl1 = \"http://example.com/impl1\";\n" + - "import module namespace impl2 = \"http://example.com/impl2\"" + - " at \"xmldb:exist:///db/impl2.xqm\";\n" + - "declare function impl1:f1($a as xs:string) as xs:string {\n" + - " $a\n" + - "};\n"; + "module namespace impl1 = \"http://example.com/impl1\";\n" + + "import module namespace impl2 = \"http://example.com/impl2\"" + + " at \"xmldb:exist:///db/impl2.xqm\";\n" + + "declare function impl1:f1($a as xs:string) {\n" + + " {$a}\n" + + "};\n"; final String module2 = "xquery version \"1.0\";\n" + - "module namespace impl2 = \"http://example.com/impl2\";\n" + - "import module namespace impl1 = \"http://example.com/impl1\"" + - " at \"xmldb:exist:///db/impl1.xqm\";\n" + - "declare function impl2:f1($a as xs:string) as xs:string {\n" + - " $a\n" + - "};\n"; + "module namespace impl2 = \"http://example.com/impl2\";\n" + + "import module namespace impl1 = \"http://example.com/impl1\"" + + " at \"xmldb:exist:///db/impl1.xqm\";\n" + + "declare function impl2:f1($a as xs:string) as xs:string {\n" + + " {$a}\n" + + "};\n"; final String query = - "import module namespace impl1 = \"http://example.com/impl1\"" + - " at \"xmldb:exist:///db/impl1.xqm\";\n" + - "\n" + - " {impl1:f1(\"to impl1\")}" + - "\n"; + "xquery version \"1.0\";\n" + + "import module namespace impl1 = \"http://example.com/impl1\"" + + " at \"xmldb:exist:///db/impl1.xqm\";\n" + + "\n" + + " {impl1:f1(\"from main\")}" + + "\n"; final BrokerPool pool = existEmbeddedServer.getBrokerPool(); final Source source = new StringSource(query); @@ -1340,18 +1341,96 @@ public void cyclic1() throws EXistException, IOException, PermissionDeniedExcept } /** - * Checks that XQST0093 is raised if there exists a sequence of modules M1 ... Mi ... M1. + * Check that for XQuery 3.1 eXist-db executes the query even when there exists a sequence of library modules MM -> LM1 -> LM2 -> LM1. + * + * See issue #4996. + * See the XQuery 3.1 spec. section: 4.12.4 Cycles + */ + @Test + public void xq31CyclicTwoLibraryModules() throws EXistException, IOException, PermissionDeniedException, LockException, SAXException, XPathException { + final String module1 = + "xquery version \"3.1\";\n" + + "module namespace impl1 = \"http://example.com/impl1\";\n" + + "import module namespace impl2 = \"http://example.com/impl2\"" + + " at \"xmldb:exist:///db/impl2.xqm\";\n" + + "declare function impl1:f1($a as xs:string) {\n" + + " {$a}\n" + + "};\n"; + + final String module2 = + "xquery version \"3.1\";\n" + + "module namespace impl2 = \"http://example.com/impl2\";\n" + + "import module namespace impl1 = \"http://example.com/impl1\"" + + " at \"xmldb:exist:///db/impl1.xqm\";\n" + + "declare function impl2:f1($a as xs:string) as xs:string {\n" + + " {$a}\n" + + "};\n"; + + final String query = + "xquery version \"3.1\";\n" + + "import module namespace impl1 = \"http://example.com/impl1\"" + + " at \"xmldb:exist:///db/impl1.xqm\";\n" + + "\n" + + " {impl1:f1(\"from main\")}" + + "\n"; + + final BrokerPool pool = existEmbeddedServer.getBrokerPool(); + final Source source = new StringSource(query); + try (final DBBroker broker = pool.get(Optional.of(pool.getSecurityManager().getSystemSubject())); + final Txn transaction = pool.getTransactionManager().beginTransaction()) { + + // store modules + storeModules(broker, transaction,"/db", + Tuple("impl1.xqm", module1), + Tuple("impl2.xqm", module2) + ); + + // execute query + final Tuple2 contextAndResult = withCompiledQuery(broker, source, compiledXQuery -> { + final Sequence result = executeQuery(broker, compiledXQuery); + return Tuple(compiledXQuery.getContext(), result); + }); + + // check that the result was correct + assertNotNull(contextAndResult._2); + assertEquals(1, contextAndResult._2.getItemCount()); + final Element doc = (Element) contextAndResult._2.itemAt(0); + assertNotNull(doc); + + final javax.xml.transform.Source actual = Input.fromDocument(doc.getOwnerDocument()).build(); + final javax.xml.transform.Source expected = Input.fromString( + "" + + "from main" + + "" + ).build(); + + final Diff diff = DiffBuilder + .compare(expected) + .withTest(actual) + .checkForSimilar() + .build(); + + assertFalse(diff.toString(), diff.hasDifferences()); + + transaction.commit(); + } + } + + /** + * Check that for XQuery 1.0. eXist-db raises XQTS0093 error when there exists a sequence of library modules MM -> LM1 -> LM2 -> LM3 -> LM1. + * + * See issue #4996. + * See the XQuery 1.0 spec. section: 4.11 Module Import */ - @Ignore("eXist-db does not have cyclic import checks, but it should!") @Test - public void cyclic2() throws EXistException, IOException, PermissionDeniedException, LockException, SAXException { + public void xq10CyclicThreeLibraryModules() throws EXistException, IOException, PermissionDeniedException, LockException, SAXException { final String module1 = "xquery version \"1.0\";\n" + "module namespace impl1 = \"http://example.com/impl1\";\n" + "import module namespace impl2 = \"http://example.com/impl2\"" + " at \"xmldb:exist:///db/impl2.xqm\";\n" + - "declare function impl1:f1($a as xs:string) as xs:string {\n" + - " $a\n" + + "declare function impl1:f1($a as xs:string) {\n" + + " {$a}\n" + "};\n"; final String module2 = @@ -1359,8 +1438,8 @@ public void cyclic2() throws EXistException, IOException, PermissionDeniedExcept "module namespace impl2 = \"http://example.com/impl2\";\n" + "import module namespace impl3 = \"http://example.com/impl3\"" + " at \"xmldb:exist:///db/impl3.xqm\";\n" + - "declare function impl2:f1($a as xs:string) as xs:string {\n" + - " $a\n" + + "declare function impl2:f1($a as xs:string) {\n" + + " {$a}\n" + "};\n"; final String module3 = @@ -1368,15 +1447,16 @@ public void cyclic2() throws EXistException, IOException, PermissionDeniedExcept "module namespace impl3 = \"http://example.com/impl3\";\n" + "import module namespace impl1 = \"http://example.com/impl1\"" + " at \"xmldb:exist:///db/impl1.xqm\";\n" + - "declare function impl3:f1($a as xs:string) as xs:string {\n" + - " $a\n" + + "declare function impl3:f1($a as xs:string) {\n" + + " {$a}\n" + "};\n"; final String query = + "xquery version \"1.0\";\n" + "import module namespace impl1 = \"http://example.com/impl1\"" + " at \"xmldb:exist:///db/impl1.xqm\";\n" + "\n" + - " {impl1:f1(\"to impl1\")}" + + " {impl1:f1(\"from main\")}" + "\n"; final BrokerPool pool = existEmbeddedServer.getBrokerPool(); @@ -1410,6 +1490,92 @@ public void cyclic2() throws EXistException, IOException, PermissionDeniedExcept } } + /** + * Check that for XQuery 3.1 eXist-db executes the query even when there exists a sequence of library modules MM -> LM1 -> LM2 -> LM3 -> LM1. + * + * See issue #4996. + * See the XQuery 3.1 spec. section: 4.12.4 Cycles + */ + @Test + public void xq31CyclicThreeLibraryModules() throws EXistException, IOException, PermissionDeniedException, LockException, SAXException, XPathException { + final String module1 = + "xquery version \"3.1\";\n" + + "module namespace impl1 = \"http://example.com/impl1\";\n" + + "import module namespace impl2 = \"http://example.com/impl2\"" + + " at \"xmldb:exist:///db/impl2.xqm\";\n" + + "declare function impl1:f1($a as xs:string) {\n" + + " {$a}\n" + + "};\n"; + + final String module2 = + "xquery version \"3.1\";\n" + + "module namespace impl2 = \"http://example.com/impl2\";\n" + + "import module namespace impl3 = \"http://example.com/impl3\"" + + " at \"xmldb:exist:///db/impl3.xqm\";\n" + + "declare function impl2:f1($a as xs:string) {\n" + + " {$a}\n" + + "};\n"; + + final String module3 = + "xquery version \"3.1\";\n" + + "module namespace impl3 = \"http://example.com/impl3\";\n" + + "import module namespace impl1 = \"http://example.com/impl1\"" + + " at \"xmldb:exist:///db/impl1.xqm\";\n" + + "declare function impl3:f1($a as xs:string) {\n" + + " {$a}\n" + + "};\n"; + + final String query = + "xquery version \"3.1\";\n" + + "import module namespace impl1 = \"http://example.com/impl1\"" + + " at \"xmldb:exist:///db/impl1.xqm\";\n" + + "\n" + + " {impl1:f1(\"from main\")}" + + "\n"; + + final BrokerPool pool = existEmbeddedServer.getBrokerPool(); + final Source source = new StringSource(query); + try (final DBBroker broker = pool.get(Optional.of(pool.getSecurityManager().getSystemSubject())); + final Txn transaction = pool.getTransactionManager().beginTransaction()) { + + // store modules + storeModules(broker, transaction,"/db", + Tuple("impl1.xqm", module1), + Tuple("impl2.xqm", module2), + Tuple("impl3.xqm", module3) + ); + + // execute query + final Tuple2 contextAndResult = withCompiledQuery(broker, source, compiledXQuery -> { + final Sequence result = executeQuery(broker, compiledXQuery); + return Tuple(compiledXQuery.getContext(), result); + }); + + // check that the result was correct + assertNotNull(contextAndResult._2); + assertEquals(1, contextAndResult._2.getItemCount()); + final Element doc = (Element) contextAndResult._2.itemAt(0); + assertNotNull(doc); + + final javax.xml.transform.Source actual = Input.fromDocument(doc.getOwnerDocument()).build(); + final javax.xml.transform.Source expected = Input.fromString( + "" + + "from main" + + "" + ).build(); + + final Diff diff = DiffBuilder + .compare(expected) + .withTest(actual) + .checkForSimilar() + .build(); + + assertFalse(diff.toString(), diff.hasDifferences()); + + transaction.commit(); + } + } + private void storeModules(final DBBroker broker, final Txn transaction, final String collectionUri, final Tuple2... modules) throws PermissionDeniedException, IOException, SAXException, LockException, EXistException { // store modules try (final Collection collection = broker.openCollection(XmldbURI.create(collectionUri), Lock.LockMode.WRITE_LOCK)) { From dae3352405abb871dffabe02bf8a8e15d945161d Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Fri, 28 Jul 2023 23:31:17 +0200 Subject: [PATCH 13/18] [test] Add RESTXQ tests for circular dependencies between XQuery Library Modules --- .../impl/AbstractClassIntegrationTest.java | 75 ++++++++ .../impl/AbstractInstanceIntegrationTest.java | 84 +++++++++ .../restxq/impl/AbstractIntegrationTest.java | 54 +++--- ...raryCircularDependencyIntegrationTest.java | 164 ++++++++++++++++++ .../restxq/impl/MediaTypeIntegrationTest.java | 2 +- ...portCircularDependencyIntegrationTest.java | 10 +- 6 files changed, 349 insertions(+), 40 deletions(-) create mode 100644 extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/AbstractClassIntegrationTest.java create mode 100644 extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/AbstractInstanceIntegrationTest.java create mode 100644 extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/LibraryCircularDependencyIntegrationTest.java diff --git a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/AbstractClassIntegrationTest.java b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/AbstractClassIntegrationTest.java new file mode 100644 index 00000000000..ce815094f63 --- /dev/null +++ b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/AbstractClassIntegrationTest.java @@ -0,0 +1,75 @@ +/* + * Copyright © 2001, Adam Retter + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.exist.extensions.exquery.restxq.impl; + +import org.apache.http.HttpHost; +import org.apache.http.client.fluent.Executor; +import org.exist.TestUtils; +import org.exist.test.ExistWebServer; +import org.junit.BeforeClass; +import org.junit.ClassRule; + +import java.io.IOException; + +public abstract class AbstractClassIntegrationTest extends AbstractIntegrationTest { + + @ClassRule + public static ExistWebServer existWebServer = new ExistWebServer(true, false, true, true); + + protected static Executor executor = null; + + @BeforeClass + public static void setupExecutor() { + executor = Executor.newInstance() + .auth(TestUtils.ADMIN_DB_USER, TestUtils.ADMIN_DB_PWD) + .authPreemptive(new HttpHost("localhost", existWebServer.getPort())); + } + + protected static String getServerUri() { + return getServerUri(existWebServer); + } + + protected static String getRestUri() { + return getRestUri(existWebServer); + } + + protected static String getRestXqUri() { + return getRestXqUri(existWebServer); + } + + protected static void enableRestXqTrigger(final String collectionPath) throws IOException { + enableRestXqTrigger(existWebServer, executor, collectionPath); + } + + protected static void storeXquery(final String collectionPath, final String xqueryFilename, final String xquery) throws IOException { + storeXquery(existWebServer, executor, collectionPath, xqueryFilename, xquery); + } + + protected static void assertRestXqResourceFunctionsCount(final int expectedCount) throws IOException { + assertRestXqResourceFunctionsCount(existWebServer, executor, expectedCount); + } +} diff --git a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/AbstractInstanceIntegrationTest.java b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/AbstractInstanceIntegrationTest.java new file mode 100644 index 00000000000..9249c0dbe81 --- /dev/null +++ b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/AbstractInstanceIntegrationTest.java @@ -0,0 +1,84 @@ +/* + * Copyright © 2001, Adam Retter + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.exist.extensions.exquery.restxq.impl; + +import org.apache.http.HttpHost; +import org.apache.http.client.fluent.Executor; +import org.exist.TestUtils; +import org.exist.test.ExistWebServer; +import org.junit.Before; +import org.junit.Rule; +import org.w3c.dom.NodeList; + +import java.io.IOException; + +public abstract class AbstractInstanceIntegrationTest extends AbstractIntegrationTest { + + @Rule + public ExistWebServer existWebServer = new ExistWebServer(true, false, true, true); + + protected Executor executor = null; + + @Before + public void setupExecutor() { + executor = Executor.newInstance() + .auth(TestUtils.ADMIN_DB_USER, TestUtils.ADMIN_DB_PWD) + .authPreemptive(new HttpHost("localhost", existWebServer.getPort())); + } + + protected String getServerUri() { + return getServerUri(existWebServer); + } + + protected String getRestUri() { + return getRestUri(existWebServer); + } + + protected String getRestXqUri() { + return getRestXqUri(existWebServer); + } + + protected void enableRestXqTrigger(final String collectionPath) throws IOException { + enableRestXqTrigger(existWebServer, executor, collectionPath); + } + + protected void storeXquery(final String collectionPath, final String xqueryFilename, final String xquery) throws IOException { + storeXquery(existWebServer, executor, collectionPath, xqueryFilename, xquery); + } + + protected void removeXquery(final String collectionPath, final String xqueryFilename) throws IOException { + removeXquery(existWebServer, executor, collectionPath, xqueryFilename); + } + + protected void assertRestXqResourceFunctionsCount(final int expectedCount) throws IOException { + assertRestXqResourceFunctionsCount(existWebServer, executor, expectedCount); + } + + protected NodeList getRestXqResourceFunctions() throws IOException { + return getRestXqResourceFunctions(existWebServer, executor); + } +} diff --git a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/AbstractIntegrationTest.java b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/AbstractIntegrationTest.java index 6415b4b05c8..f13116f02d0 100644 --- a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/AbstractIntegrationTest.java +++ b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/AbstractIntegrationTest.java @@ -26,20 +26,16 @@ */ package org.exist.extensions.exquery.restxq.impl; -import org.apache.http.HttpHost; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.client.fluent.Executor; import org.apache.http.client.fluent.Request; import org.apache.http.entity.ContentType; -import org.exist.TestUtils; import org.exist.collections.CollectionConfiguration; import org.exist.dom.memtree.SAXAdapter; import org.exist.test.ExistWebServer; import org.exist.util.ExistSAXParserFactory; import org.exquery.restxq.Namespace; -import org.junit.BeforeClass; -import org.junit.ClassRule; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NodeList; @@ -59,12 +55,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -public class AbstractIntegrationTest { - - @ClassRule - public static ExistWebServer existWebServer = new ExistWebServer(true, false, true, true); - - protected static Executor executor = null; +public abstract class AbstractIntegrationTest { private static String COLLECTION_CONFIG = "\n" + @@ -76,44 +67,48 @@ public class AbstractIntegrationTest { private static ContentType XQUERY_CONTENT_TYPE = ContentType.create("application/xquery", UTF_8); - protected static String getServerUri() { + protected static String getServerUri(final ExistWebServer existWebServer) { return "http://localhost:" + existWebServer.getPort(); } - protected static String getRestUri() { - return getServerUri() + "/rest"; - } - - protected static String getRestXqUri() { - return getServerUri() + "/restxq"; + protected static String getRestUri(final ExistWebServer existWebServer) { + return getServerUri(existWebServer) + "/rest"; } - @BeforeClass - public static void setupExecutor() { - executor = Executor.newInstance() - .auth(TestUtils.ADMIN_DB_USER, TestUtils.ADMIN_DB_PWD) - .authPreemptive(new HttpHost("localhost", existWebServer.getPort())); + protected static String getRestXqUri(final ExistWebServer existWebServer) { + return getServerUri(existWebServer) + "/restxq"; } - protected static void enableRestXqTrigger(final String collectionPath) throws IOException { + protected static void enableRestXqTrigger(final ExistWebServer existWebServer, final Executor executor, final String collectionPath) throws IOException { final HttpResponse response = executor.execute(Request - .Put(getRestUri() + "/db/system/config" + collectionPath + "/" + CollectionConfiguration.DEFAULT_COLLECTION_CONFIG_FILE) + .Put(getRestUri(existWebServer) + "/db/system/config" + collectionPath + "/" + CollectionConfiguration.DEFAULT_COLLECTION_CONFIG_FILE) .bodyString(COLLECTION_CONFIG, ContentType.APPLICATION_XML.withCharset(UTF_8)) ).returnResponse(); assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); } - protected static void storeXquery(final String collectionPath, final String xqueryFilename, final String xquery) throws IOException { + protected static void storeXquery(final ExistWebServer existWebServer, final Executor executor, final String collectionPath, final String xqueryFilename, final String xquery) throws IOException { final HttpResponse response = executor.execute(Request - .Put(getRestUri() + collectionPath + "/" + xqueryFilename) + .Put(getRestUri(existWebServer) + collectionPath + "/" + xqueryFilename) .bodyString(xquery, XQUERY_CONTENT_TYPE) ).returnResponse(); assertEquals(HttpStatus.SC_CREATED, response.getStatusLine().getStatusCode()); } - protected static void assertRestXqResourceFunctionsCount(final int expectedCount) throws IOException { + protected static void removeXquery(final ExistWebServer existWebServer, final Executor executor, final String collectionPath, final String xqueryFilename) throws IOException { + final HttpResponse response = executor.execute(Request + .Delete(getRestUri(existWebServer) + collectionPath + "/" + xqueryFilename) + ).returnResponse(); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + } + + protected static void assertRestXqResourceFunctionsCount(final ExistWebServer existWebServer, final Executor executor, final int expectedCount) throws IOException { + assertEquals(expectedCount, getRestXqResourceFunctions(existWebServer, executor).getLength()); + } + + protected static NodeList getRestXqResourceFunctions(final ExistWebServer existWebServer, final Executor executor) throws IOException { final HttpResponse response = executor.execute(Request - .Get(getRestUri() + "/db/?_query=rest:resource-functions()") + .Get(getRestUri(existWebServer) + "/db/?_query=rest:resource-functions()") ).returnResponse(); assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); @@ -130,8 +125,7 @@ protected static void assertRestXqResourceFunctionsCount(final int expectedCount assertEquals(1, resourceFunctionsList.getLength()); final Element resourceFunctionsElem = (Element) resourceFunctionsList.item(0); - final NodeList resourceFunctionList = resourceFunctionsElem.getElementsByTagNameNS(Namespace.ANNOTATION_NS, "resource-function"); - assertEquals(expectedCount, resourceFunctionList.getLength()); + return resourceFunctionsElem.getElementsByTagNameNS(Namespace.ANNOTATION_NS, "resource-function"); } protected static Document parseXml(final InputStream inputStream) throws IOException { diff --git a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/LibraryCircularDependencyIntegrationTest.java b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/LibraryCircularDependencyIntegrationTest.java new file mode 100644 index 00000000000..133a1aa1489 --- /dev/null +++ b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/LibraryCircularDependencyIntegrationTest.java @@ -0,0 +1,164 @@ +/* + * Copyright © 2001, Adam Retter + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package org.exist.extensions.exquery.restxq.impl; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Map; + +import static com.evolvedbinary.j8fu.tuple.Tuple.Tuple; +import static org.exist.util.MapUtil.hashMap; +import static org.junit.Assert.assertEquals; + +/** + * Test for XQuery Library Modules that depend on each other, i.e. a circular dependency. + * See issue #1010. + * + * mod1.xqm contains the Resource Functions, and depends on mod2.xqm, mod2.xqm depends on mod3.xqm which in turn depends on mod2.xqm. + */ +@RunWith(Parameterized.class) +public class LibraryCircularDependencyIntegrationTest extends AbstractInstanceIntegrationTest { + + /** + * All possibilities for the order that the modules could be stored to the database in. + */ + @Parameterized.Parameters(name = "{0} {1} {2}") + public static java.util.Collection data() { + return Arrays.asList(new Object[][] { + { XQUERY_MOD1_FILENAME, XQUERY_MOD2_FILENAME, XQUERY_MOD3_FILENAME } + }); + } + + private static String TEST_COLLECTION = "/db/restxq/library-circular-dependency-integration-test"; + + private static String XQUERY_MOD1_FILENAME = "mod1.xqm"; + private static String XQUERY_MOD2_FILENAME = "mod2.xqm"; + private static String XQUERY_MOD3_FILENAME = "mod3.xqm"; + + private static String MOD1_XQUERY = + "xquery version \"3.1\";\n" + + "\n" + + "module namespace mod1 = \"mod1\";\n" + + "import module namespace mod2 = \"mod2\" at \"" + XQUERY_MOD2_FILENAME + "\";\n" + + "\n" + + "declare namespace rest = \"http://exquery.org/ns/restxq\";\n" + + "\n" + + "declare\n" + + " %rest:GET\n" + + " %rest:path(\"/mod1\")\n" + + "function mod1:f1() {\n" + + " {mod2:f2()}\n" + + "};"; + + private static String MOD2_XQUERY = + "xquery version \"3.1\";\n" + + "\n" + + "module namespace mod2 = \"mod2\";\n" + + "import module namespace mod3 = \"mod3\" at \"" + XQUERY_MOD3_FILENAME + "\";\n" + + "\n" + + "declare function mod2:f2() {\n" + + " {mod3:f3()}\n" + + "};" + + "\n" + + "declare function mod2:other() {\n" + + " other\n" + + "};"; + + private static String MOD3_XQUERY = + "xquery version \"3.1\";\n" + + "\n" + + "module namespace mod3 = \"mod3\";\n" + + "import module namespace mod2 = \"mod2\" at \"" + XQUERY_MOD2_FILENAME + "\";\n" + + "\n" + + "declare function mod3:f3() {\n" + + " {mod2:other()}\n" + + "};"; + + private static Map filenameToXQuery = hashMap( + Tuple(XQUERY_MOD1_FILENAME, MOD1_XQUERY), + Tuple(XQUERY_MOD2_FILENAME, MOD2_XQUERY), + Tuple(XQUERY_MOD3_FILENAME, MOD3_XQUERY) + ); + + private static final int MAX_WAIT_PERIOD = 10 * 1000; // 10 seconds + private static final int WAIT_INTERVAL = 1000; // 1 second + + @Parameterized.Parameter(0) + public String storeFirstModuleFilename; + @Parameterized.Parameter(1) + public String storeSecondModuleFilename; + @Parameterized.Parameter(2) + public String storeThirdModuleFilename; + + @Before + public void enableRestXq() throws IOException { + enableRestXqTrigger(TEST_COLLECTION); + } + + @After + public void removeResourceFunctions() throws IOException, InterruptedException { + removeXquery(TEST_COLLECTION, storeThirdModuleFilename); + removeXquery(TEST_COLLECTION, storeSecondModuleFilename); + removeXquery(TEST_COLLECTION, storeFirstModuleFilename); + + assertRestXqResourceFunctionsCount(0); + } + + @Test + public void storeCircularXqueryLibraryModules() throws IOException, InterruptedException { + final String firstModuleContent = filenameToXQuery.get(storeFirstModuleFilename); + storeXquery(TEST_COLLECTION, storeFirstModuleFilename, firstModuleContent); + assertRestXqResourceFunctionsCount(0); + + final String secondModuleContent = filenameToXQuery.get(storeSecondModuleFilename); + storeXquery(TEST_COLLECTION, storeSecondModuleFilename, secondModuleContent); + assertRestXqResourceFunctionsCount(0); + + final String thirdModuleContent = filenameToXQuery.get(storeThirdModuleFilename); + storeXquery(TEST_COLLECTION, storeThirdModuleFilename, thirdModuleContent); + + // RESTXQ is eventually consistent in its approach to try and connect the dependencies for compilation, so we need to give it a little time to do so... + int restXqResourceFunctionsCount = 0; + long timeSlept = 0; + while (timeSlept < MAX_WAIT_PERIOD) { + restXqResourceFunctionsCount = getRestXqResourceFunctions().getLength(); + if (restXqResourceFunctionsCount == 1) { + break; + } + + Thread.sleep(WAIT_INTERVAL); + timeSlept += WAIT_INTERVAL; + } + assertEquals(1, restXqResourceFunctionsCount); + } +} diff --git a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/MediaTypeIntegrationTest.java b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/MediaTypeIntegrationTest.java index f0a7127ada3..e71987f8c4d 100644 --- a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/MediaTypeIntegrationTest.java +++ b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/MediaTypeIntegrationTest.java @@ -41,7 +41,7 @@ import static org.junit.Assert.assertEquals; import static java.nio.charset.StandardCharsets.UTF_8; -public class MediaTypeIntegrationTest extends AbstractIntegrationTest { +public class MediaTypeIntegrationTest extends AbstractClassIntegrationTest { private static String TEST_COLLECTION = "/db/restxq/media-type-integration-test"; diff --git a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/SelfImportCircularDependencyIntegrationTest.java b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/SelfImportCircularDependencyIntegrationTest.java index 100686894b4..8a4d0ba4cd4 100644 --- a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/SelfImportCircularDependencyIntegrationTest.java +++ b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/SelfImportCircularDependencyIntegrationTest.java @@ -35,15 +35,7 @@ * Test for XQuery Library Modules that import themselves, i.e. a self circular dependency. * See issue #3448. */ -public class SelfImportCircularDependencyIntegrationTest extends AbstractIntegrationTest { - - private static String COLLECTION_CONFIG = - "\n" + - "\n" + - " \n" + - " \n" + - " \n" + - ""; +public class SelfImportCircularDependencyIntegrationTest extends AbstractClassIntegrationTest { private static String TEST_COLLECTION = "/db/restxq/self-import-circular-dependency-integration-test"; From 9e54ea011f352a5d4d0a95feae8e1539cf8b0d9d Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Sat, 29 Jul 2023 16:36:04 +0200 Subject: [PATCH 14/18] [test] Add explicit namespace declaration --- .../extensions/exquery/restxq/impl/MediaTypeIntegrationTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/MediaTypeIntegrationTest.java b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/MediaTypeIntegrationTest.java index e71987f8c4d..2574c373c80 100644 --- a/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/MediaTypeIntegrationTest.java +++ b/extensions/exquery/restxq/src/test/java/org/exist/extensions/exquery/restxq/impl/MediaTypeIntegrationTest.java @@ -50,6 +50,7 @@ public class MediaTypeIntegrationTest extends AbstractClassIntegrationTest { "\n" + "module namespace mod1 = \"http://mod1\";\n" + "\n" + + "declare namespace rest = \"http://exquery.org/ns/restxq\";\n" + "declare namespace output = \"http://www.w3.org/2010/xslt-xquery-serialization\";\n" + "\n" + "declare %private variable $mod1:data := document { AdamRetter } ;\n" + From 4b92888ce3a787b3abe03882559cbf29b88b967a Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Sat, 29 Jul 2023 16:55:40 +0200 Subject: [PATCH 15/18] [refactor] Swap arguments for internal consistency purposes --- .../java/org/exist/xquery/ModuleContext.java | 10 ++++------ .../java/org/exist/xquery/XQueryContext.java | 18 +++++++++--------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/exist-core/src/main/java/org/exist/xquery/ModuleContext.java b/exist-core/src/main/java/org/exist/xquery/ModuleContext.java index 4ae437712b2..858ff2cdf18 100644 --- a/exist-core/src/main/java/org/exist/xquery/ModuleContext.java +++ b/exist-core/src/main/java/org/exist/xquery/ModuleContext.java @@ -58,19 +58,17 @@ public class ModuleContext extends XQueryContext { private static final Logger LOG = LogManager.getLogger(ModuleContext.class); private XQueryContext parentContext; - private String modulePrefix; private String moduleNamespace; + private String modulePrefix; private final String location; - public ModuleContext(final XQueryContext parentContext, final String modulePrefix, final String moduleNamespace, - final String location) { + public ModuleContext(final XQueryContext parentContext, final String moduleNamespace, final String modulePrefix, final String location) { super(parentContext != null ? parentContext.db : null, parentContext != null ? parentContext.getConfiguration() : null, null, false); - - this.modulePrefix = modulePrefix; this.moduleNamespace = moduleNamespace; + this.modulePrefix = modulePrefix; this.location = location; setParentContext(parentContext); @@ -196,7 +194,7 @@ public void updateContext(final XQueryContext from) { @Override public XQueryContext copyContext() { - final ModuleContext ctx = new ModuleContext(parentContext, modulePrefix, moduleNamespace, location); + final ModuleContext ctx = new ModuleContext(parentContext, moduleNamespace, modulePrefix, location); copyFields(ctx); try { ctx.declareNamespace(modulePrefix, moduleNamespace); diff --git a/exist-core/src/main/java/org/exist/xquery/XQueryContext.java b/exist-core/src/main/java/org/exist/xquery/XQueryContext.java index ebcf3911027..969ae5581b2 100644 --- a/exist-core/src/main/java/org/exist/xquery/XQueryContext.java +++ b/exist-core/src/main/java/org/exist/xquery/XQueryContext.java @@ -591,7 +591,7 @@ public Optional getRepository() { } // build a module object from the source - final ExternalModule module = compileOrBorrowModule(prefix, namespace, location, src); + final ExternalModule module = compileOrBorrowModule(namespace, prefix, location, src); return module; } catch (final PermissionDeniedException e) { @@ -2522,7 +2522,7 @@ public boolean tailRecursiveCall(final FunctionSignature signature) { } final Source moduleSource = new DBSource(getBroker().getBrokerPool(), (BinaryDocument) sourceDoc, true); - return compileOrBorrowModule(prefix, namespaceURI, location, moduleSource); + return compileOrBorrowModule(namespaceURI, prefix, location, moduleSource); } catch (final PermissionDeniedException e) { throw moduleLoadException("Permission denied to read module source from location hint URI '" + location + ".", location, e); @@ -2559,7 +2559,7 @@ public boolean tailRecursiveCall(final FunctionSignature signature) { throw moduleLoadException("Permission denied to read module source from location hint URI '" + location + ".", location, e); } - return compileOrBorrowModule(prefix, namespaceURI, location, moduleSource); + return compileOrBorrowModule(namespaceURI, prefix, location, moduleSource); } protected XPathException moduleLoadException(final String message, final String moduleLocation) @@ -2591,8 +2591,8 @@ public Iterator getMappedModuleURIs() { /** * Compile of borrow an already compile module from the cache. * - * @param prefix the module namespace prefix * @param namespaceURI the module namespace URI + * @param prefix the module namespace prefix * @param location the location hint * @param source the source for the module * @@ -2600,9 +2600,9 @@ public Iterator getMappedModuleURIs() { * * @throws XPathException if the module could not be loaded (XQST0059) or compiled (XPST0003) */ - private ExternalModule compileOrBorrowModule(final String prefix, final String namespaceURI, final String location, + private ExternalModule compileOrBorrowModule(final String namespaceURI, final String prefix, final String location, final Source source) throws XPathException { - final ExternalModule module = compileModule(prefix, namespaceURI, location, source); + final ExternalModule module = compileModule(namespaceURI, prefix, location, source); if (module != null) { addModule(module.getNamespaceURI(), module); declareModuleVars(module); @@ -2613,14 +2613,14 @@ private ExternalModule compileOrBorrowModule(final String prefix, final String n /** * Compile an XQuery Module * - * @param prefix the namespace prefix of the module. * @param namespaceURI the namespace URI of the module. + * @param prefix the namespace prefix of the module. * @param location the location of the module * @param source the source of the module. * @return The compiled module, or null if the source is not a module * @throws XPathException if the module could not be loaded (XQST0059) or compiled (XPST0003) */ - private @Nullable ExternalModule compileModule(final String prefix, String namespaceURI, final String location, + private @Nullable ExternalModule compileModule(String namespaceURI, final String prefix, final String location, final Source source) throws XPathException { if (LOG.isDebugEnabled()) { LOG.debug("Loading module from {}", location); @@ -2641,7 +2641,7 @@ private ExternalModule compileOrBorrowModule(final String prefix, final String n } final ExternalModuleImpl modExternal = new ExternalModuleImpl(namespaceURI, prefix); - final XQueryContext modContext = new ModuleContext(this, prefix, namespaceURI, location); + final XQueryContext modContext = new ModuleContext(this, namespaceURI, prefix, location); modExternal.setContext(modContext); final XQueryLexer lexer = new XQueryLexer(modContext, reader); final XQueryParser parser = new XQueryParser(lexer); From 15e62ea62955693d1b92c8340a5a19c58e9ea50b Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Sat, 29 Jul 2023 18:08:39 +0200 Subject: [PATCH 16/18] [bugfix] Cyclic dependency import detection for XQuery 1.0 (i.e. XQST0093) --- exist-core/pom.xml | 11 ++ .../java/org/exist/xquery/ModuleContext.java | 53 ++++++- .../java/org/exist/xquery/XQueryContext.java | 131 ++++++++++++++++++ 3 files changed, 192 insertions(+), 3 deletions(-) diff --git a/exist-core/pom.xml b/exist-core/pom.xml index 90163e4ad44..70cd7b8b863 100644 --- a/exist-core/pom.xml +++ b/exist-core/pom.xml @@ -514,6 +514,17 @@ ant + + org.jgrapht + jgrapht-core + 1.5.2 + + + + org.jgrapht + jgrapht-opt + 1.5.2 + junit diff --git a/exist-core/src/main/java/org/exist/xquery/ModuleContext.java b/exist-core/src/main/java/org/exist/xquery/ModuleContext.java index 858ff2cdf18..b1a4d29a6d5 100644 --- a/exist-core/src/main/java/org/exist/xquery/ModuleContext.java +++ b/exist-core/src/main/java/org/exist/xquery/ModuleContext.java @@ -48,10 +48,10 @@ /** - * Subclass of {@link org.exist.xquery.XQueryContext} for - * imported modules. + * Subclass of {@link org.exist.xquery.XQueryContext} for imported modules. * * @author wolf + * @author Adam Retter */ public class ModuleContext extends XQueryContext { @@ -66,7 +66,8 @@ public ModuleContext(final XQueryContext parentContext, final String moduleNames super(parentContext != null ? parentContext.db : null, parentContext != null ? parentContext.getConfiguration() : null, null, - false); + false, + null); this.moduleNamespace = moduleNamespace; this.modulePrefix = modulePrefix; this.location = location; @@ -95,6 +96,52 @@ public void setModuleNamespace(final String prefix, final String namespaceURI) { this.moduleNamespace = namespaceURI; } + @Override + protected void addModuleVertex(final ModuleVertex moduleVertex) { + getRootContext().addModuleVertex(moduleVertex); + } + + protected boolean hasModuleVertex(final ModuleVertex moduleVertex) { + return getRootContext().hasModuleVertex(moduleVertex); + } + + @Override + protected void addModuleEdge(final ModuleVertex source, final ModuleVertex sink) { + getRootContext().addModuleEdge(source, sink); + } + + @Override + protected boolean hasModulePath(final ModuleVertex source, final ModuleVertex sink) { + return getRootContext().hasModulePath(source, sink); + } + + @Override + public @Nullable Module[] importModule(@Nullable String namespaceURI, @Nullable String prefix, @Nullable AnyURIValue[] locationHints) throws XPathException { + final ModuleVertex thisModuleVertex = new ModuleVertex(moduleNamespace, location); + + for (final AnyURIValue locationHint : locationHints) { + final ModuleVertex imporedModuleVertex = new ModuleVertex(namespaceURI, locationHint.toString()); + + if (!hasModuleVertex(imporedModuleVertex)) { + addModuleVertex(imporedModuleVertex); + } else { + // Check if there is already a path from the imported module to this module + if (getXQueryVersion() == 10 && namespaceURI != null && locationHints != null && hasModulePath(imporedModuleVertex, thisModuleVertex)) { + throw new XPathException(ErrorCodes.XQST0093, "Detected cyclic import between modules: " + getModuleNamespace() + " at: " + getLocation() + ", and: " + namespaceURI + " at: " + locationHint.toString()); + } + } + + if (!hasModuleVertex(thisModuleVertex)) { + // NOTE(AR) may occur when the actual module has a different namespace from that of the `import module namespace`... will later raise an XQST0047 error + addModuleVertex(thisModuleVertex); + } + + addModuleEdge(thisModuleVertex, imporedModuleVertex); + } + + return super.importModule(namespaceURI, prefix, locationHints); + } + @Override protected @Nullable Module importModuleFromLocation(final String namespaceURI, @Nullable final String prefix, final AnyURIValue locationHint) throws XPathException { // guard against self-recursive import - see: https://github.com/eXist-db/exist/issues/3448 diff --git a/exist-core/src/main/java/org/exist/xquery/XQueryContext.java b/exist-core/src/main/java/org/exist/xquery/XQueryContext.java index 969ae5581b2..323e85943b5 100644 --- a/exist-core/src/main/java/org/exist/xquery/XQueryContext.java +++ b/exist-core/src/main/java/org/exist/xquery/XQueryContext.java @@ -33,6 +33,7 @@ import java.nio.file.Paths; import java.util.*; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiFunction; import java.util.function.Consumer; @@ -94,6 +95,13 @@ import org.exist.xquery.update.Modification; import org.exist.xquery.util.SerializerUtils; import org.exist.xquery.value.*; +import org.jgrapht.alg.interfaces.ShortestPathAlgorithm; +import org.jgrapht.alg.shortestpath.TransitNodeRoutingShortestPath; +import org.jgrapht.graph.DefaultEdge; +import org.jgrapht.graph.DefaultGraphType; +import org.jgrapht.opt.graph.fastutil.FastutilMapGraph; +import org.jgrapht.util.ConcurrencyUtil; +import org.jgrapht.util.SupplierUtil; import org.w3c.dom.Node; import static com.evolvedbinary.j8fu.OptionalUtil.or; @@ -224,6 +232,11 @@ public class XQueryContext implements BinaryValueManager, Context { */ private Object2ObjectMap allModules = new Object2ObjectOpenHashMap<>(); + /** + * Describes a graph of all the modules and how they import each other. + */ + private @Nullable final FastutilMapGraph modulesDependencyGraph; + /** * Used to save current state when modules are imported dynamically */ @@ -454,6 +467,10 @@ public XQueryContext(@Nullable final Database db, @Nullable final Configuration } protected XQueryContext(@Nullable final Database db, @Nullable final Configuration configuration, @Nullable final Profiler profiler, final boolean loadDefaults) { + this(db, configuration, profiler, loadDefaults, new FastutilMapGraph<>(null, SupplierUtil.createDefaultEdgeSupplier(), DefaultGraphType.directedPseudograph().asUnweighted())); + } + + protected XQueryContext(@Nullable final Database db, @Nullable final Configuration configuration, @Nullable final Profiler profiler, final boolean loadDefaults, final @Nullable FastutilMapGraph modulesDependencyGraph) { this.db = db; // if needed, fallback to db.getConfiguration @@ -474,6 +491,8 @@ protected XQueryContext(@Nullable final Database db, @Nullable final Configurati this.profiler = new Profiler(null); } + this.modulesDependencyGraph = modulesDependencyGraph; + this.watchdog = new XQueryWatchDog(this); // load configuration defaults @@ -1529,6 +1548,65 @@ public void addModule(final String namespaceURI, final Module module) { addRootModule(namespaceURI, module); } + /** + * Add a vertex to the Modules Dependency Graph. + * + * @param moduleVertex the module vertex + */ + protected void addModuleVertex(final ModuleVertex moduleVertex) { + modulesDependencyGraph.addVertex(moduleVertex); + } + + /** + * Check if a vertex exists in the Modules Dependency Graph. + * + * @param moduleVertex the module vertex to look for + * + * @return true if the module vertex exists, false otherwise + */ + protected boolean hasModuleVertex(final ModuleVertex moduleVertex) { + return modulesDependencyGraph.containsVertex(moduleVertex); + } + + /** + * Add an edge between two Modules in the Dependency Graph. + * + * @param source the importing module + * @param sink the imported module + */ + protected void addModuleEdge(final ModuleVertex source, final ModuleVertex sink) { + modulesDependencyGraph.addEdge(source, sink); + } + + /** + * Look for a path between two Modules in the Dependency Graph. + * + * @param source the module to start searching from + * @param sink the destination module to attempt to reach + * + * @return true, if there is a path between the mdoules, false otherwise + */ + protected boolean hasModulePath(final ModuleVertex source, final ModuleVertex sink) { + if (modulesDependencyGraph == null) { + return false; + } + + ThreadPoolExecutor executor = null; + try { + executor = ConcurrencyUtil.createThreadPoolExecutor(2); + final ShortestPathAlgorithm spa = new TransitNodeRoutingShortestPath<>(modulesDependencyGraph, executor); + return spa.getPath(source, sink) != null; + } finally { + if (executor != null) { + try { + ConcurrencyUtil.shutdownExecutionService(executor); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + } + protected void setRootModules(final String namespaceURI, @Nullable final Module[] modules) { if (modules == null) { allModules.remove(namespaceURI); // unbind the module @@ -2641,6 +2719,11 @@ private ExternalModule compileOrBorrowModule(final String namespaceURI, final St } final ExternalModuleImpl modExternal = new ExternalModuleImpl(namespaceURI, prefix); + + // NOTE(AR) this is needed to support cyclic imports in XQuery 3.1, see: https://github.com/eXist-db/exist/pull/4996 + addModule(namespaceURI, modExternal); + addModuleVertex(new ModuleVertex(namespaceURI, location)); + final XQueryContext modContext = new ModuleContext(this, namespaceURI, prefix, location); modExternal.setContext(modContext); final XQueryLexer lexer = new XQueryLexer(modContext, reader); @@ -3547,4 +3630,52 @@ public String getStringValue() { return sb.toString(); } + + @Immutable + public static class ModuleVertex { + private final String namespaceURI; + private final String location; + + public ModuleVertex(final String namespaceURI) { + this.namespaceURI = namespaceURI; + this.location = null; + } + + public ModuleVertex(final String namespaceURI, final String location) { + this.namespaceURI = namespaceURI; + this.location = location; + } + + @Override + public boolean equals(final Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + final ModuleVertex that = (ModuleVertex) o; + if (!namespaceURI.equals(that.namespaceURI)) { + return false; + } + return location.equals(that.location); + } + + @Override + public int hashCode() { + int result = namespaceURI.hashCode(); + result = 31 * result + location.hashCode(); + return result; + } + + @Override + public String toString() { + return "Module{" + + "namespaceURI='" + namespaceURI + '\'' + + "location='" + location + '\'' + + '}'; + } + } } From 6679db095f902a62750cc1dade5658f669641edb Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Mon, 31 Jul 2023 16:53:04 +0200 Subject: [PATCH 17/18] [optimisation] Lazy initialisation of Modules Dependency Graph --- .../java/org/exist/xquery/ModuleContext.java | 3 +- .../java/org/exist/xquery/XQueryContext.java | 52 +++++++++++-------- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/exist-core/src/main/java/org/exist/xquery/ModuleContext.java b/exist-core/src/main/java/org/exist/xquery/ModuleContext.java index b1a4d29a6d5..67c28b38b80 100644 --- a/exist-core/src/main/java/org/exist/xquery/ModuleContext.java +++ b/exist-core/src/main/java/org/exist/xquery/ModuleContext.java @@ -66,8 +66,7 @@ public ModuleContext(final XQueryContext parentContext, final String moduleNames super(parentContext != null ? parentContext.db : null, parentContext != null ? parentContext.getConfiguration() : null, null, - false, - null); + false); this.moduleNamespace = moduleNamespace; this.modulePrefix = modulePrefix; this.location = location; diff --git a/exist-core/src/main/java/org/exist/xquery/XQueryContext.java b/exist-core/src/main/java/org/exist/xquery/XQueryContext.java index 323e85943b5..7a1e533d4b2 100644 --- a/exist-core/src/main/java/org/exist/xquery/XQueryContext.java +++ b/exist-core/src/main/java/org/exist/xquery/XQueryContext.java @@ -95,6 +95,7 @@ import org.exist.xquery.update.Modification; import org.exist.xquery.util.SerializerUtils; import org.exist.xquery.value.*; +import org.jgrapht.Graph; import org.jgrapht.alg.interfaces.ShortestPathAlgorithm; import org.jgrapht.alg.shortestpath.TransitNodeRoutingShortestPath; import org.jgrapht.graph.DefaultEdge; @@ -235,7 +236,8 @@ public class XQueryContext implements BinaryValueManager, Context { /** * Describes a graph of all the modules and how they import each other. */ - private @Nullable final FastutilMapGraph modulesDependencyGraph; + private @Nullable Graph modulesDependencyGraph; + private @Nullable ThreadPoolExecutor modulesDependencyGraphSPExecutor; /** * Used to save current state when modules are imported dynamically @@ -467,10 +469,6 @@ public XQueryContext(@Nullable final Database db, @Nullable final Configuration } protected XQueryContext(@Nullable final Database db, @Nullable final Configuration configuration, @Nullable final Profiler profiler, final boolean loadDefaults) { - this(db, configuration, profiler, loadDefaults, new FastutilMapGraph<>(null, SupplierUtil.createDefaultEdgeSupplier(), DefaultGraphType.directedPseudograph().asUnweighted())); - } - - protected XQueryContext(@Nullable final Database db, @Nullable final Configuration configuration, @Nullable final Profiler profiler, final boolean loadDefaults, final @Nullable FastutilMapGraph modulesDependencyGraph) { this.db = db; // if needed, fallback to db.getConfiguration @@ -491,8 +489,6 @@ protected XQueryContext(@Nullable final Database db, @Nullable final Configurati this.profiler = new Profiler(null); } - this.modulesDependencyGraph = modulesDependencyGraph; - this.watchdog = new XQueryWatchDog(this); // load configuration defaults @@ -1459,6 +1455,15 @@ public void reset(final boolean keepGlobals) { httpContext = null; } + if (modulesDependencyGraphSPExecutor != null) { + try { + ConcurrencyUtil.shutdownExecutionService(modulesDependencyGraphSPExecutor); + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + } + modulesDependencyGraphSPExecutor = null; + } + analyzed = false; } @@ -1548,13 +1553,21 @@ public void addModule(final String namespaceURI, final Module module) { addRootModule(namespaceURI, module); } + private Graph getModulesDependencyGraph() { + // NOTE(AR) intentionally lazily initialised! + if (modulesDependencyGraph == null) { + this.modulesDependencyGraph = new FastutilMapGraph<>(null, SupplierUtil.createDefaultEdgeSupplier(), DefaultGraphType.directedPseudograph().asUnweighted()); + } + return modulesDependencyGraph; + } + /** * Add a vertex to the Modules Dependency Graph. * * @param moduleVertex the module vertex */ protected void addModuleVertex(final ModuleVertex moduleVertex) { - modulesDependencyGraph.addVertex(moduleVertex); + getModulesDependencyGraph().addVertex(moduleVertex); } /** @@ -1565,7 +1578,7 @@ protected void addModuleVertex(final ModuleVertex moduleVertex) { * @return true if the module vertex exists, false otherwise */ protected boolean hasModuleVertex(final ModuleVertex moduleVertex) { - return modulesDependencyGraph.containsVertex(moduleVertex); + return getModulesDependencyGraph().containsVertex(moduleVertex); } /** @@ -1575,7 +1588,7 @@ protected boolean hasModuleVertex(final ModuleVertex moduleVertex) { * @param sink the imported module */ protected void addModuleEdge(final ModuleVertex source, final ModuleVertex sink) { - modulesDependencyGraph.addEdge(source, sink); + getModulesDependencyGraph().addEdge(source, sink); } /** @@ -1591,20 +1604,13 @@ protected boolean hasModulePath(final ModuleVertex source, final ModuleVertex si return false; } - ThreadPoolExecutor executor = null; - try { - executor = ConcurrencyUtil.createThreadPoolExecutor(2); - final ShortestPathAlgorithm spa = new TransitNodeRoutingShortestPath<>(modulesDependencyGraph, executor); - return spa.getPath(source, sink) != null; - } finally { - if (executor != null) { - try { - ConcurrencyUtil.shutdownExecutionService(executor); - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); - } - } + // NOTE(AR) intentionally lazily initialised! + if (modulesDependencyGraphSPExecutor == null) { + modulesDependencyGraphSPExecutor = ConcurrencyUtil.createThreadPoolExecutor(2); } + + final ShortestPathAlgorithm spa = new TransitNodeRoutingShortestPath<>(getModulesDependencyGraph(), modulesDependencyGraphSPExecutor); + return spa.getPath(source, sink) != null; } protected void setRootModules(final String namespaceURI, @Nullable final Module[] modules) { From 78c34d269fa1bfd9c2fd1c84aa98d208d4ea75ac Mon Sep 17 00:00:00 2001 From: Adam Retter Date: Mon, 31 Jul 2023 21:29:40 +0200 Subject: [PATCH 18/18] [bugfix] Allow for one ExistXqueryRegistry per BrokerPool instance --- .../java/org/exist/storage/NativeBroker.java | 2 +- .../impl/RestXqServiceRegistryManager.java | 23 +++++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/exist-core/src/main/java/org/exist/storage/NativeBroker.java b/exist-core/src/main/java/org/exist/storage/NativeBroker.java index 5aded217d84..8d81cbb8820 100644 --- a/exist-core/src/main/java/org/exist/storage/NativeBroker.java +++ b/exist-core/src/main/java/org/exist/storage/NativeBroker.java @@ -248,8 +248,8 @@ public NativeBroker(final BrokerPool pool, final Configuration config) throws EX this.indexConfiguration = (IndexSpec) config.getProperty(Indexer.PROPERTY_INDEXER_CONFIG); this.xmlSerializerPool = new XmlSerializerPool(this, config, 5); + pushSubject(pool.getSecurityManager().getSystemSubject()); try { - pushSubject(pool.getSecurityManager().getSystemSubject()); //TODO : refactor so that we can, //1) customize the different properties (file names, cache settings...) //2) have a consistent READ-ONLY behaviour (based on *mandatory* files ?) diff --git a/extensions/exquery/restxq/src/main/java/org/exist/extensions/exquery/restxq/impl/RestXqServiceRegistryManager.java b/extensions/exquery/restxq/src/main/java/org/exist/extensions/exquery/restxq/impl/RestXqServiceRegistryManager.java index f651ab7d08e..b614e9ae47c 100644 --- a/extensions/exquery/restxq/src/main/java/org/exist/extensions/exquery/restxq/impl/RestXqServiceRegistryManager.java +++ b/extensions/exquery/restxq/src/main/java/org/exist/extensions/exquery/restxq/impl/RestXqServiceRegistryManager.java @@ -32,20 +32,31 @@ import org.exquery.restxq.RestXqServiceRegistry; import org.exquery.restxq.impl.RestXqServiceRegistryImpl; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.Map; + /** * * @author Adam Retter */ public final class RestXqServiceRegistryManager { - private final static Logger LOG = LogManager.getLogger(RestXqServiceRegistryManager.class); + private static final Logger LOG = LogManager.getLogger(RestXqServiceRegistryManager.class); - private static RestXqServiceRegistryImpl registry = null; + private static Map registries = null; public static synchronized RestXqServiceRegistry getRegistry(final BrokerPool pool) { - - if(registry == null) { + + RestXqServiceRegistryImpl registry = null; + if (registries == null) { + registries = new IdentityHashMap<>(); + } else { + registry = registries.get(pool); + } + + if (registry == null) { LOG.info("Initialising RESTXQ..."); registry = new RestXqServiceRegistryImpl(); @@ -64,7 +75,9 @@ public static synchronized RestXqServiceRegistry getRegistry(final BrokerPool po //NOTE: must load registry before listening for registered events registry.addListener(persistence); - LOG.info("RESTXQ is ready."); + LOG.info("RESTXQ is ready for eXist-db BrokerPool: " + pool.getId()); + + registries.put(pool, registry); } return registry;