From e599cab5e32883ea4088880d42c5e1514be0dee4 Mon Sep 17 00:00:00 2001 From: Bert Frees Date: Thu, 19 Dec 2024 15:29:48 +0100 Subject: [PATCH] git subrepo pull framework subrepo: subdir: "framework" merged: "182e1106f8" upstream: origin: "git@github.com:daisy/pipeline-framework.git" branch: "master" commit: "e97e38124f" git-subrepo: version: "0.3.1" origin: "???" commit: "???" --- framework/.gitrepo | 4 +- framework/bom/pom.xml | 18 +- framework/calabash-adapter/pom.xml | 4 +- framework/common-utils/pom.xml | 4 +- .../main/java/org/daisy/common/file/URLs.java | 191 +++++++++-------- .../common/java/JarIsolatedClassLoader.java | 140 ++++++++++++ .../common-utils/src/test/java/URLsTest.java | 135 ++++++------ framework/framework-core/pom.xml | 5 +- .../pipeline/datatypes/DatatypeService.java | 4 +- .../org/daisy/pipeline/job/AbstractJob.java | 178 +--------------- .../pipeline/job/AbstractJobContext.java | 5 +- .../daisy/pipeline/job/JobManagerFactory.java | 31 --- .../daisy/pipeline/job/JobResourcesDir.java | 6 + .../org/daisy/pipeline/job/URIMapper.java | 58 ----- .../pipeline/job/impl/DefaultJobBuilder.java | 28 +-- .../pipeline/job/impl/DefaultJobManager.java | 16 +- .../org/daisy/pipeline/job/impl/IOHelper.java | 7 +- .../daisy/pipeline/job/impl/JobURIUtils.java | 19 -- .../org/daisy/pipeline/script/Script.java | 19 ++ .../daisy/pipeline/script/ScriptInput.java | 101 ++++++++- .../daisy/pipeline/script/ScriptRegistry.java | 23 +- .../script/ScriptServiceProvider.java | 7 + .../daisy/pipeline/script/XProcScript.java | 192 +++++++++++++++-- .../impl/DynamicResultProvider.java | 13 +- .../impl/StatusResultProvider.java | 2 +- .../script/impl/StaxXProcScriptParser.java | 30 ++- .../{job => script}/impl/XProcDecorator.java | 134 ++++++------ .../src/test/java/FrameworkCoreTest.java | 3 +- .../src/test/java/MockXProcEngine.java | 22 ++ .../pipeline/job/impl/JobURIUtilsTest.java | 78 +++---- .../org/daisy/pipeline/job/impl/Mock.java | 2 +- .../pipeline/job/impl/URIMapperTest.java | 40 ---- .../job/impl/VolatileJobStorageTest.java | 8 +- .../JobResultSetBuilderTest.java | 64 +++--- .../{job => script}/StatusPortTest.java | 17 +- .../impl/DynamicResultProviderPartsTest.java | 3 +- .../impl/DynamicResultProviderTest.java | 17 +- .../impl/XProcDecoratorTest.java | 199 ++++++++---------- .../test/resources/OSGI-INF/xproc-engine.xml | 7 + framework/framework-persistence/pom.xml | 4 +- .../persistence/impl/job/PersistentJob.java | 2 +- .../impl/job/PersistentJobContext.java | 16 +- .../impl/job/PersistentMapper.java | 28 ++- .../pipeline/persistence/impl/job/Mocks.java | 31 ++- .../impl/job/PersistentJobContextTest.java | 4 +- framework/logging-appender/pom.xml | 4 +- framework/modules-registry/pom.xml | 4 +- framework/parent/pom.xml | 8 +- framework/persistence-derby/pom.xml | 3 +- .../src/test/java/MockXProcEngine.java | 22 ++ .../src/test/java/TestBase.java | 3 +- .../test/resources/OSGI-INF/xproc-engine.xml | 7 + framework/persistence-mysql/pom.xml | 2 +- framework/pom.xml | 2 +- framework/saxon-adapter/pom.xml | 4 +- framework/utils/clientlib-java-jaxb/pom.xml | 2 +- .../pom.xml | 2 +- .../utils/xproc-engine-daisy-pipeline/pom.xml | 2 +- framework/webservice/pom.xml | 4 +- .../restlet/impl/PipelineWebService.java | 17 +- framework/woodstox-osgi-adapter/pom.xml | 2 +- framework/xproc-api/pom.xml | 2 +- 62 files changed, 1114 insertions(+), 895 deletions(-) create mode 100644 framework/common-utils/src/main/java/org/daisy/common/java/JarIsolatedClassLoader.java delete mode 100644 framework/framework-core/src/main/java/org/daisy/pipeline/job/URIMapper.java create mode 100644 framework/framework-core/src/main/java/org/daisy/pipeline/script/ScriptServiceProvider.java rename framework/framework-core/src/main/java/org/daisy/pipeline/{job => script}/impl/DynamicResultProvider.java (95%) rename framework/framework-core/src/main/java/org/daisy/pipeline/{job => script}/impl/StatusResultProvider.java (95%) rename framework/framework-core/src/main/java/org/daisy/pipeline/{job => script}/impl/XProcDecorator.java (69%) create mode 100644 framework/framework-core/src/test/java/MockXProcEngine.java delete mode 100644 framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/URIMapperTest.java rename framework/framework-core/src/test/java/org/daisy/pipeline/{job => script}/JobResultSetBuilderTest.java (77%) rename framework/framework-core/src/test/java/org/daisy/pipeline/{job => script}/StatusPortTest.java (84%) rename framework/framework-core/src/test/java/org/daisy/pipeline/{job => script}/impl/DynamicResultProviderPartsTest.java (97%) rename framework/framework-core/src/test/java/org/daisy/pipeline/{job => script}/impl/DynamicResultProviderTest.java (65%) rename framework/framework-core/src/test/java/org/daisy/pipeline/{job => script}/impl/XProcDecoratorTest.java (76%) create mode 100644 framework/framework-core/src/test/resources/OSGI-INF/xproc-engine.xml create mode 100644 framework/persistence-derby/src/test/java/MockXProcEngine.java create mode 100644 framework/persistence-derby/src/test/resources/OSGI-INF/xproc-engine.xml diff --git a/framework/.gitrepo b/framework/.gitrepo index b6eb6624fa..c2f0fb13c1 100644 --- a/framework/.gitrepo +++ b/framework/.gitrepo @@ -6,6 +6,6 @@ [subrepo] remote = git@github.com:daisy/pipeline-framework.git branch = master - commit = e9a0e7138b6a1e508fe0734413ab6b6136a0a3da - parent = 6ab8727d70af90110701604c630ed8e280a28b46 + commit = e97e38124f1c8b446f27e4d05f5b434358144f55 + parent = 0e4ac3656cd5011b8557e9dcfa4894fc3ff6d17d cmdver = 0.3.1 diff --git a/framework/bom/pom.xml b/framework/bom/pom.xml index 9e54f82a2d..e6db4c3f91 100644 --- a/framework/bom/pom.xml +++ b/framework/bom/pom.xml @@ -12,7 +12,7 @@ org.daisy.pipeline framework-bom - 1.14.21-SNAPSHOT + 1.14.22-SNAPSHOT pom DAISY Pipeline 2 :: Framework BoM @@ -45,32 +45,32 @@ org.daisy.pipeline calabash-adapter - 6.2.0 + 6.2.1 org.daisy.pipeline common-utils - 6.1.0 + 6.2.0 org.daisy.pipeline framework-core - 9.0.0 + 10.0.0 org.daisy.pipeline framework-persistence - 2.1.10 + 2.1.11 org.daisy.pipeline logging-appender - 2.1.4 + 2.1.5 org.daisy.pipeline modules-registry - 5.0.0-SNAPSHOT + 5.0.0 org.daisy.pipeline @@ -80,12 +80,12 @@ org.daisy.pipeline saxon-adapter - 5.5.0-SNAPSHOT + 5.5.0 org.daisy.pipeline webservice - 3.5.0 + 3.5.1 org.daisy.pipeline diff --git a/framework/calabash-adapter/pom.xml b/framework/calabash-adapter/pom.xml index c9383a4ba5..140c4d045b 100644 --- a/framework/calabash-adapter/pom.xml +++ b/framework/calabash-adapter/pom.xml @@ -6,12 +6,12 @@ org.daisy.pipeline framework-parent - 1.14.21-SNAPSHOT + 1.14.22-SNAPSHOT ../parent calabash-adapter - 6.2.1-SNAPSHOT + 6.2.2-SNAPSHOT bundle DAISY Pipeline 2 :: Calabash adapter for the XProc API diff --git a/framework/common-utils/pom.xml b/framework/common-utils/pom.xml index ae3d8dd4f7..0c40988850 100644 --- a/framework/common-utils/pom.xml +++ b/framework/common-utils/pom.xml @@ -4,12 +4,12 @@ org.daisy.pipeline framework-parent - 1.14.21-SNAPSHOT + 1.14.22-SNAPSHOT ../parent common-utils - 6.1.1-SNAPSHOT + 6.2.1-SNAPSHOT bundle DAISY Pipeline 2 :: Common Utilities diff --git a/framework/common-utils/src/main/java/org/daisy/common/file/URLs.java b/framework/common-utils/src/main/java/org/daisy/common/file/URLs.java index 4c96077ea4..941eae265b 100644 --- a/framework/common-utils/src/main/java/org/daisy/common/file/URLs.java +++ b/framework/common-utils/src/main/java/org/daisy/common/file/URLs.java @@ -9,7 +9,6 @@ import java.net.URL; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.BasicFileAttributeView; -import static java.nio.file.Files.walkFileTree; import java.nio.file.Files; import java.nio.file.FileSystem; import java.nio.file.FileSystemNotFoundException; @@ -154,54 +153,74 @@ public static URL getResourceFromJAR(String resource, Class context) { if (OSGiHelper.inOSGiContext()) return OSGiHelper.getResourceFromJAR(resource, context); else { - URL jarFileURL = context.getProtectionDomain().getCodeSource().getLocation(); - if ("location:local".equals(jarFileURL.toString()) || jarFileURL.toString().startsWith("mvn:")) - throw new RuntimeException("expected file URI"); - File jarFile = new File(asURI(jarFileURL)); + File jarFile = getCurrentJAR(context); logger.trace("Getting resource {} from JAR (current class: {}; JAR file: {})", resource, context, jarFile); - if (resource.startsWith("/")) - resource = resource.substring(1); - boolean requestedDirectory = resource.endsWith("/"); - if (requestedDirectory) - resource = resource.substring(0, resource.length() - 1); if (!jarFile.exists()) throw new RuntimeException("coding error"); - else if (jarFile.isDirectory()) { - File f = new File(jarFile, resource); - if (!f.exists()) - throw new RuntimeException("file does not exist"); - else if (!f.isDirectory() && requestedDirectory) - throw new RuntimeException("is not a directory"); - else - return asURL(f); } + Path p = getResourceFromJAR(resource, jarFile.toPath()); + URL u = asURL(p.toUri()); + if (isDirectory(p) && !u.toString().endsWith("/")) + u = asURL(u.toString() + "/"); + try { + p.getFileSystem().close(); + } catch (UnsupportedOperationException e) { + // default file system + } catch (IOException e) { + throw new RuntimeException(e); + } + return u; + } + } + + public static File getCurrentJAR(Class context) { + URL jarFileURL = context.getProtectionDomain().getCodeSource().getLocation(); + if ("location:local".equals(jarFileURL.toString()) || jarFileURL.toString().startsWith("mvn:")) + throw new RuntimeException("expected file URI"); + File jarFile = new File(URLs.asURI(jarFileURL)); + if (!jarFile.exists()) + throw new RuntimeException("coding error"); + return jarFile; + } + + /** + * Closing the file system of the returned path is the responsibility of the caller. + */ + public static Path getResourceFromJAR(String resource, Path jarFilePath) { + if (resource.startsWith("/")) + resource = resource.substring(1); + boolean requestedDirectory = resource.endsWith("/"); + if (requestedDirectory) + resource = resource.substring(0, resource.length() - 1); + Path f = null; { + if (isDirectory(jarFilePath)) + f = jarFilePath.resolve(resource); else { FileSystem fs; { try { - fs = FileSystems.newFileSystem(asURI("jar:" + asURI(jarFileURL)), fsEnv); } + fs = FileSystems.newFileSystem(jarFilePath, (ClassLoader)null); } catch (IOException e) { throw new RuntimeException(e); }} try { - Path f = fs.getPath("/" + resource); - boolean isDirectory = isDirectory(f); - if (!isDirectory && requestedDirectory) - throw new RuntimeException("is not a directory"); - else + f = fs.getPath("/" + resource); + } finally { + if (f == null) try { - return new URL("jar:" + jarFileURL + "!/" + resource + (isDirectory ? "/" : "")); } - catch (MalformedURLException e) { - throw new RuntimeException(e); }} - finally { - try { - fs.close(); } - catch (IOException e) { - throw new RuntimeException(e); } + fs.close(); } + catch (IOException e) { + throw new RuntimeException(e); } } } } + if (!Files.exists(f)) + throw new RuntimeException("file does not exist: " + resource); + else if (requestedDirectory && !isDirectory(f)) + throw new RuntimeException("not a directory: " + resource); + else + return f; } /** - * @param resource The (not URL-encoded) path of a directory inside the specified JAR or class directory + * @param directory The (not URL-encoded) path of a directory inside the specified JAR or class directory * @param context A class from the JAR or class directory that is to be searched for resources * @return A list of resource paths (not URL-encoded) */ @@ -213,55 +232,61 @@ public static Iterator listResourcesFromJAR(String directory, Class c if ("location:local".equals(jarFileURL.toString()) || jarFileURL.toString().startsWith("mvn:")) throw new RuntimeException("expected file URI"); File jarFile = new File(asURI(jarFileURL)); - if (directory.startsWith("/")) - directory = directory.substring(1); - if (directory.endsWith("/")) - directory = directory.substring(0, directory.length() - 1); if (!jarFile.exists()) throw new RuntimeException("coding error"); - else if (jarFile.isDirectory()) { - File d = new File(jarFile, directory); - if (!d.exists()) - throw new RuntimeException("file does not exist"); - else if (!d.isDirectory()) - throw new RuntimeException("is not a directory"); - else { - ImmutableList.Builder resources = ImmutableList.builder(); - for (File f : d.listFiles()) - resources.add(directory + "/" + f.getName() + (f.isDirectory() ? "/" : "")); - return resources.build().iterator(); }} + return listResourcesFromJAR(directory, jarFile.toPath()); + } + } + + /** + * @param directory The (not URL-encoded) path of a directory inside the specified JAR or class directory + * @param jarFilePath The location of the JAR file in a file system. Does not need to be be representable by a {@link File}. + * @return A list of resource paths (not URL-encoded) + */ + public static Iterator listResourcesFromJAR(String directory, Path jarFilePath) { + if (directory.startsWith("/")) + directory = directory.substring(1); + if (directory.endsWith("/")) + directory = directory.substring(0, directory.length() - 1); + final String _directory = directory; + final ImmutableList.Builder resources = ImmutableList.builder(); + Path d; + FileSystem fs = null; + try { + if (isDirectory(jarFilePath)) + d = jarFilePath.resolve(directory); else { - FileSystem fs; { - try { - fs = FileSystems.newFileSystem(asURI("jar:" + asURI(jarFileURL)), fsEnv); } - catch (IOException e) { - throw new RuntimeException(e); }} try { - Path d = fs.getPath("/" + directory); - if (!isDirectory(d)) - throw new RuntimeException("is not a directory"); - final ImmutableList.Builder resources = ImmutableList.builder(); - final String _directory = directory; - try { - walkFileTree(d, EnumSet.noneOf(FileVisitOption.class), 1, new SimpleFileVisitor() { - public FileVisitResult visitFile(Path f, BasicFileAttributes _) throws IOException { - String fileName = f.getFileName().toString(); - if (!fileName.endsWith("/") && isDirectory(f)) - fileName += "/"; - resources.add(_directory + "/" + fileName); - return FileVisitResult.CONTINUE; }}); } - catch (NoSuchFileException e) { - throw new RuntimeException(e); } - catch (IOException e) { - throw new RuntimeException(e); } - return resources.build().iterator(); } - finally { - try { - fs.close(); } - catch (IOException e) { - throw new RuntimeException(e); } - } + fs = FileSystems.newFileSystem(jarFilePath, (ClassLoader)null); } + catch (IOException e) { + throw new RuntimeException(e); } + d = fs.getPath("/" + directory); } + if (!Files.exists(d)) + throw new RuntimeException("file does not exist: " + directory); + else if (!isDirectory(d)) + throw new RuntimeException("not a directory: " + directory); + else { + try { + Files.walkFileTree(d, EnumSet.noneOf(FileVisitOption.class), 1, new SimpleFileVisitor() { + public FileVisitResult visitFile(Path f, BasicFileAttributes _) throws IOException { + String fileName = f.getFileName().toString(); + if (!fileName.endsWith("/") && isDirectory(f)) + fileName += "/"; + resources.add(_directory + "/" + fileName); + return FileVisitResult.CONTINUE; }}); } + catch (NoSuchFileException e) { + throw new RuntimeException(e); } + catch (IOException e) { + throw new RuntimeException(e); } + return resources.build().iterator(); + } + } finally { + if (fs != null) + try { + fs.close(); } + catch (IOException e) { + throw new RuntimeException(e); } } } @@ -271,9 +296,9 @@ public FileVisitResult visitFile(Path f, BasicFileAttributes _) throws IOExcepti private static boolean isDirectory(Path p) throws RuntimeException { BasicFileAttributes a; { try { - a = java.nio.file.Files.getFileAttributeView(p, BasicFileAttributeView.class).readAttributes(); } + a = Files.getFileAttributeView(p, BasicFileAttributeView.class).readAttributes(); } catch (NoSuchFileException e) { - throw new RuntimeException("file does not exist"); } + throw new RuntimeException("file does not exist: " + p); } catch (FileSystemNotFoundException e) { throw new RuntimeException(e); } catch (IOException e) { @@ -303,12 +328,12 @@ static URL getResourceFromJAR(String resource, Class context) { if (resource.endsWith("/")) { url = bundle.getEntry(resource.substring(0, resource.length() - 1)); if (url != null) - throw new RuntimeException("is not a directory"); } + throw new RuntimeException("not a directory: " + resource); } else { url = bundle.getEntry(resource + "/"); if (url != null) return asURL(encode(url)); } - throw new RuntimeException("file does not exist"); } + throw new RuntimeException("file does not exist: " + resource); } else { url = asURL(encode(url)); if (!url.toString().endsWith("/") @@ -329,9 +354,9 @@ static Iterator listResourcesFromJAR(String directory, Class context) Enumeration resources = bundle.getEntryPaths(directory); if (resources == null) { if (bundle.getEntry(directory.substring(0, directory.length() - 1)) != null) - throw new RuntimeException("is not a directory"); + throw new RuntimeException("not a directory: " + directory); else - throw new RuntimeException("file does not exist"); } + throw new RuntimeException("file does not exist: " + directory); } else return Iterators.forEnumeration(resources); } diff --git a/framework/common-utils/src/main/java/org/daisy/common/java/JarIsolatedClassLoader.java b/framework/common-utils/src/main/java/org/daisy/common/java/JarIsolatedClassLoader.java new file mode 100644 index 0000000000..a50835c388 --- /dev/null +++ b/framework/common-utils/src/main/java/org/daisy/common/java/JarIsolatedClassLoader.java @@ -0,0 +1,140 @@ +package org.daisy.common.java; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Enumeration; +import java.util.function.Consumer; +import java.util.HashSet; +import java.util.Iterator; +import java.util.jar.Manifest; +import java.util.Set; +import java.util.TreeSet; + +import org.daisy.common.file.URLs; + +/** + * Child-first delegating class loader that loads from JAR files embedded within the current JAR. + * + * Note that, unlike {@link URLClassLoader}, {@link JarIsolatedClassLoader} takes into account the + * "Class-Path" defined in the MANIFEST.MF file of JAR files. + */ +public abstract class JarIsolatedClassLoader extends URLClassLoader { + + private final Set classNames = new TreeSet<>(); + + protected JarIsolatedClassLoader() { + super(new URL[]{}); + } + + /** + * Add the current JAR file to the class path of this class loader. Or in other words, set up + * this class loader to load classes and resources located directly within the current JAR file. + * + * Calling this method is best to be avoided if possible, because it means that these classes + * and resources are also on the main class path, so it can not be guaranteed anymore that a + * class is not loaded by more than one class loader. (This is due to the fact that {@link + * URLClassLoader} uses parent-first delegation.) + */ + protected void addCurrentJar() { + File currentJar = URLs.getCurrentJAR(getClass()); + addURL(URLs.asURL(currentJar)); + listClasses(currentJar.toPath(), "/", classNames::add); + } + + private static void listClasses(Path jar, String dir, Consumer collect) { + Iterator resources = URLs.listResourcesFromJAR(dir, jar); + while (resources.hasNext()) { + String p = resources.next(); + if (p.endsWith(".class")) { + p = p.substring(0, p.length() - 6).replaceAll("/", "."); + collect.accept(p); + } else if (p.endsWith("/")) + listClasses(jar, p, collect); + } + } + + /** + * Add the JAR file at the specified path within the current JAR to the class path of this class + * loader. This is done in a recursive way: the MANIFEST.MF of the JAR file is read, and all the + * paths in "Class-Path" resolved relatively against {@code path} and then added to the class + * path with {@link #addJarRecursively}. + */ + protected void addJarRecursively(String path) { + addJarRecursively(URLs.getResourceFromJAR(path, URLs.getCurrentJAR(getClass()).toPath())); + } + + private final Set pathsAdded = new HashSet<>(); + + private void addJarRecursively(Path jar) { + if (pathsAdded.add(jar)) { + // copy to temporary file if needed, because URL and ClassLoader do not support nested JARs + Path realJar = jar; + if (jar.toUri().toString().matches("^jar:.+\\.jar$")) { + try { + realJar = Files.createTempFile(null, ".jar"); + Files.copy(jar, Files.newOutputStream(realJar)); + } catch (IOException e) { + throw new RuntimeException(e); + } + realJar.toFile().deleteOnExit(); + } + addURL(URLs.asURL(realJar.toUri())); + listClasses(realJar, "/", classNames::add); + // URLClassLoader does not take into account MANIFEST.MF (see + // https://bugs.java.com/bugdatabase/view_bug.do?bug_id=JDK-8168647), + // so we need to do this ourselves + Manifest manifest; { + Path f = null; + try { + f = URLs.getResourceFromJAR("/META-INF/MANIFEST.MF", jar); + } catch (RuntimeException e) { + } + if (f != null) { + try { + manifest = new Manifest(Files.newInputStream(f)); + } catch (IOException e) { + throw new RuntimeException(e); + } + String classPath = manifest.getMainAttributes().getValue("Class-Path"); + if (classPath != null) + for (String p : classPath.trim().split("\\s+")) { + Path dep = jar.getParent().resolve(p); + if (Files.exists(dep)) // ignore dependency if it doesn't exist + addJarRecursively(dep); + } + } + } + } + } + + // override because by default loadClass first delegates to the parent classloader + @Override + protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + Class loadedClass = findLoadedClass(name); + if (loadedClass == null) { + if (classNames.contains(name)) + try { + loadedClass = findClass(name); + } catch (ClassNotFoundException e) { + // should not happen, but try parent classloader + } + if (loadedClass == null) + // delegate to parent classloader + loadedClass = super.loadClass(name, resolve); + } + if (resolve) + resolveClass(loadedClass); + return loadedClass; + } + + + // override in order to avoid discovery of SPI services from outside this JAR + @Override + public Enumeration getResources(String name) throws IOException { + return findResources(name); + } +} diff --git a/framework/common-utils/src/test/java/URLsTest.java b/framework/common-utils/src/test/java/URLsTest.java index e26547195e..083df7de8e 100644 --- a/framework/common-utils/src/test/java/URLsTest.java +++ b/framework/common-utils/src/test/java/URLsTest.java @@ -16,10 +16,7 @@ import org.hamcrest.Matcher; import org.hamcrest.TypeSafeMatcher; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.fail; +import org.junit.Assert; import org.junit.Test; import org.ops4j.pax.exam.util.PathUtils; @@ -55,34 +52,34 @@ public void testGetResourceFromJAR() throws Exception { Class context = URLsTest.class; try { URLs.getResourceFromJAR("/unexisting", context); - fail("expected RuntimeException"); } + Assert.fail("expected RuntimeException"); } catch (RuntimeException e) { - assertEquals("file does not exist", e.getMessage()); } + Assert.assertTrue(e.getMessage().startsWith("file does not exist:")); } try { URLs.getResourceFromJAR("/dir/file1/", context); - fail("expected RuntimeException"); } + Assert.fail("expected RuntimeException"); } catch (RuntimeException e) { - assertEquals("is not a directory", e.getMessage()); } + Assert.assertTrue(e.getMessage().startsWith("not a directory:")); } { String actual = URLs.getResourceFromJAR("/dir/file1", context).toString(); if (OSGiHelper.inOSGiContext()) - assertThat(actual, matchesPattern("^bundle://.*/dir/file1$")); + Assert.assertThat(actual, matchesPattern("^bundle://.*/dir/file1$")); else - assertEquals(testClassesDir + "dir/file1", actual); + Assert.assertEquals(testClassesDir + "dir/file1", actual); } { String actual = URLs.getResourceFromJAR("/dir/file 2", context).toString(); if (OSGiHelper.inOSGiContext()) - assertThat(actual, matchesPattern("^bundle://.*/dir/file%202$")); + Assert.assertThat(actual, matchesPattern("^bundle://.*/dir/file%202$")); else - assertEquals(testClassesDir + "dir/file%202", actual); + Assert.assertEquals(testClassesDir + "dir/file%202", actual); } { String actual = URLs.getResourceFromJAR("/dir/subdir", context).toString(); if (OSGiHelper.inOSGiContext()) - assertThat(actual, matchesPattern("^bundle://.*/dir/subdir/$")); + Assert.assertThat(actual, matchesPattern("^bundle://.*/dir/subdir/$")); else - assertEquals(testClassesDir + "dir/subdir/", actual); + Assert.assertEquals(testClassesDir + "dir/subdir/", actual); } } { @@ -90,33 +87,33 @@ public void testGetResourceFromJAR() throws Exception { Class context = URLs.class; try { URLs.getResourceFromJAR("/unexisting", context); - fail("expected RuntimeException"); } + Assert.fail("expected RuntimeException"); } catch (RuntimeException e) { - assertEquals("file does not exist", e.getMessage()); } + Assert.assertTrue(e.getMessage().startsWith("file does not exist:")); } try { URLs.getResourceFromJAR("/org/daisy/common/file/URLs.class/", context); - fail("expected RuntimeException"); } + Assert.fail("expected RuntimeException"); } catch (RuntimeException e) { - assertEquals("is not a directory", e.getMessage()); } + Assert.assertTrue(e.getMessage().startsWith("not a directory:")); } { String actual = URLs.getResourceFromJAR("/org/daisy/common/file/URLs.class", context).toString(); if (OSGiHelper.inOSGiContext()) - assertThat(actual, matchesPattern("^bundle://.*/org/daisy/common/file/URLs\\.class$")); + Assert.assertThat(actual, matchesPattern("^bundle://.*/org/daisy/common/file/URLs\\.class$")); else try { - assertEquals(classesDir + "org/daisy/common/file/URLs.class", actual); } + Assert.assertEquals(classesDir + "org/daisy/common/file/URLs.class", actual); } catch (AssertionError e) { - assertEquals("jar:" + jarFile + "!/org/daisy/common/file/URLs.class", actual); } + Assert.assertEquals("jar:" + jarFile + "!/org/daisy/common/file/URLs.class", actual); } } { String actual = URLs.getResourceFromJAR("/org/daisy/common/file", context).toString(); if (OSGiHelper.inOSGiContext()) - assertThat(actual, matchesPattern("^bundle://.*/org/daisy/common/file/$")); + Assert.assertThat(actual, matchesPattern("^bundle://.*/org/daisy/common/file/$")); else try { - assertEquals(classesDir + "org/daisy/common/file/", actual); } + Assert.assertEquals(classesDir + "org/daisy/common/file/", actual); } catch (AssertionError e) { - assertEquals("jar:" + jarFile + "!/org/daisy/common/file/", actual); } + Assert.assertEquals("jar:" + jarFile + "!/org/daisy/common/file/", actual); } } } { @@ -124,27 +121,27 @@ public void testGetResourceFromJAR() throws Exception { Class context = com.google.common.base.Function.class; try { URLs.getResourceFromJAR("/unexisting", context); - fail("expected RuntimeException"); } + Assert.fail("expected RuntimeException"); } catch (RuntimeException e) { - assertEquals("file does not exist", e.getMessage()); } + Assert.assertTrue(e.getMessage().startsWith("file does not exist:")); } try { URLs.getResourceFromJAR("/com/google/common/base/Function.class/", context); - fail("expected RuntimeException"); } + Assert.fail("expected RuntimeException"); } catch (RuntimeException e) { - assertEquals("is not a directory", e.getMessage()); } + Assert.assertTrue(e.getMessage().startsWith("not a directory:")); } { String actual = URLs.getResourceFromJAR("/com/google/common/base/Function.class", context).toString(); if (OSGiHelper.inOSGiContext()) - assertThat(actual, matchesPattern("^bundle://.*/com/google/common/base/Function\\.class$")); + Assert.assertThat(actual, matchesPattern("^bundle://.*/com/google/common/base/Function\\.class$")); else - assertThat(actual, matchesPattern("^jar:file:.*!/com/google/common/base/Function\\.class$")); + Assert.assertThat(actual, matchesPattern("^jar:file:.*!/com/google/common/base/Function\\.class$")); } { String actual = URLs.getResourceFromJAR("/com/google/common/base", context).toString(); if (OSGiHelper.inOSGiContext()) - assertThat(actual, matchesPattern("^bundle://.*/com/google/common/base/$")); + Assert.assertThat(actual, matchesPattern("^bundle://.*/com/google/common/base/$")); else - assertThat(actual, matchesPattern("^jar:file:.*!/com/google/common/base/$")); + Assert.assertThat(actual, matchesPattern("^jar:file:.*!/com/google/common/base/$")); } } } @@ -156,20 +153,20 @@ public void testListResourcesFromJAR() { Class context = URLsTest.class; try { URLs.listResourcesFromJAR("/unexisting", context); - fail("expected RuntimeException"); } + Assert.fail("expected RuntimeException"); } catch (RuntimeException e) { - assertEquals("file does not exist", e.getMessage()); } + Assert.assertTrue(e.getMessage().startsWith("file does not exist:")); } try { URLs.listResourcesFromJAR("/dir/file1", context); - fail("expected RuntimeException"); } + Assert.fail("expected RuntimeException"); } catch (RuntimeException e) { - assertEquals("is not a directory", e.getMessage()); } + Assert.assertTrue( e.getMessage().startsWith("not a directory:")); } { Iterator i = sort(URLs.listResourcesFromJAR("/dir", context)); - assertEquals("dir/file 2", i.next()); - assertEquals("dir/file1", i.next()); - assertEquals("dir/subdir/", i.next()); - assertFalse(i.hasNext()); + Assert.assertEquals("dir/file 2", i.next()); + Assert.assertEquals("dir/file1", i.next()); + Assert.assertEquals("dir/subdir/", i.next()); + Assert.assertFalse(i.hasNext()); } } { @@ -177,20 +174,20 @@ public void testListResourcesFromJAR() { Class context = URLs.class; try { URLs.listResourcesFromJAR("/unexisting", context); - fail("expected RuntimeException"); } + Assert.fail("expected RuntimeException"); } catch (RuntimeException e) { - assertEquals("file does not exist", e.getMessage()); } + Assert.assertTrue(e.getMessage().startsWith("file does not exist:")); } try { URLs.listResourcesFromJAR("/org/daisy/common/file/URLs.class", context); - fail("expected RuntimeException"); } + Assert.fail("expected RuntimeException"); } catch (RuntimeException e) { - assertEquals("is not a directory", e.getMessage()); } + Assert.assertTrue( e.getMessage().startsWith("not a directory:")); } { Iterator i = sort(URLs.listResourcesFromJAR("/org/daisy/common/file", context)); - assertEquals("org/daisy/common/file/URLs$1.class", i.next()); - assertEquals("org/daisy/common/file/URLs$OSGiHelper.class", i.next()); - assertEquals("org/daisy/common/file/URLs.class", i.next()); - assertFalse(i.hasNext()); + Assert.assertEquals("org/daisy/common/file/URLs$1.class", i.next()); + Assert.assertEquals("org/daisy/common/file/URLs$OSGiHelper.class", i.next()); + Assert.assertEquals("org/daisy/common/file/URLs.class", i.next()); + Assert.assertFalse(i.hasNext()); } } { @@ -198,33 +195,33 @@ public void testListResourcesFromJAR() { Class context = com.google.common.base.Function.class; try { URLs.listResourcesFromJAR("/unexisting", context); - fail("expected RuntimeException"); } + Assert.fail("expected RuntimeException"); } catch (RuntimeException e) { - assertEquals("file does not exist", e.getMessage()); } + Assert.assertTrue(e.getMessage().startsWith("file does not exist:")); } try { URLs.listResourcesFromJAR("/com/google/common/base/Function.class", context); - fail("expected RuntimeException"); } + Assert.fail("expected RuntimeException"); } catch (RuntimeException e) { - assertEquals("is not a directory", e.getMessage()); } + Assert.assertTrue( e.getMessage().startsWith("not a directory:")); } { Iterator i = sort(URLs.listResourcesFromJAR("/com/google/common", context)); - assertEquals("com/google/common/annotations/", i.next()); - assertEquals("com/google/common/base/", i.next()); - assertEquals("com/google/common/cache/", i.next()); - assertEquals("com/google/common/collect/", i.next()); - assertEquals("com/google/common/escape/", i.next()); - assertEquals("com/google/common/eventbus/", i.next()); - assertEquals("com/google/common/graph/", i.next()); - assertEquals("com/google/common/hash/", i.next()); - assertEquals("com/google/common/html/", i.next()); - assertEquals("com/google/common/io/", i.next()); - assertEquals("com/google/common/math/", i.next()); - assertEquals("com/google/common/net/", i.next()); - assertEquals("com/google/common/primitives/", i.next()); - assertEquals("com/google/common/reflect/", i.next()); - assertEquals("com/google/common/util/", i.next()); - assertEquals("com/google/common/xml/", i.next()); - assertFalse(i.hasNext()); + Assert.assertEquals("com/google/common/annotations/", i.next()); + Assert.assertEquals("com/google/common/base/", i.next()); + Assert.assertEquals("com/google/common/cache/", i.next()); + Assert.assertEquals("com/google/common/collect/", i.next()); + Assert.assertEquals("com/google/common/escape/", i.next()); + Assert.assertEquals("com/google/common/eventbus/", i.next()); + Assert.assertEquals("com/google/common/graph/", i.next()); + Assert.assertEquals("com/google/common/hash/", i.next()); + Assert.assertEquals("com/google/common/html/", i.next()); + Assert.assertEquals("com/google/common/io/", i.next()); + Assert.assertEquals("com/google/common/math/", i.next()); + Assert.assertEquals("com/google/common/net/", i.next()); + Assert.assertEquals("com/google/common/primitives/", i.next()); + Assert.assertEquals("com/google/common/reflect/", i.next()); + Assert.assertEquals("com/google/common/util/", i.next()); + Assert.assertEquals("com/google/common/xml/", i.next()); + Assert.assertFalse(i.hasNext()); } } } diff --git a/framework/framework-core/pom.xml b/framework/framework-core/pom.xml index 65cb15d917..ec1b2c23f7 100644 --- a/framework/framework-core/pom.xml +++ b/framework/framework-core/pom.xml @@ -4,11 +4,11 @@ org.daisy.pipeline framework-parent - 1.14.21-SNAPSHOT + 1.14.22-SNAPSHOT ../parent framework-core - 9.0.1-SNAPSHOT + 10.0.1-SNAPSHOT bundle DAISY Pipeline 2 :: Framework Core @@ -180,6 +180,7 @@ + MockXProcEngine, org.daisy.pipeline.datatypes.impl.Datatype_dtbook_mydatatype, org.daisy.pipeline.script.impl.XProcScript_unit_test_script diff --git a/framework/framework-core/src/main/java/org/daisy/pipeline/datatypes/DatatypeService.java b/framework/framework-core/src/main/java/org/daisy/pipeline/datatypes/DatatypeService.java index b7c44b5303..a0e5280ba4 100644 --- a/framework/framework-core/src/main/java/org/daisy/pipeline/datatypes/DatatypeService.java +++ b/framework/framework-core/src/main/java/org/daisy/pipeline/datatypes/DatatypeService.java @@ -70,7 +70,9 @@ public Document asDocument() throws Exception { } public ValidationResult validate(String content) { try { - Integer.parseInt(content); + int i = Integer.parseInt(content); + if (i < 0) + return notValid("Negative integer: " + content); return valid(); } catch (NumberFormatException e) { return notValid("Not an integer: " + content); diff --git a/framework/framework-core/src/main/java/org/daisy/pipeline/job/AbstractJob.java b/framework/framework-core/src/main/java/org/daisy/pipeline/job/AbstractJob.java index 94fb3f12d0..4c1ee3dc89 100644 --- a/framework/framework-core/src/main/java/org/daisy/pipeline/job/AbstractJob.java +++ b/framework/framework-core/src/main/java/org/daisy/pipeline/job/AbstractJob.java @@ -1,39 +1,16 @@ package org.daisy.pipeline.job; import java.io.File; -import java.io.InputStream; -import java.io.IOException; import java.net.URI; -import java.util.List; -import java.util.Optional; import java.util.function.Consumer; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.transform.Result; - -import com.google.common.base.Supplier; - import org.daisy.common.messaging.Message.Level; import org.daisy.common.messaging.MessageBuilder; import org.daisy.common.priority.Priority; -import org.daisy.common.xml.DocumentBuilder; -import org.daisy.common.xproc.XProcEngine; import org.daisy.common.xproc.XProcErrorException; -import org.daisy.common.xproc.XProcInput; -import org.daisy.common.xproc.XProcOutput; -import org.daisy.common.xproc.XProcPipeline; -import org.daisy.common.xproc.XProcResult; import org.daisy.pipeline.clients.Client; -import org.daisy.pipeline.job.impl.DynamicResultProvider; -import org.daisy.pipeline.job.impl.IOHelper; import org.daisy.pipeline.job.impl.JobURIUtils; -import org.daisy.pipeline.job.impl.StatusResultProvider; -import org.daisy.pipeline.job.impl.XProcDecorator; import org.daisy.pipeline.script.Script; -import org.daisy.pipeline.script.ScriptPort; -import org.daisy.pipeline.script.XProcOptionMetadata; -import org.daisy.pipeline.script.XProcScript; -import org.daisy.pipeline.script.XProcScript.XProcScriptOption; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,8 +18,6 @@ import org.slf4j.MarkerFactory; import org.slf4j.MDC; -import org.w3c.dom.Document; - /** * The Class Job defines the execution unit. */ @@ -53,19 +28,15 @@ public abstract class AbstractJob implements Job { private volatile Status status = Status.IDLE; protected Priority priority; // used in PersistentJob protected AbstractJobContext ctxt; // used in PersistentJob - private final XProcEngine xprocEngine; - private final List inputParsers; private final boolean managed; private boolean closed = false; /** * @param managed Whether the Job will be managed by a {@link JobManager}. */ - protected AbstractJob(AbstractJobContext ctxt, Priority priority, XProcEngine xprocEngine, List inputParsers, boolean managed) { + protected AbstractJob(AbstractJobContext ctxt, Priority priority, boolean managed) { this.ctxt = ctxt; this.priority = priority != null ? priority : Priority.MEDIUM; - this.xprocEngine = xprocEngine; - this.inputParsers = inputParsers; this.managed = managed; } @@ -76,7 +47,7 @@ protected AbstractJob(AbstractJob job) { // for use in PersistentJob protected AbstractJob(AbstractJobContext ctxt, AbstractJob job) { - this(ctxt, job.priority, job.xprocEngine, job.inputParsers, job.managed); + this(ctxt, job.priority, job.managed); } @Override @@ -176,16 +147,13 @@ public synchronized void managedRun() { else run = true; Script script = ctxt.getScript(); - if (!(script instanceof XProcScript)) - throw new IllegalStateException("Don't know how to run script: " + script); // used in JobLogFileAppender MDC.put("jobid", getId().toString()); logger.info("Starting to log to job's log file: " + getId().toString()); changeStatus(Status.RUNNING); - XProcPipeline pipeline = null; - if (ctxt.messageBus == null || ctxt.properties == null || xprocEngine == null) + if (ctxt.messageBus == null || ctxt.properties == null) // This means we've tried to execute a PersistentJob that was read from the // database. This should not happen because upon creation jobs are // immediately submitted to DefaultJobExecutionService, which keeps them in @@ -193,18 +161,11 @@ public synchronized void managedRun() { // are not added to the execution queue upon launching Pipeline. throw new IllegalStateException(); try { - pipeline = xprocEngine.load(((XProcScript)script).getURI()); - XProcDecorator decorator = XProcDecorator.from((XProcScript)script, ctxt.uriMapper, inputParsers); - XProcInput input = decorator.decorate(ctxt.input); - XProcResult result = pipeline.run(input, () -> ctxt.messageBus, ctxt.properties); - XProcOutput output = decorator.decorate(new XProcOutput.Builder().build()); - result.writeTo(output); // writes to files and/or streams specified in output - ctxt.results = buildResultSet((XProcScript)script, input, output, ctxt.uriMapper, newResultSetBuilder(script)); + JobResultSet.Builder resultBuilder = newResultSetBuilder(script); + Job.Status status = script.run(ctxt.input, ctxt.properties, ctxt.messageBus, resultBuilder, ctxt.resultDir); + ctxt.results = resultBuilder.build(); onResultsChanged(); - if (checkStatusPort((XProcScript)script, output)) - changeStatus(Status.SUCCESS); - else - changeStatus(Status.FAIL); + changeStatus(status); } catch (OutOfMemoryError e) { changeStatus( Status.ERROR); ctxt.messageBus.append(new MessageBuilder() @@ -264,129 +225,4 @@ public boolean equals(Object object) { protected JobResultSet.Builder newResultSetBuilder(Script script) { return new JobResultSet.Builder(script); } - - // package private for unit tests - static JobResultSet buildResultSet(XProcScript script, XProcInput inputs, XProcOutput outputs, URIMapper mapper) - throws IOException { - return buildResultSet(script, inputs, outputs, mapper, new JobResultSet.Builder(script)); - } - - private static JobResultSet buildResultSet(XProcScript script, XProcInput inputs, XProcOutput outputs, - URIMapper mapper, JobResultSet.Builder builder) throws IOException { - - // iterate over output ports - for (ScriptPort port : script.getOutputPorts()) { - String mediaType = port.getMediaType(); - - // check if it is implemented as an output option - XProcScriptOption option = script.getResultOption(port.getName()); - if (option != null) { - if (inputs.getOptions().get(option.getXProcOptionName()) == null) - // option was not set - continue; - if (XProcOptionMetadata.ANY_FILE_URI.equals(option.getType().getId())) { - URI path; { - Object val = inputs.getOptions().get(option.getXProcOptionName()); - try { - path = URI.create((String)val); - } catch (ClassCastException e) { - throw new RuntimeException( - "Expected string value for option " + option.getName() - + " but got: " + val.getClass()); - } - } - File f = new File(path); - if (f.exists()) { - builder = builder.addResult(port.getName(), - mapper.unmapOutput(path).toString(), - f, - mediaType); - } - } else if (XProcOptionMetadata.ANY_DIR_URI.equals(option.getType().getId())) { - String dir; { - Object val = inputs.getOptions().get(option.getXProcOptionName()); - try { - dir = (String)val; - } catch (ClassCastException e) { - throw new RuntimeException( - "Expected string value for option " + option.getName() - + " but got: " + val.getClass()); - } - } - // scan the directory to get all files inside and write them to the XProcOutput - for (File f : IOHelper.treeFileList(new File(URI.create(dir)))) { - URI path = f.toURI(); - builder = builder.addResult(port.getName(), - mapper.unmapOutput(path).toString(), - f, - mediaType); - } - } - } else { - Supplier resultProvider = outputs.getResultProvider(port.getName()); - if (resultProvider == null) - // XProcDecorator makes sure this can not happen - continue; - if (!(resultProvider instanceof DynamicResultProvider)) - // XProcDecorator makes sure this can not happen - throw new RuntimeException( - "Result supplier is expected to be a DynamicResultProvider but got: " + resultProvider); - for (Result result : ((DynamicResultProvider)resultProvider).providedResults()) { - String sysId = result.getSystemId(); - if (sysId == null) - // XProcDecorator makes sure this can not happen - throw new RuntimeException( - "Result is expected to be a DynamicResult but got: " + result); - URI path = URI.create(sysId); - builder = builder.addResult(port.getName(), - mapper.unmapOutput(path).toString(), - new File(path), - mediaType); - } - } - } - return builder.build(); - } - - /** - * Check the validation status from the status port and get it's value. - */ - // package private for unit tests - static boolean checkStatusPort(XProcScript script, XProcOutput outputs) { - Optional statusPort = script.getStatusPort(); - if (statusPort.isPresent()) { - Supplier provider = outputs.getResultProvider(statusPort.get().getName()); - if (provider != null && provider instanceof StatusResultProvider) { // should always be true - boolean ok = true; - for (InputStream status : ((StatusResultProvider)provider).read()) { - ok &= processStatus(status); - } - return ok; - } - } - return true; - } - - /** - * Read the XML file to check that validation status is equal to "ok". - */ - // package private for unit tests - static boolean processStatus(InputStream status) { - // check the contents of the xml and check if result is "ok" - // - try { - DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); - docBuilderFactory.setNamespaceAware(true); - javax.xml.parsers.DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder(); - Document doc = docBuilder.parse(status); - String result = doc.getDocumentElement().getAttribute("result"); - if (result == null || result.isEmpty()) { - throw new RuntimeException("No result attribute was found in the status port"); - } - return result.equalsIgnoreCase("ok"); - - } catch (Exception e) { - throw new RuntimeException("Error processing status file", e); - } - } } diff --git a/framework/framework-core/src/main/java/org/daisy/pipeline/job/AbstractJobContext.java b/framework/framework-core/src/main/java/org/daisy/pipeline/job/AbstractJobContext.java index 558e872f57..fc4561180f 100644 --- a/framework/framework-core/src/main/java/org/daisy/pipeline/job/AbstractJobContext.java +++ b/framework/framework-core/src/main/java/org/daisy/pipeline/job/AbstractJobContext.java @@ -1,5 +1,6 @@ package org.daisy.pipeline.job; +import java.io.File; import java.net.URI; import java.util.function.Consumer; import java.util.List; @@ -23,7 +24,7 @@ public abstract class AbstractJobContext { // accessed in DefaultJobBuilder, PersistentJobContext and AbstractJob protected ScriptInput input; - protected URIMapper uriMapper; + protected File resultDir; protected JobResultSet results; // accessed in DefaultJobBuilder and AbstractJob @@ -46,7 +47,7 @@ protected AbstractJobContext(AbstractJobContext from) { this.results = from.results; this.script = from.script; this.input = from.input; - this.uriMapper = from.uriMapper; + this.resultDir = from.resultDir; this.monitor = from.monitor; this.messageBus = from.messageBus; this.statusListeners = from.statusListeners; diff --git a/framework/framework-core/src/main/java/org/daisy/pipeline/job/JobManagerFactory.java b/framework/framework-core/src/main/java/org/daisy/pipeline/job/JobManagerFactory.java index a6ae68eaa0..b20b3803d2 100644 --- a/framework/framework-core/src/main/java/org/daisy/pipeline/job/JobManagerFactory.java +++ b/framework/framework-core/src/main/java/org/daisy/pipeline/job/JobManagerFactory.java @@ -6,7 +6,6 @@ import org.daisy.common.properties.Properties; import org.daisy.common.properties.Properties.Property; import org.daisy.common.xml.DocumentBuilder; -import org.daisy.common.xproc.XProcEngine; import org.daisy.pipeline.clients.Client; import org.daisy.pipeline.job.impl.DefaultJobBuilder; import org.daisy.pipeline.job.impl.DefaultJobExecutionService; @@ -49,8 +48,6 @@ public class JobManagerFactory implements JobFactory { @Override public JobFactory.JobBuilder newJob(BoundScript boundScript) { return new DefaultJobBuilder(JobMonitorFactory.LIVE_MONITOR_FACTORY, - xprocEngine, - inputParsers, null, boundScript, false, @@ -81,8 +78,6 @@ public JobManager createFor(JobBatchId batchId) { public JobManager createFor(Client client) { return new DefaultJobManager(client, monitorFactory, - xprocEngine, - inputParsers, storage.filterBy(client), executionService.filterBy(client), logLevelProperty); @@ -99,8 +94,6 @@ public JobManager createFor(Client client) { public JobManager createFor(Client client, JobBatchId batchId) { return new DefaultJobManager(client, monitorFactory, - xprocEngine, - inputParsers, storage.filterBy(client).filterBy(batchId), executionService.filterBy(client), logLevelProperty); @@ -108,8 +101,6 @@ public JobManager createFor(Client client, JobBatchId batchId) { private JobStorage storage; private JobMonitorFactory monitorFactory; - private XProcEngine xprocEngine; - private final List inputParsers = new ArrayList<>(); private JobExecutionService executionService; @Activate @@ -138,26 +129,4 @@ protected void init() { protected void setJobStorage(JobStorage storage) { this.storage = storage; } - - @Reference( - name = "xproc-engine", - unbind = "-", - service = XProcEngine.class, - cardinality = ReferenceCardinality.MANDATORY, - policy = ReferencePolicy.STATIC - ) - protected void setXProcEngine(XProcEngine xprocEngine) { - this.xprocEngine = xprocEngine; - } - - @Reference( - name = "input-parser", - unbind = "-", - service = DocumentBuilder.class, - cardinality = ReferenceCardinality.MANDATORY, - policy = ReferencePolicy.STATIC - ) - protected void addInputParser(DocumentBuilder parser) { - inputParsers.add(parser); - } } diff --git a/framework/framework-core/src/main/java/org/daisy/pipeline/job/JobResourcesDir.java b/framework/framework-core/src/main/java/org/daisy/pipeline/job/JobResourcesDir.java index 3359f50e2b..3b2a4e8954 100644 --- a/framework/framework-core/src/main/java/org/daisy/pipeline/job/JobResourcesDir.java +++ b/framework/framework-core/src/main/java/org/daisy/pipeline/job/JobResourcesDir.java @@ -16,9 +16,11 @@ */ public final class JobResourcesDir implements JobResources { + private final File baseDir; private final Map> resources; public JobResourcesDir(File baseDir) { + this.baseDir = baseDir; ImmutableMap.Builder> mapBuilder = ImmutableMap.builder(); try { int baselen = baseDir.getCanonicalPath().length(); @@ -49,4 +51,8 @@ public Supplier getResource(String name) { public Iterable getNames() { return resources.keySet(); } + + public File getBaseDir() { + return baseDir; + } } diff --git a/framework/framework-core/src/main/java/org/daisy/pipeline/job/URIMapper.java b/framework/framework-core/src/main/java/org/daisy/pipeline/job/URIMapper.java deleted file mode 100644 index 0d64c0f730..0000000000 --- a/framework/framework-core/src/main/java/org/daisy/pipeline/job/URIMapper.java +++ /dev/null @@ -1,58 +0,0 @@ -package org.daisy.pipeline.job; - -import java.net.URI; - -public final class URIMapper{ - private final URI inputBase; - private final URI outputBase; - - public URIMapper(URI inputBase,URI outputBase){ - this.inputBase=inputBase; - this.outputBase=outputBase; - } - public URI mapInput(URI relative){ - return inputBase.resolve(relative); - } - public URI mapOutput(URI relative){ - return outputBase.resolve(relative); - } - public URI unmapInput(URI absolute){ - return inputBase.relativize(absolute); - } - public URI unmapOutput(URI absolute){ - return outputBase.relativize(absolute); - } - - /** - * Gets the inputBase for this instance. - * - * @return The inputBase. - */ - public URI getInputBase() { - return this.inputBase; - } - - /** - * Gets the outputBase for this instance. - * - * @return The outputBase. - */ - public URI getOutputBase() { - return this.outputBase; - } - - @Override - public int hashCode() { - return this.inputBase.hashCode()+this.outputBase.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if(obj==null || ! (obj instanceof URIMapper)) - return false; - URIMapper other=(URIMapper)obj; - return this.inputBase.equals(other.inputBase)&&this.outputBase.equals(other.outputBase); - - } - -} diff --git a/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/DefaultJobBuilder.java b/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/DefaultJobBuilder.java index 768de0ffb5..0021d7f7af 100644 --- a/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/DefaultJobBuilder.java +++ b/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/DefaultJobBuilder.java @@ -1,9 +1,9 @@ package org.daisy.pipeline.job.impl; +import java.io.File; import java.io.IOException; import java.util.LinkedList; import java.util.function.Consumer; -import java.util.List; import com.google.common.base.Optional; @@ -12,8 +12,6 @@ import org.daisy.common.properties.Properties; import org.daisy.common.properties.Properties.Property; import org.daisy.common.priority.Priority; -import org.daisy.common.xml.DocumentBuilder; -import org.daisy.common.xproc.XProcEngine; import org.daisy.pipeline.clients.Client; import org.daisy.pipeline.job.Job; import org.daisy.pipeline.job.AbstractJob; @@ -22,7 +20,6 @@ import org.daisy.pipeline.job.JobIdFactory; import org.daisy.pipeline.job.JobManager; import org.daisy.pipeline.job.JobMonitorFactory; -import org.daisy.pipeline.job.JobResources; import org.daisy.pipeline.job.JobResultSet; import org.daisy.pipeline.job.StatusNotifier; import org.daisy.pipeline.script.BoundScript; @@ -33,8 +30,6 @@ public class DefaultJobBuilder implements JobManager.JobBuilder { private final JobMonitorFactory monitorFactory; - private final XProcEngine xprocEngine; - private final List inputParsers; private final Client client; private final BoundScript boundScript; private final boolean managed; @@ -48,15 +43,11 @@ public class DefaultJobBuilder implements JobManager.JobBuilder { * @param managed Whether the Job will be managed by a JobManager. */ public DefaultJobBuilder(JobMonitorFactory monitorFactory, - XProcEngine xprocEngine, - List inputParsers, Client client, BoundScript boundScript, boolean managed, Property logLevelProperty) { this.monitorFactory = monitorFactory; - this.xprocEngine = xprocEngine; - this.inputParsers = inputParsers; this.client = client; this.boundScript = boundScript; this.managed = managed; @@ -109,14 +100,13 @@ public Optional build() { results = JobResultSet.EMPTY; script = boundScript.getScript(); input = boundScript.getInput(); - JobResources resources = input.getResources(); - uriMapper = resources != null - ? JobURIUtils.newURIMapper(id.toString()) - : JobURIUtils.newOutputURIMapper(id.toString()); - if (resources != null) { - logger.debug("Storing the resource collection"); // because not persisted - IOHelper.dump(resources, uriMapper); - } + resultDir = IOHelper.makeDirs(JobURIUtils.getJobOutputDir(id.toString())); + logger.debug("Storing inputs"); // because we want to be able to download the + // context files of persisted jobs, even though + // JobResources are not persisted and only the + // systemId of ScriptInput are persisted + File contextDir = IOHelper.makeDirs(JobURIUtils.getJobContextDir(id.toString())); + input = input.storeToDisk(contextDir); properties = Properties.getSnapshot(); Level messagesThreshold; { try { @@ -136,7 +126,7 @@ public void unlisten(Consumer listener) { statusListeners.remove(listener); }}}; monitor = monitorFactory.newJobMonitor(id, messageBus, statusNotifier); }}; - AbstractJob job = new AbstractJob(ctxt, priority, xprocEngine, inputParsers, managed) {}; + AbstractJob job = new AbstractJob(ctxt, priority, managed) {}; if (!managed && closeOnExit) job = new VolatileJob(job); if (!managed) diff --git a/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/DefaultJobManager.java b/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/DefaultJobManager.java index ee02b14358..ac127e9c74 100644 --- a/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/DefaultJobManager.java +++ b/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/DefaultJobManager.java @@ -3,8 +3,6 @@ import java.util.List; import org.daisy.common.properties.Properties.Property; -import org.daisy.common.xml.DocumentBuilder; -import org.daisy.common.xproc.XProcEngine; import org.daisy.pipeline.clients.Client; import org.daisy.pipeline.job.AbstractJob; import org.daisy.pipeline.job.Job; @@ -16,13 +14,13 @@ import org.daisy.pipeline.job.JobStorage; import org.daisy.pipeline.script.BoundScript; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.google.common.base.Optional; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * DefaultJobManager allows to manage the jobs submitted to the daisy pipeline 2 */ @@ -33,8 +31,6 @@ public class DefaultJobManager implements JobManager { private final Client client; private final JobMonitorFactory monitorFactory; - private final XProcEngine xprocEngine; - private final List inputParsers; private final JobStorage storage; private final JobExecutionService executionService; private final Property logLevelProperty; @@ -45,15 +41,11 @@ public class DefaultJobManager implements JobManager { */ public DefaultJobManager(Client client, JobMonitorFactory monitorFactory, - XProcEngine xprocEngine, - List inputParsers, JobStorage storage, JobExecutionService executionService, Property logLevelProperty) { this.client = client; this.monitorFactory = monitorFactory; - this.xprocEngine = xprocEngine; - this.inputParsers = inputParsers; this.storage = storage; this.executionService = executionService; this.logLevelProperty = logLevelProperty; @@ -109,8 +101,6 @@ public Iterable deleteAll() { @Override public JobManager.JobBuilder newJob(BoundScript boundScript) { return new DefaultJobBuilder(monitorFactory, - xprocEngine, - inputParsers, client, boundScript, true, diff --git a/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/IOHelper.java b/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/IOHelper.java index 7edbc645b5..2507681073 100644 --- a/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/IOHelper.java +++ b/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/IOHelper.java @@ -11,7 +11,6 @@ import java.util.List; import org.daisy.pipeline.job.JobResources; -import org.daisy.pipeline.job.URIMapper; /** * IO related utities. @@ -71,11 +70,11 @@ public static void dump(InputStream is,URI base,URI path) throws IOException { * @param context the context * @throws IOException Signals that an I/O exception has occurred. */ - public static void dump(JobResources resources,URIMapper mapper) throws IOException { + public static void dump(JobResources resources, URI base) throws IOException { for (String path : resources.getNames()) { try { - IOHelper.dump(resources.getResource(path).get(),mapper.getInputBase() - , new URI(null, null, path.replace("\\", "/"), null, null)); + IOHelper.dump(resources.getResource(path).get(), base, + new URI(null, null, path.replace("\\", "/"), null, null)); } catch (URISyntaxException e) { throw new RuntimeException("Resource path could not be converted to URI: " + path, e); } diff --git a/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/JobURIUtils.java b/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/JobURIUtils.java index 29c261bbcb..7748878e08 100644 --- a/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/JobURIUtils.java +++ b/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/JobURIUtils.java @@ -6,7 +6,6 @@ import java.nio.file.Files; import org.daisy.common.properties.Properties; -import org.daisy.pipeline.job.URIMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -18,24 +17,6 @@ public class JobURIUtils { private final static Logger logger = LoggerFactory.getLogger(JobURIUtils.class); - /** - * Returns an idle URI mapping, in case we are expecting absolute URIs all the time. - */ - static URIMapper newOutputURIMapper(String jobId) throws IOException { - File outputDir = IOHelper.makeDirs(getJobOutputDir(jobId)); - return new URIMapper(URI.create(""),outputDir.toURI()); - } - - /** - * Returns a URI mapper which builds a directory structure based on the job ID. - */ - static URIMapper newURIMapper(String jobId) throws IOException { - //based on the the id - File contextDir = IOHelper.makeDirs(getJobContextDir(jobId)); - File outputDir = IOHelper.makeDirs(getJobOutputDir(jobId)); - return new URIMapper(contextDir.toURI(),outputDir.toURI()); - } - public static File getLogFile(String jobId) { return getLogFile(jobId, true); } diff --git a/framework/framework-core/src/main/java/org/daisy/pipeline/script/Script.java b/framework/framework-core/src/main/java/org/daisy/pipeline/script/Script.java index 67b2d7bb00..e089c1a8a6 100644 --- a/framework/framework-core/src/main/java/org/daisy/pipeline/script/Script.java +++ b/framework/framework-core/src/main/java/org/daisy/pipeline/script/Script.java @@ -1,5 +1,8 @@ package org.daisy.pipeline.script; +import java.io.File; +import java.io.IOException; + import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -9,6 +12,10 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import org.daisy.common.messaging.MessageAppender; +import org.daisy.pipeline.job.Job.Status; +import org.daisy.pipeline.job.JobResultSet; + /** * Script description. * @@ -16,6 +23,18 @@ */ public abstract class Script { + /** + * Run the script with an input + * + * @param propertiess Properties that may influence the behavior of the script + * @param resultBuilder For storing the results + */ + public abstract Status run(ScriptInput input, + Map properties, + MessageAppender messages, + JobResultSet.Builder resultBuilder, + File resultDir) throws IOException; + /** * Builder for {@link Script} objects. */ diff --git a/framework/framework-core/src/main/java/org/daisy/pipeline/script/ScriptInput.java b/framework/framework-core/src/main/java/org/daisy/pipeline/script/ScriptInput.java index e6a873247a..54dd0adbcc 100644 --- a/framework/framework-core/src/main/java/org/daisy/pipeline/script/ScriptInput.java +++ b/framework/framework-core/src/main/java/org/daisy/pipeline/script/ScriptInput.java @@ -1,12 +1,16 @@ package org.daisy.pipeline.script; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.InputStream; import java.io.IOException; +import java.io.Reader; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; +import java.nio.file.Files; import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; @@ -18,9 +22,12 @@ import javax.xml.transform.Source; import javax.xml.transform.sax.SAXSource; +import com.google.common.io.CharStreams; + import org.daisy.common.transform.LazySaxSourceProvider; import org.daisy.pipeline.job.JobResources; import org.daisy.pipeline.job.JobResourcesDir; +import org.daisy.pipeline.job.impl.IOHelper; import org.xml.sax.InputSource; @@ -83,7 +90,7 @@ public Builder withInput(String port, Source source) throws IllegalArgumentExcep InputSource is = SAXSource.sourceToInputSource(source); if (is == null || (is.getByteStream() == null && is.getCharacterStream() == null)) { String sysId = source.getSystemId(); - if (sysId == null) { + if (sysId == null || "".equals(sysId)) { throw new IllegalArgumentException( "Input is expected to either be a stream or have non empty system ID"); } @@ -236,6 +243,7 @@ public ScriptInput build() { private final JobResources resources; private final Map inputs; private final Map> options; + private boolean storedToDisk = false; private final static List emptySources = ImmutableList.of(); private final static List emptyValues = ImmutableList.of(); @@ -272,6 +280,97 @@ public JobResources getResources() { return resources; } + /** + * Ensure all documents on input ports are stored to disk + * + * @param baseDir The directory does not have to exist yet, and may be + * {@code null}, in which case a temporary directory will be + * created. + */ + public ScriptInput storeToDisk(File baseDir) throws IOException { + if (storedToDisk) + return this; + boolean everythingStored = true; + if (this.resources != null) + everythingStored = false; + else + ports: for (String port : this.inputs.keySet()) + for (Source src : this.inputs.get(port)) + if (!isStoredOnDisk(src)) { + everythingStored = false; + break ports; } + if (everythingStored) { + storedToDisk = true; + return this; + } + JobResources resources = this.resources; + Map inputs = Maps.newHashMap(); + if (this.resources != null) { + if (baseDir == null) + baseDir = Files.createTempDirectory(null).toFile(); + baseDir.mkdirs(); + IOHelper.dump(this.resources, baseDir.toURI()); + resources = new JobResourcesDir(baseDir); + // just to be sure we don't use the same directory for files that come from the ZIP context + // and inline documents from the job request XML + baseDir = null; + } + inputs = Maps.newHashMap(); + for (String port : this.inputs.keySet()) { + SourceSequence sources = new SourceSequence(); + inputs.put(port, sources); + int inputCnt = 0; // number of inputs for this port + for (Source src : this.inputs.get(port)) { + InputSource is = SAXSource.sourceToInputSource(src); + if (is != null && (is.getByteStream() != null || is.getCharacterStream() != null)) { + // this is the case when the document comes from the job request XML for instance + if (baseDir == null) + baseDir = Files.createTempDirectory(null).toFile(); + // give the file a name that resembles the original name if possible + String fileName; { + try { + fileName = new File(is.getSystemId()).getName(); + } catch (Throwable e) { + fileName = ""; + } + if ("".equals(fileName)) + fileName = port + '-' + inputCnt + ".xml"; + } + File f = new File(baseDir, fileName); + InputStream s = is.getByteStream(); + if (s == null) { + Reader reader = is.getCharacterStream(); + String encoding = is.getEncoding(); + if (encoding == null) + encoding = "UTF-8"; + s = new ByteArrayInputStream(CharStreams.toString(reader).getBytes(encoding)); + } + f.getParentFile().mkdirs(); + f.deleteOnExit(); + IOHelper.dump(s, new FileOutputStream(f)); + sources.add(new LazySaxSourceProvider(f.toURI().toASCIIString())); + } else + sources.add(src); + inputCnt++; + } + } + ScriptInput i = new ScriptInput(resources, inputs, options); + i.storedToDisk = true; + return i; + } + + private static boolean isStoredOnDisk(Source src) { + InputSource is = SAXSource.sourceToInputSource(src); + if (is != null && (is.getByteStream() != null || is.getCharacterStream() != null)) + return false; + else + return true; + } + + public ScriptInput storeToDisk() throws IOException { + return storeToDisk(null); + } + private static class SourceSequence implements Iterable { private Iterable iterable; diff --git a/framework/framework-core/src/main/java/org/daisy/pipeline/script/ScriptRegistry.java b/framework/framework-core/src/main/java/org/daisy/pipeline/script/ScriptRegistry.java index 6572200884..912f090c8d 100644 --- a/framework/framework-core/src/main/java/org/daisy/pipeline/script/ScriptRegistry.java +++ b/framework/framework-core/src/main/java/org/daisy/pipeline/script/ScriptRegistry.java @@ -62,16 +62,35 @@ protected void activate() { cardinality = ReferenceCardinality.MULTIPLE, policy = ReferencePolicy.STATIC ) - protected void register(XProcScriptService script) { + protected void register(ScriptService script) { logger.debug("Registering script {}", script.getId()); descriptors.put(script.getId(), script); } - protected void unregister(XProcScriptService script) { + protected void unregister(ScriptService script) { logger.debug("Unregistering script {}", script.getId()); descriptors.remove(script.getId()); } + @Reference( + name = "script-service-provider", + unbind = "-", + service = ScriptServiceProvider.class, + cardinality = ReferenceCardinality.MULTIPLE, + policy = ReferencePolicy.STATIC + ) + protected void register(ScriptServiceProvider scripts) { + logger.debug("Registering scripts from {}", scripts); + for (ScriptService s : scripts.getScripts()) + register(s); + } + + protected void unregister(ScriptServiceProvider scripts) { + logger.debug("Unregistering scripts from {}", scripts); + for (ScriptService s : scripts.getScripts()) + unregister(s); + } + /** * The parser to load {@link Script} objects from XProc files. */ diff --git a/framework/framework-core/src/main/java/org/daisy/pipeline/script/ScriptServiceProvider.java b/framework/framework-core/src/main/java/org/daisy/pipeline/script/ScriptServiceProvider.java new file mode 100644 index 0000000000..719fa6bffd --- /dev/null +++ b/framework/framework-core/src/main/java/org/daisy/pipeline/script/ScriptServiceProvider.java @@ -0,0 +1,7 @@ +package org.daisy.pipeline.script; + +public interface ScriptServiceProvider { + + public Iterable> getScripts(); + +} diff --git a/framework/framework-core/src/main/java/org/daisy/pipeline/script/XProcScript.java b/framework/framework-core/src/main/java/org/daisy/pipeline/script/XProcScript.java index 2cc8abc6cc..2327622e1a 100644 --- a/framework/framework-core/src/main/java/org/daisy/pipeline/script/XProcScript.java +++ b/framework/framework-core/src/main/java/org/daisy/pipeline/script/XProcScript.java @@ -1,5 +1,8 @@ package org.daisy.pipeline.script; +import java.io.File; +import java.io.InputStream; +import java.io.IOException; import java.net.URI; import java.util.ArrayList; import java.util.HashMap; @@ -12,21 +15,39 @@ import javax.xml.namespace.NamespaceContext; import javax.xml.namespace.QName; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.Result; import com.google.common.base.Joiner; +import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; +import org.daisy.common.messaging.MessageAppender; import org.daisy.common.properties.Properties; +import org.daisy.common.xml.DocumentBuilder; +import org.daisy.common.xproc.XProcEngine; +import org.daisy.common.xproc.XProcInput; import org.daisy.common.xproc.XProcOptionInfo; +import org.daisy.common.xproc.XProcOutput; +import org.daisy.common.xproc.XProcPipeline; import org.daisy.common.xproc.XProcPortInfo; +import org.daisy.common.xproc.XProcResult; import org.daisy.pipeline.datatypes.DatatypeRegistry; import org.daisy.pipeline.datatypes.DatatypeService; +import org.daisy.pipeline.job.impl.IOHelper; +import org.daisy.pipeline.job.Job.Status; +import org.daisy.pipeline.job.JobResultSet; +import org.daisy.pipeline.script.impl.DynamicResultProvider; +import org.daisy.pipeline.script.impl.StatusResultProvider; +import org.daisy.pipeline.script.impl.XProcDecorator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; + /** * XProc based implementation of {@link Script} */ @@ -41,13 +62,8 @@ public final class XProcScript extends Script { private final Optional statusPort; final boolean someOptionsUseProperty; // whether one or more options have a default value that depends // on a (possibly settable) property - - /** - * The URI of the XProc pipeline. - */ - public URI getURI() { - return uri; - } + private final XProcEngine xprocEngine; + private final List inputParsers; public XProcScriptOption getInputOption(String name) { return inputOptions.get(name); @@ -70,6 +86,25 @@ public String toString() { return String.format("XProcScript[name=%s]", this.getName()); } + @Override + public Status run(ScriptInput input, + Map properties, + MessageAppender messages, + JobResultSet.Builder resultBuilder, + File resultDir) throws IOException { + XProcPipeline pipeline = xprocEngine.load(uri); + XProcDecorator decorator = XProcDecorator.from(this, resultDir, inputParsers); + XProcInput xprocInput = decorator.decorate(input); + XProcResult xprocResult = pipeline.run(xprocInput, () -> messages, properties); + XProcOutput xprocOutput = decorator.decorate(new XProcOutput.Builder().build()); + xprocResult.writeTo(xprocOutput); // writes to files and/or streams specified in output + buildResultSet(xprocInput, xprocOutput, resultDir, resultBuilder); + if (checkStatusPort(xprocOutput)) + return Status.SUCCESS; + else + return Status.FAIL; + } + /** * Builder for {@link XProcScript} objects. */ @@ -80,17 +115,23 @@ public static class Builder extends Script.Builder { private final List tempOptions = new ArrayList<>(); private final Map resultOptions = new HashMap<>(); private ScriptPort statusPort; + private final XProcEngine xprocEngine; + private final List inputParsers; private final DatatypeRegistry datatypes; - public Builder(XProcScriptService descriptor, URI uri, DatatypeRegistry datatypes) { + public Builder(XProcScriptService descriptor, URI uri, XProcEngine xprocEngine, List inputParsers, DatatypeRegistry datatypes) { super(descriptor); this.uri = uri; + this.xprocEngine = xprocEngine; + this.inputParsers = inputParsers; this.datatypes = datatypes; } - public Builder(String id, String version, URI uri, DatatypeRegistry datatypes) { + public Builder(String id, String version, URI uri, XProcEngine xprocEngine, List inputParsers, DatatypeRegistry datatypes) { super(id, version); this.uri = uri; + this.xprocEngine = xprocEngine; + this.inputParsers = inputParsers; this.datatypes = datatypes; } @@ -208,7 +249,7 @@ public Builder withOutputPort(XProcPortInfo info, XProcPortMetadata metadata) { */ @Override public XProcScript build() { - return new XProcScript(uri, id, version, shortName, description, homepage, + return new XProcScript(xprocEngine, inputParsers, uri, id, version, shortName, description, homepage, inputPorts, outputPorts, options, inputFilesets, outputFilesets, inputOptions, tempOptions, resultOptions, statusPort); } @@ -269,7 +310,8 @@ private String uniqueOutputPortName(QName qName) { } } - private XProcScript(URI uri, String id, String version, String name, String description, String homepage, + private XProcScript(XProcEngine xprocEngine, List inputParsers, + URI uri, String id, String version, String name, String description, String homepage, Map inputPorts, Map outputPorts, Map options, List inputFilesets, List outputFilesets, Map inputOptions, List tempOptions, @@ -284,6 +326,8 @@ private XProcScript(URI uri, String id, String version, String name, String desc || Iterables.any(inputOptions.values(), o -> o.usesProperty) || Iterables.any(tempOptions, o -> o.usesProperty) || Iterables.any(resultOptions.values(), o -> o.usesProperty); + this.xprocEngine = xprocEngine; + this.inputParsers = inputParsers; } @Override @@ -592,7 +636,7 @@ else if (datatype == DatatypeService.XS_INTEGER || throw new IllegalArgumentException("can not convert value to integer: " + value, e); } else if (datatype == DatatypeService.XS_BOOLEAN) - return "true".equals(value.toLowerCase()) || "1".equals(value); + return datatype.validate(value).isValid(); else throw new RuntimeException("coding error"); } @@ -644,4 +688,128 @@ public String getMediaType() { return metadata.getMediaType(); } } + + private void buildResultSet(XProcInput inputs, XProcOutput outputs, + File resultDir, JobResultSet.Builder builder) throws IOException { + + // iterate over output ports + for (ScriptPort port : getOutputPorts()) { + String mediaType = port.getMediaType(); + + // check if it is implemented as an output option + XProcScriptOption option = getResultOption(port.getName()); + if (option != null) { + if (inputs.getOptions().get(option.getXProcOptionName()) == null) + // option was not set + continue; + if (XProcOptionMetadata.ANY_FILE_URI.equals(option.getType().getId())) { + URI path; { + Object val = inputs.getOptions().get(option.getXProcOptionName()); + try { + path = URI.create((String)val); + } catch (ClassCastException e) { + throw new RuntimeException( + "Expected string value for option " + option.getName() + + " but got: " + val.getClass()); + } + } + File f = new File(path); + if (f.exists()) { + builder = builder.addResult(port.getName(), + resultDir.toURI().relativize(path).toString(), + f, + mediaType); + } + } else if (XProcOptionMetadata.ANY_DIR_URI.equals(option.getType().getId())) { + String dir; { + Object val = inputs.getOptions().get(option.getXProcOptionName()); + try { + dir = (String)val; + } catch (ClassCastException e) { + throw new RuntimeException( + "Expected string value for option " + option.getName() + + " but got: " + val.getClass()); + } + } + // scan the directory to get all files inside and write them to the XProcOutput + for (File f : IOHelper.treeFileList(new File(URI.create(dir)))) { + URI path = f.toURI(); + builder = builder.addResult(port.getName(), + resultDir.toURI().relativize(path).toString(), + f, + mediaType); + } + } + } else { + Supplier resultProvider = outputs.getResultProvider(port.getName()); + if (resultProvider == null) + // XProcDecorator makes sure this can not happen + continue; + if (!(resultProvider instanceof DynamicResultProvider)) + // XProcDecorator makes sure this can not happen + throw new RuntimeException( + "Result supplier is expected to be a DynamicResultProvider but got: " + resultProvider); + for (Result result : ((DynamicResultProvider)resultProvider).providedResults()) { + String sysId = result.getSystemId(); + if (sysId == null) + // XProcDecorator makes sure this can not happen + throw new RuntimeException( + "Result is expected to be a DynamicResult but got: " + result); + URI path = URI.create(sysId); + builder = builder.addResult(port.getName(), + resultDir.toURI().relativize(path).toString(), + new File(path), + mediaType); + } + } + } + } + + // for unit tests + JobResultSet buildResultSet(XProcInput inputs, XProcOutput outputs, File resultDir) throws IOException { + JobResultSet.Builder builder = new JobResultSet.Builder(this); + buildResultSet(inputs, outputs, resultDir, builder); + return builder.build(); + } + + /** + * Check the validation status from the status port and get it's value. + */ + // package private for unit tests + boolean checkStatusPort(XProcOutput outputs) { + Optional statusPort = getStatusPort(); + if (statusPort.isPresent()) { + Supplier provider = outputs.getResultProvider(statusPort.get().getName()); + if (provider != null && provider instanceof StatusResultProvider) { // should always be true + boolean ok = true; + for (InputStream status : ((StatusResultProvider)provider).read()) { + ok &= processStatus(status); + } + return ok; + } + } + return true; + } + + /** + * Read the XML file to check that validation status is equal to "ok". + */ + // package private for unit tests + static boolean processStatus(InputStream status) { + // check the contents of the xml and check if result is "ok" + // + try { + DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance(); + docBuilderFactory.setNamespaceAware(true); + javax.xml.parsers.DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder(); + Document doc = docBuilder.parse(status); + String result = doc.getDocumentElement().getAttribute("result"); + if (result == null || result.isEmpty()) { + throw new RuntimeException("No result attribute was found in the status port"); + } + return result.equalsIgnoreCase("ok"); + } catch (Exception e) { + throw new RuntimeException("Error processing status file", e); + } + } } diff --git a/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/DynamicResultProvider.java b/framework/framework-core/src/main/java/org/daisy/pipeline/script/impl/DynamicResultProvider.java similarity index 95% rename from framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/DynamicResultProvider.java rename to framework/framework-core/src/main/java/org/daisy/pipeline/script/impl/DynamicResultProvider.java index d9f86bbcee..f96c1bf1ce 100644 --- a/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/DynamicResultProvider.java +++ b/framework/framework-core/src/main/java/org/daisy/pipeline/script/impl/DynamicResultProvider.java @@ -1,5 +1,6 @@ -package org.daisy.pipeline.job.impl; +package org.daisy.pipeline.script.impl; +import java.io.File; import java.net.URI; import java.util.Collection; import java.util.Collections; @@ -11,8 +12,6 @@ import com.google.common.base.Supplier; import com.google.common.collect.Lists; -import org.daisy.pipeline.job.URIMapper; - /** * This class is not thread-safe if several threads are generating results at the same time. * not that likely use case @@ -23,17 +22,17 @@ public final class DynamicResultProvider implements Supplier{ private final Supplier backingProvider; private final String portName; private final String portMimetype; - private final URIMapper mapper; + private final URI resultDir; private String constantSystemId = null; private String prefix = null; private String suffix = null; - public DynamicResultProvider(Supplier backingProvider, String portName, String portMimetype, URIMapper mapper) { + public DynamicResultProvider(Supplier backingProvider, String portName, String portMimetype, File resultDir) { this.backingProvider = backingProvider; this.portName = portName; this.portMimetype = portMimetype; - this.mapper = mapper; + this.resultDir = resultDir.toURI(); }; /** @@ -69,7 +68,7 @@ public Result get() { if (res == null) { if (prefix == null) { String parts[] = getDynamicResultProviderParts(portName, constantSystemId, portMimetype); - prefix = mapper.mapOutput(URI.create(parts[0])).toString(); + prefix = resultDir.resolve(URI.create(parts[0])).toString(); suffix = parts[1]; } String sysId = count == 0 ? diff --git a/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/StatusResultProvider.java b/framework/framework-core/src/main/java/org/daisy/pipeline/script/impl/StatusResultProvider.java similarity index 95% rename from framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/StatusResultProvider.java rename to framework/framework-core/src/main/java/org/daisy/pipeline/script/impl/StatusResultProvider.java index 6321d1b308..5109f13d4c 100644 --- a/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/StatusResultProvider.java +++ b/framework/framework-core/src/main/java/org/daisy/pipeline/script/impl/StatusResultProvider.java @@ -1,4 +1,4 @@ -package org.daisy.pipeline.job.impl; +package org.daisy.pipeline.script.impl; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; diff --git a/framework/framework-core/src/main/java/org/daisy/pipeline/script/impl/StaxXProcScriptParser.java b/framework/framework-core/src/main/java/org/daisy/pipeline/script/impl/StaxXProcScriptParser.java index 05b223b047..9968575cc0 100644 --- a/framework/framework-core/src/main/java/org/daisy/pipeline/script/impl/StaxXProcScriptParser.java +++ b/framework/framework-core/src/main/java/org/daisy/pipeline/script/impl/StaxXProcScriptParser.java @@ -3,8 +3,10 @@ import java.io.IOException; import java.io.InputStream; import java.net.URL; +import java.util.ArrayList; import java.util.function.Predicate; import java.util.LinkedList; +import java.util.List; import javax.xml.namespace.QName; import javax.xml.stream.XMLEventReader; @@ -17,6 +19,8 @@ import org.daisy.common.stax.EventProcessor; import org.daisy.common.stax.StaxEventHelper; import org.daisy.common.stax.StaxEventHelper.EventPredicates; +import org.daisy.common.xml.DocumentBuilder; +import org.daisy.common.xproc.XProcEngine; import org.daisy.common.xproc.XProcPipelineInfo; import org.daisy.pipeline.datatypes.DatatypeRegistry; import org.daisy.pipeline.script.Script; @@ -48,6 +52,8 @@ public class StaxXProcScriptParser { private static final Logger logger = LoggerFactory.getLogger(StaxXProcScriptParser.class); private XMLInputFactory xmlInputFactory; + private XProcEngine xprocEngine; + private final List inputParsers = new ArrayList<>(); private DatatypeRegistry datatypeRegistry; @Reference( @@ -61,6 +67,28 @@ protected void setFactory(XMLInputFactory factory) { xmlInputFactory = factory; } + @Reference( + name = "xproc-engine", + unbind = "-", + service = XProcEngine.class, + cardinality = ReferenceCardinality.MANDATORY, + policy = ReferencePolicy.STATIC + ) + protected void setXProcEngine(XProcEngine xprocEngine) { + this.xprocEngine = xprocEngine; + } + + @Reference( + name = "input-parser", + unbind = "-", + service = DocumentBuilder.class, + cardinality = ReferenceCardinality.MULTIPLE, + policy = ReferencePolicy.STATIC + ) + protected void addInputParser(DocumentBuilder parser) { + inputParsers.add(parser); + } + @Reference( name = "datatype-registry", unbind = "-", @@ -109,7 +137,7 @@ public XProcScript parse(final XProcScriptService descriptor) { infoParser.setFactory(xmlInputFactory); try { XProcPipelineInfo info = infoParser.parse(descriptor.getURL()); - scriptBuilder = new XProcScript.Builder(descriptor, info.getURI(), datatypeRegistry); + scriptBuilder = new XProcScript.Builder(descriptor, info.getURI(), xprocEngine, inputParsers, datatypeRegistry); URL descUrl = descriptor.getURL(); is = descUrl.openConnection().getInputStream(); reader = xmlInputFactory.createXMLEventReader(is); diff --git a/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/XProcDecorator.java b/framework/framework-core/src/main/java/org/daisy/pipeline/script/impl/XProcDecorator.java similarity index 69% rename from framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/XProcDecorator.java rename to framework/framework-core/src/main/java/org/daisy/pipeline/script/impl/XProcDecorator.java index 147d741195..e31775ee1b 100644 --- a/framework/framework-core/src/main/java/org/daisy/pipeline/job/impl/XProcDecorator.java +++ b/framework/framework-core/src/main/java/org/daisy/pipeline/script/impl/XProcDecorator.java @@ -1,11 +1,7 @@ -package org.daisy.pipeline.job.impl; +package org.daisy.pipeline.script.impl; -import java.io.ByteArrayInputStream; import java.io.File; -import java.io.FileOutputStream; -import java.io.InputStream; import java.io.IOException; -import java.io.Reader; import java.net.URI; import java.util.ArrayList; import java.util.Arrays; @@ -23,12 +19,12 @@ import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.google.common.io.CharStreams; import org.daisy.common.xml.DocumentBuilder; import org.daisy.common.xproc.XProcInput; import org.daisy.common.xproc.XProcOutput; -import org.daisy.pipeline.job.URIMapper; +import org.daisy.pipeline.job.JobResources; +import org.daisy.pipeline.job.JobResourcesDir; import org.daisy.pipeline.script.XProcScript; import org.daisy.pipeline.script.XProcScript.XProcScriptOption; import org.daisy.pipeline.script.ScriptInput; @@ -48,7 +44,7 @@ public class XProcDecorator { private static final Logger logger = LoggerFactory.getLogger(XProcDecorator.class); private final XProcScript script; - private final URIMapper mapper; + private final File resultDir; private List inputParsers; /** @@ -56,24 +52,29 @@ public class XProcDecorator { * * @param contextDir The contextDir for this instance. */ - private XProcDecorator(XProcScript script, URIMapper mapper, List inputParsers) { + private XProcDecorator(XProcScript script, File resultDir, List inputParsers) { this.script = script; - this.mapper = mapper; + this.resultDir = resultDir; this.inputParsers = inputParsers; } - public static XProcDecorator from(XProcScript script, URIMapper mapper, List inputParsers) throws IOException { - return new XProcDecorator(script, mapper, inputParsers); + public static XProcDecorator from(XProcScript script, File resultDir, List inputParsers) throws IOException { + return new XProcDecorator(script, resultDir, inputParsers); } public XProcInput decorate(ScriptInput input) { logger.debug(String.format("Translating inputs for script :%s",script)); XProcInput.Builder decorated = new XProcInput.Builder(); - try{ + try { + // Store everything to disk just in case it hasn't been done before. + // We need this for two reasons: + // - to make sure documents on input ports have a non-empty system ID (they don't need to be stored on disk per se though) + // - to make sure documents can be passed as a file path to XProc options + input = input.storeToDisk(); decorateInputPorts(script, input, decorated); decorateOptions(script, input, decorated); - }catch(IOException ex){ - throw new RuntimeException("Error translating inputs",ex); + } catch(IOException e) { + throw new RuntimeException("Error translating inputs", e); } return decorated.build(); } @@ -105,7 +106,7 @@ public XProcOutput decorate(XProcOutput output) { new DynamicResultProvider(prov, port.getName(), port.getMediaType(), - mapper)); + resultDir)); } } Optional statusPort = script.getStatusPort(); @@ -126,25 +127,22 @@ public XProcOutput decorate(XProcOutput output) { void decorateInputPorts(final XProcScript script, final ScriptInput input, XProcInput.Builder builder) throws IOException { for (ScriptPort port : script.getInputPorts()) { if (script.getInputOption(port.getName()) == null) { - // number of inputs for this port - int inputCnt = 0; for (Source src : input.getInput(port.getName())) { InputSource is = SAXSource.sourceToInputSource(src); - URI relUri = null; { - if (is != null && (is.getByteStream() != null || is.getCharacterStream() != null)) - // this is the case when no zip context was provided (all comes from the xml) - relUri = URI.create(port.getName() + '-' + inputCnt + ".xml"); - else { - try { - relUri = URI.create(src.getSystemId()); - } catch (Exception e) { - throw new RuntimeException( - "Error parsing uri when building the input port" + port.getName(), e); - } - } + // make sure documents on input ports have a non-empty base URI + if (src.getSystemId() == null + || "".equals(src.getSystemId()) + || (is != null && (is.getByteStream() != null || is.getCharacterStream() != null))) + throw new IllegalStateException(); // should not happen because ScripInput.storeToDisk() was called + // make relative file paths absolute + try { + URI baseURI = URI.create(src.getSystemId()); + baseURI = resolveRelativePath(baseURI, input); + src.setSystemId(baseURI.toString()); + } catch (Exception e) { + throw new RuntimeException( + "Error parsing URI when building the input port" + port.getName(), e); } - URI uri = mapper.mapInput(relUri); - src.setSystemId(uri.toString()); String mediaType = port.getMediaType(); if (mediaType != null && !(src instanceof DOMSource)) { mediaType = mediaType.trim(); @@ -172,13 +170,11 @@ else if (types.removeIf(p::supportsContentType)) "Input on port " + port.getName() + " (media-type: " + mediaType + ") could not be parsed"); Source domSrc = new DOMSource(doc); builder.withInput(port.getName(), () -> domSrc); - inputCnt++; continue; } } } builder.withInput(port.getName(), () -> src); - inputCnt++; } } } @@ -199,34 +195,23 @@ void decorateInputOptions(XProcScript script, ScriptInput input, XProcInput.Buil XProcScriptOption option = script.getInputOption(port.getName()); if (option != null) { List value = new ArrayList<>(); - for (Source source : input.getInput(port.getName())) { - InputSource is = SAXSource.sourceToInputSource(source); - if (is != null && (is.getByteStream() != null || is.getCharacterStream() != null)) { - // document is not stored on disk so we can't pass a file path to the XProc option - // store it to a temporary location - String sysId = is.getSystemId(); - // give the file a name that resembles the original name - File f = File.createTempFile("input", sysId != null ? new File(sysId).getName() : null); - f.deleteOnExit(); - InputStream stream = is.getByteStream(); - if (stream == null) { - Reader reader = is.getCharacterStream(); - String encoding = is.getEncoding(); - if (encoding == null) - encoding = "UTF-8"; - stream = new ByteArrayInputStream(CharStreams.toString(reader).getBytes(encoding)); - } - IOHelper.dump(is.getByteStream(), new FileOutputStream(f)); - value.add(f.toURI().toString()); - } else { - String sysId = source.getSystemId(); - try { - value.add(mapper.mapInput(URI.create(sysId)).toString()); - } catch (IllegalArgumentException e) { - throw new RuntimeException( - String.format("Error parsing URI for option %s: %s", option.getName(), sysId)); - } + for (Source src : input.getInput(port.getName())) { + InputSource is = SAXSource.sourceToInputSource(src); + // make sure documents are stored on disk so we can pass a file path to the XProc option + if (src.getSystemId() == null + || "".equals(src.getSystemId()) + || (is != null && (is.getByteStream() != null || is.getCharacterStream() != null))) + throw new IllegalStateException(); // should not happen because ScripInput.storeToDisk() was called + // make relative file paths absolute + try { + URI baseURI = URI.create(src.getSystemId()); + baseURI = resolveRelativePath(baseURI, input); + src.setSystemId(baseURI.toString()); + } catch (Exception e) { + throw new RuntimeException( + "Error parsing URI for option %s: %s" + option.getName(), e); } + value.add(src.getSystemId()); } resolvedInput.withOption( option.getXProcOptionName(), @@ -247,7 +232,11 @@ void decorateInputOptions(XProcScript script, ScriptInput input, XProcInput.Buil XProcDecorator::notEmpty), v -> { try { - return mapper.mapInput(URI.create(v)).toString(); + URI u = URI.create(v); + u = resolveRelativePath(u, input); // ScriptInput does not check "anyDirURI" options, + // so this will fail if it is a relative path but + // no context ZIP was provided! + return u.toString(); } catch (IllegalArgumentException e) { throw new RuntimeException( String.format("Error parsing URI for option %s: %s", option.getName(), v)); @@ -280,6 +269,25 @@ void decorateOutputOptions(XProcScript script, ScriptInput input, XProcInput.Bui } } + /** + * @param uri The base URI of a document on an input port of the provided {@link ScriptInput}. + */ + private static URI resolveRelativePath(URI uri, ScriptInput input) { + if (uri.isAbsolute()) { // absolute means URI has scheme component + if (!"file".equals(uri.getScheme()) || uri.isOpaque()) + throw new IllegalStateException(); // should not happen if the URI comes from a document on an input + // port: ScripInput does not allow this + // URI is a file URI with an absolute file path + return uri; + } else { + // URI is a relative path + JobResources resources = input.getResources(); + if (!(resources instanceof JobResourcesDir)) + throw new IllegalStateException(); // should not happen because ScripInput.storeToDisk() was called + return ((JobResourcesDir)resources).getBaseDir().toURI().resolve(uri); + } + } + private final HashSet generatedOutputs = Sets.newHashSet(); private URI decorateOutputOption(XProcScriptOption option) { @@ -300,7 +308,7 @@ private URI decorateOutputOption(XProcScriptOption option) { generatedOutputs.add(uri); } try { - return mapper.mapOutput(URI.create(uri)); + return resultDir.toURI().resolve(URI.create(uri)); } catch (IllegalArgumentException e) { throw new RuntimeException(String.format("Error parsing URI for option %s: %s", option.getName(), uri), e); } diff --git a/framework/framework-core/src/test/java/FrameworkCoreTest.java b/framework/framework-core/src/test/java/FrameworkCoreTest.java index e9904b19d4..78d65f571a 100644 --- a/framework/framework-core/src/test/java/FrameworkCoreTest.java +++ b/framework/framework-core/src/test/java/FrameworkCoreTest.java @@ -30,7 +30,8 @@ public String[] testDependencies() { public TestProbeBuilder probeConfiguration(TestProbeBuilder probe) { // FIXME: can not delete this yet because it can not be generated with maven-bundle-plugin probe.setHeader("Service-Component", "OSGI-INF/script.xml," - + "OSGI-INF/datatype.xml"); + + "OSGI-INF/datatype.xml," + + "OSGI-INF/xproc-engine.xml"); return probe; } diff --git a/framework/framework-core/src/test/java/MockXProcEngine.java b/framework/framework-core/src/test/java/MockXProcEngine.java new file mode 100644 index 0000000000..b8c28ab898 --- /dev/null +++ b/framework/framework-core/src/test/java/MockXProcEngine.java @@ -0,0 +1,22 @@ +import java.net.URI; + +import org.daisy.common.xproc.XProcEngine; +import org.daisy.common.xproc.XProcErrorException; +import org.daisy.common.xproc.XProcInput; +import org.daisy.common.xproc.XProcPipeline; +import org.daisy.common.xproc.XProcPipelineInfo; +import org.daisy.common.xproc.XProcResult; + +import org.osgi.service.component.annotations.Component; + +@Component( + name = "mock-xproc-engine", + service = { XProcEngine.class } +) +public class MockXProcEngine implements XProcEngine { + + public XProcPipeline load(URI uri) { throw new UnsupportedOperationException(); } + public XProcPipelineInfo getInfo(URI uri) { throw new UnsupportedOperationException(); } + public XProcResult run(URI uri, XProcInput data) throws XProcErrorException { throw new UnsupportedOperationException(); } + +} diff --git a/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/JobURIUtilsTest.java b/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/JobURIUtilsTest.java index 459c53697d..55098b923c 100644 --- a/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/JobURIUtilsTest.java +++ b/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/JobURIUtilsTest.java @@ -1,83 +1,87 @@ package org.daisy.pipeline.job.impl; import java.io.File; +import java.io.IOException; import java.net.URI; +import java.nio.file.Files; + +import org.apache.commons.io.FileUtils; import org.daisy.pipeline.job.JobId; import org.daisy.pipeline.job.JobIdFactory; -import org.daisy.pipeline.job.URIMapper; -import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; -import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Test; -public class JobURIUtilsTest { - JobId id; - String oldIoBase=""; - File jobsDir; +public class JobURIUtilsTest { + + static JobId id; + static String oldIoBase=""; + static File tmpdir; + static File jobsDir; + static URI jobsDirURI; - @Before - public void setUp() { + @BeforeClass + public static void setUp() { oldIoBase = System.getProperty("org.daisy.pipeline.data"); - File tmpdir = new File(System.getProperty("java.io.tmpdir")); + try { + tmpdir = Files.createTempDirectory(null).toFile(); + } catch (IOException e) { + throw new RuntimeException(e); + } System.setProperty("org.daisy.pipeline.data", tmpdir.toString()); jobsDir = new File(tmpdir, "jobs"); - id= JobIdFactory.newId(); - + jobsDirURI = URI.create(jobsDir.toURI().toString() + "/"); + id = JobIdFactory.newId(); } - @After - public void tearDown(){ - if(oldIoBase!=null) + @AfterClass + public static void tearDown(){ + try { + FileUtils.deleteDirectory(tmpdir); + } catch (IOException e) { + throw new RuntimeException(e); + } + if (oldIoBase != null) System.setProperty("org.daisy.pipeline.data", oldIoBase); } @Test - public void outputUriMapper() throws Exception{ - URIMapper mapper = JobURIUtils.newOutputURIMapper(id.toString()); - Assert.assertEquals(URI.create(""),mapper.getInputBase()); - String commonBase=jobsDir.toURI().toString()+id.toString()+"/"; - Assert.assertEquals(URI.create(commonBase+JobURIUtils.IO_OUTPUT_SUBDIR+"/"),mapper.getOutputBase()); - - } - - @Test - public void idBasedUriMapper() throws Exception{ - URIMapper mapper = JobURIUtils.newURIMapper(id.toString()); - String commonBase=jobsDir.toURI().toString()+id.toString()+"/"; - Assert.assertEquals(URI.create(commonBase+JobURIUtils.IO_DATA_SUBDIR+"/"),mapper.getInputBase()); - Assert.assertEquals(URI.create(commonBase+JobURIUtils.IO_OUTPUT_SUBDIR+"/"),mapper.getOutputBase()); - + public void testJobContextDir() throws Exception { + Assert.assertEquals(new File(jobsDirURI.resolve(URI.create(String.format("%s/%s/", id.toString(), JobURIUtils.IO_DATA_SUBDIR)))), + JobURIUtils.getJobContextDir(id.toString())); } @Test public void getLogFile() throws Exception{ - URI expected= jobsDir.toURI().resolve(URI.create(String.format("%s/%s.log",id.toString(), id.toString()))); + URI expected = jobsDirURI.resolve(URI.create(String.format("%s/%s.log", id.toString(), id.toString()))); Assert.assertEquals(JobURIUtils.getLogFile(id.toString()).toURI(), expected); - } + @Test public void getLogFileExsists() throws Exception{ Assert.assertTrue(JobURIUtils.getLogFile(id.toString()).exists()); - } @Test public void getJobBase() throws Exception{ - URI expected= jobsDir.toURI().resolve(URI.create(String.format("%s/", id.toString()))); + URI expected = jobsDirURI.resolve(URI.create(String.format("%s/", id.toString()))); Assert.assertEquals(JobURIUtils.getJobBaseDir(id.toString()).toURI(), expected); } @Test public void getJobContextDir() throws Exception{ File context = JobURIUtils.getJobContextDir(id.toString()); - Assert.assertEquals(new File(JobURIUtils.getJobBaseDir(id.toString()), JobURIUtils.IO_DATA_SUBDIR).getAbsolutePath(), context.getAbsolutePath()); + Assert.assertEquals(new File(JobURIUtils.getJobBaseDir(id.toString()), JobURIUtils.IO_DATA_SUBDIR).getAbsolutePath(), + context.getAbsolutePath()); } @Test public void getJobOutputDir() throws Exception{ - File output=JobURIUtils.getJobOutputDir(id.toString()); - Assert.assertEquals(new File(JobURIUtils.getJobBaseDir(id.toString()), JobURIUtils.IO_OUTPUT_SUBDIR).getAbsolutePath(), output.getAbsolutePath()); + File output = JobURIUtils.getJobOutputDir(id.toString()); + Assert.assertEquals(new File(JobURIUtils.getJobBaseDir(id.toString()), JobURIUtils.IO_OUTPUT_SUBDIR).getAbsolutePath(), + output.getAbsolutePath()); } } diff --git a/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/Mock.java b/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/Mock.java index 3a558c7d2e..b71b6a1d46 100644 --- a/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/Mock.java +++ b/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/Mock.java @@ -278,7 +278,7 @@ public ScriptGenerator(int inputs, int optionOutputsFile, public XProcScript generate() { - XProcScript.Builder builder = new XProcScript.Builder("", "", null, null); + XProcScript.Builder builder = new XProcScript.Builder("", "", null, null, null, null); // inputs for (int i = 0; i < inputs; i++) { diff --git a/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/URIMapperTest.java b/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/URIMapperTest.java deleted file mode 100644 index 1b4234400c..0000000000 --- a/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/URIMapperTest.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.daisy.pipeline.job.impl; - -import java.net.URI; - -import org.daisy.pipeline.job.URIMapper; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -public class URIMapperTest { - URI baseIn=URI.create("file:/in/"); - URI baseOut=URI.create("file:/out/"); - URI empty=URI.create(""); - URI uri1=URI.create("one/"); - @Before - public void setUp() { - - } - - @Test - public void mapEmpty() throws Exception{ - URIMapper mapper=new URIMapper(empty,empty); - Assert.assertEquals(mapper.mapInput(uri1),uri1); - } - @Test - public void unmapEmpty() throws Exception{ - URIMapper mapper=new URIMapper(empty,empty); - Assert.assertEquals(mapper.unmapInput(uri1),uri1); - } - @Test - public void mapOutput() throws Exception{ - URIMapper mapper=new URIMapper(baseIn,baseOut); - Assert.assertEquals(mapper.mapOutput(uri1).toString(),baseOut.toString()+uri1.toString()); - } - @Test - public void unmapOutput() throws Exception{ - URIMapper mapper=new URIMapper(baseIn,baseOut); - Assert.assertEquals(mapper.unmapOutput(mapper.mapOutput(uri1)),uri1); - } -} diff --git a/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/VolatileJobStorageTest.java b/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/VolatileJobStorageTest.java index 56a9b145d0..2fd4c38e46 100644 --- a/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/VolatileJobStorageTest.java +++ b/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/VolatileJobStorageTest.java @@ -38,10 +38,10 @@ public class VolatileJobStorageTest { @Before public void setUp() { storage = new VolatileJobStorage(); - job1 = new VolatileJob(new AbstractJob(new Mock.MockedJobContext(cl, batchId1), Priority.MEDIUM, null, null, true) {}); - job2 = new VolatileJob(new AbstractJob(new Mock.MockedJobContext(cl, batchId2), Priority.MEDIUM, null, null, true) {}); - job1OtherCli = new VolatileJob(new AbstractJob(new Mock.MockedJobContext(clOther, batchId1), Priority.MEDIUM, null, null, true) {}); - job2OtherCli = new VolatileJob(new AbstractJob(new Mock.MockedJobContext(clOther, batchId2), Priority.MEDIUM, null, null, true) {}); + job1 = new VolatileJob(new AbstractJob(new Mock.MockedJobContext(cl, batchId1), Priority.MEDIUM, true) {}); + job2 = new VolatileJob(new AbstractJob(new Mock.MockedJobContext(cl, batchId2), Priority.MEDIUM, true) {}); + job1OtherCli = new VolatileJob(new AbstractJob(new Mock.MockedJobContext(clOther, batchId1), Priority.MEDIUM, true) {}); + job2OtherCli = new VolatileJob(new AbstractJob(new Mock.MockedJobContext(clOther, batchId2), Priority.MEDIUM, true) {}); } @Test diff --git a/framework/framework-core/src/test/java/org/daisy/pipeline/job/JobResultSetBuilderTest.java b/framework/framework-core/src/test/java/org/daisy/pipeline/script/JobResultSetBuilderTest.java similarity index 77% rename from framework/framework-core/src/test/java/org/daisy/pipeline/job/JobResultSetBuilderTest.java rename to framework/framework-core/src/test/java/org/daisy/pipeline/script/JobResultSetBuilderTest.java index d46e48ae28..b9648e7051 100644 --- a/framework/framework-core/src/test/java/org/daisy/pipeline/job/JobResultSetBuilderTest.java +++ b/framework/framework-core/src/test/java/org/daisy/pipeline/script/JobResultSetBuilderTest.java @@ -1,4 +1,4 @@ -package org.daisy.pipeline.job; +package org.daisy.pipeline.script; import java.io.File; import java.io.FileWriter; @@ -12,27 +12,25 @@ import org.apache.commons.io.FileUtils; +import com.google.common.base.Supplier; +import com.google.common.collect.Lists; + import org.daisy.common.xproc.XProcInput; import org.daisy.common.xproc.XProcOutput; import org.daisy.pipeline.job.JobResult; import org.daisy.pipeline.job.JobResultSet; -import org.daisy.pipeline.job.URIMapper; +import org.daisy.pipeline.job.impl.IOHelper; import org.daisy.pipeline.job.impl.Mock; -import org.daisy.pipeline.job.impl.XProcDecorator; -import org.daisy.pipeline.script.ScriptInput; -import org.daisy.pipeline.script.XProcScript; +import org.daisy.pipeline.script.impl.XProcDecorator; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import com.google.common.base.Supplier; -import com.google.common.collect.Lists; - public class JobResultSetBuilderTest { - URIMapper mapper; + File resultDir; XProcScript script; JobResultSet.Builder builder ; XProcOutput output; @@ -43,13 +41,13 @@ public class JobResultSetBuilderTest { public void setUp() throws IOException{ script= new Mock.ScriptGenerator.Builder().withOutputPort("sequence", "xml", true, false) .withOutputPorts(2).withOptionOutputsFile(1).withOptionOutputsDir(1).build().generate(); - URI tmp=new File(System.getProperty("java.io.tmpdir")).toURI(); + File tmp = new File(System.getProperty("java.io.tmpdir")); oldIoBase=System.getProperty("org.daisy.pipeline.data"); - System.setProperty("org.daisy.pipeline.data", new File(tmp).toString()); - mapper = new URIMapper(tmp.resolve("inputs/"),tmp.resolve("outputs/")); + System.setProperty("org.daisy.pipeline.data", tmp.toString()); + resultDir = IOHelper.makeDirs(new File(tmp, "outputs")); QName optDir=Mock.ScriptGenerator.getOptionOutputDirName(0); QName optName=Mock.ScriptGenerator.getOptionOutputFileName(0); - XProcDecorator trans = XProcDecorator.from(script, mapper, null); + XProcDecorator trans = XProcDecorator.from(script, resultDir, null); output = trans.decorate(new XProcOutput.Builder().build()); input = trans.decorate(new ScriptInput.Builder().build()); // create result files @@ -76,7 +74,7 @@ private static File writeResult(File result) throws IOException { @After public void tearDown() { try { - FileUtils.deleteDirectory(new File(mapper.getOutputBase())); + FileUtils.deleteDirectory(resultDir); } catch (IOException e) { throw new RuntimeException(e); } @@ -91,9 +89,9 @@ public void outputPort() throws Exception{ File f = null; try { f = writeResult(res.get()); - JobResultSet rSet = AbstractJob.buildResultSet(script, input, output, mapper); + JobResultSet rSet = script.buildResultSet(input, output, resultDir); List jobs=Lists.newLinkedList(rSet.getResults(outName)); - Assert.assertEquals(mapper.mapOutput(URI.create("output-0/output-0.xml")), jobs.get(0).getPath().toURI()); + Assert.assertEquals(new File(resultDir, "output-0/output-0.xml"), jobs.get(0).getPath()); Assert.assertEquals("output-0/output-0.xml", jobs.get(0).getIdx().toString()); } finally { if (f != null) f.delete(); @@ -104,7 +102,7 @@ public void outputPort() throws Exception{ public void outputPortNullCheck() throws Exception{ String outName = Mock.ScriptGenerator.getOutputName(0); XProcOutput output = new XProcOutput.Builder().build(); - JobResultSet rSet = AbstractJob.buildResultSet(script, input, output, mapper); + JobResultSet rSet = script.buildResultSet(input, output, resultDir); List jobs=Lists.newLinkedList(rSet.getResults(outName)); Assert.assertEquals(jobs.size(),0); } @@ -118,10 +116,10 @@ public void outputPortSequence() throws Exception{ try { f1 = writeResult(res.get()); f2 = writeResult(res.get()); - JobResultSet rSet = AbstractJob.buildResultSet(script, input, output, mapper); + JobResultSet rSet = script.buildResultSet(input, output, resultDir); List jobs=Lists.newLinkedList(rSet.getResults(outName)); Assert.assertEquals(jobs.size(),2); - Assert.assertEquals(mapper.mapOutput(URI.create("sequence/sequence.xml")), jobs.get(0).getPath().toURI()); + Assert.assertEquals(new File(resultDir, "sequence/sequence.xml"), jobs.get(0).getPath()); Assert.assertEquals("sequence/sequence.xml", jobs.get(0).getIdx().toString()); Assert.assertEquals("xml",jobs.get(0).getMediaType()); } finally { @@ -135,23 +133,23 @@ public void nonDynamicProviderResults() throws Exception{ String outName = Mock.ScriptGenerator.getOutputName(0); // undecorated output XProcOutput output = new XProcOutput.Builder().withOutput(outName, Mock.getResultProvider("foo.xml")).build(); - AbstractJob.buildResultSet(script, input, output, mapper); + script.buildResultSet(input, output, resultDir); } @Test public void optionsOutputFile() throws Exception{ QName optName=Mock.ScriptGenerator.getOptionOutputFileName(0); - JobResultSet rSet = AbstractJob.buildResultSet(script, input, output, mapper); + JobResultSet rSet = script.buildResultSet(input, output, resultDir); List jobs = Lists.newLinkedList(rSet.getResults(optName.getLocalPart())); Assert.assertEquals(1, jobs.size()); - Assert.assertEquals(mapper.mapOutput(URI.create("option-output-file-0.xml")), jobs.get(0).getPath().toURI()); + Assert.assertEquals(new File(resultDir, "option-output-file-0.xml"), jobs.get(0).getPath()); Assert.assertEquals("option-output-file-0.xml", jobs.get(0).getIdx().toString()); } @Test public void optionsOutputDirSize() throws Exception{ QName optName=Mock.ScriptGenerator.getOptionOutputDirName(0); - JobResultSet rSet = AbstractJob.buildResultSet(script, input, output, mapper); + JobResultSet rSet = script.buildResultSet(input, output, resultDir); List jobs=Lists.newLinkedList(rSet.getResults(optName.getLocalPart())); Assert.assertEquals(3,jobs.size()); } @@ -159,22 +157,22 @@ public void optionsOutputDirSize() throws Exception{ @Test public void optionsOutputURIs() throws Exception{ QName optName=Mock.ScriptGenerator.getOptionOutputDirName(0); - JobResultSet rSet = AbstractJob.buildResultSet(script, input, output, mapper); + JobResultSet rSet = script.buildResultSet(input, output, resultDir); List jobs=Lists.newLinkedList(rSet.getResults(optName.getLocalPart())); Assert.assertEquals(3, jobs.size()); - HashSet uris= new HashSet(); - uris.add(mapper.mapOutput(URI.create("option-output-dir-0/dos.xml"))); - uris.add(mapper.mapOutput(URI.create("option-output-dir-0/uno.xml"))); - uris.add(mapper.mapOutput(URI.create("option-output-dir-0/tres.xml"))); - Assert.assertTrue(uris.contains(jobs.get(0).getPath().toURI())); - Assert.assertTrue(uris.contains(jobs.get(1).getPath().toURI())); - Assert.assertTrue(uris.contains(jobs.get(2).getPath().toURI())); + HashSet uris= new HashSet<>(); + uris.add(new File(resultDir, "option-output-dir-0/dos.xml")); + uris.add(new File(resultDir, "option-output-dir-0/uno.xml")); + uris.add(new File(resultDir, "option-output-dir-0/tres.xml")); + Assert.assertTrue(uris.contains(jobs.get(0).getPath())); + Assert.assertTrue(uris.contains(jobs.get(1).getPath())); + Assert.assertTrue(uris.contains(jobs.get(2).getPath())); } @Test public void optionsOutputIdx() throws Exception{ QName optName=Mock.ScriptGenerator.getOptionOutputDirName(0); - JobResultSet rSet = AbstractJob.buildResultSet(script, input, output, mapper); + JobResultSet rSet = script.buildResultSet(input, output, resultDir); List jobs=Lists.newLinkedList(rSet.getResults(optName.getLocalPart())); Assert.assertEquals(3, jobs.size()); HashSet uris= new HashSet(); @@ -193,7 +191,7 @@ public void buildResultSet() throws Exception{ File f = null; try { f = writeResult(res.get()); - JobResultSet rSet = AbstractJob.buildResultSet(script, input, output, mapper); + JobResultSet rSet = script.buildResultSet(input, output, resultDir); Assert.assertEquals(5,rSet.getResults().size()); } finally { if (f != null) f.delete(); diff --git a/framework/framework-core/src/test/java/org/daisy/pipeline/job/StatusPortTest.java b/framework/framework-core/src/test/java/org/daisy/pipeline/script/StatusPortTest.java similarity index 84% rename from framework/framework-core/src/test/java/org/daisy/pipeline/job/StatusPortTest.java rename to framework/framework-core/src/test/java/org/daisy/pipeline/script/StatusPortTest.java index fca8fe7cae..a9e5e0006e 100644 --- a/framework/framework-core/src/test/java/org/daisy/pipeline/job/StatusPortTest.java +++ b/framework/framework-core/src/test/java/org/daisy/pipeline/script/StatusPortTest.java @@ -1,4 +1,4 @@ -package org.daisy.pipeline.job; +package org.daisy.pipeline.script; import java.io.IOException; import java.io.UnsupportedEncodingException; @@ -8,8 +8,7 @@ import org.daisy.common.xproc.XProcOutput; import org.daisy.pipeline.job.impl.Mock; -import org.daisy.pipeline.job.impl.XProcDecorator; -import org.daisy.pipeline.script.XProcScript; +import org.daisy.pipeline.script.impl.XProcDecorator; import org.junit.Assert; import org.junit.Before; @@ -42,41 +41,41 @@ void writeStatus(String statusXml) throws IOException, UnsupportedEncodingExcept @Test public void withoutStatusPort() { - boolean ok = AbstractJob.checkStatusPort(script, outputs); + boolean ok = script.checkStatusPort(outputs); Assert.assertTrue("should return true when no status port is present",ok); } @Test public void statusPortOk() throws IOException, UnsupportedEncodingException { writeStatus(XML_OK); - boolean ok = AbstractJob.checkStatusPort(script, outputs); + boolean ok = script.checkStatusPort(outputs); Assert.assertTrue("should return true when status document says 'ok'",ok); } @Test public void statusPortError() throws IOException, UnsupportedEncodingException { writeStatus(XML_ERR); - boolean ok = AbstractJob.checkStatusPort(script, outputs); + boolean ok = script.checkStatusPort(outputs); Assert.assertFalse("should return false when status document says 'error'",ok); } @Test(expected =RuntimeException.class) public void invalidStatusXml() throws IOException, UnsupportedEncodingException { writeStatus(XML_INVALID); - AbstractJob.checkStatusPort(script, outputs); + script.checkStatusPort(outputs); } @Test public void multipleStatusOk() throws IOException, UnsupportedEncodingException { writeStatus(XML_OK); writeStatus(XML_OK); - boolean ok = AbstractJob.checkStatusPort(script, outputs); + boolean ok = script.checkStatusPort(outputs); Assert.assertTrue("should return true if all status documents say 'ok'",ok); } @Test public void multipleStatusErr() throws IOException, UnsupportedEncodingException { writeStatus(XML_OK); writeStatus(XML_ERR); - boolean ok = AbstractJob.checkStatusPort(script, outputs); + boolean ok = script.checkStatusPort(outputs); Assert.assertFalse("should return false if at least one status documents say 'error'",ok); } } diff --git a/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/DynamicResultProviderPartsTest.java b/framework/framework-core/src/test/java/org/daisy/pipeline/script/impl/DynamicResultProviderPartsTest.java similarity index 97% rename from framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/DynamicResultProviderPartsTest.java rename to framework/framework-core/src/test/java/org/daisy/pipeline/script/impl/DynamicResultProviderPartsTest.java index 5aae7475a6..904bad3416 100644 --- a/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/DynamicResultProviderPartsTest.java +++ b/framework/framework-core/src/test/java/org/daisy/pipeline/script/impl/DynamicResultProviderPartsTest.java @@ -1,7 +1,8 @@ -package org.daisy.pipeline.job.impl; +package org.daisy.pipeline.script.impl; import javax.xml.namespace.QName; +import org.daisy.pipeline.job.impl.Mock; import org.daisy.pipeline.script.Script; import org.junit.Assert; diff --git a/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/DynamicResultProviderTest.java b/framework/framework-core/src/test/java/org/daisy/pipeline/script/impl/DynamicResultProviderTest.java similarity index 65% rename from framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/DynamicResultProviderTest.java rename to framework/framework-core/src/test/java/org/daisy/pipeline/script/impl/DynamicResultProviderTest.java index f1038d178b..9a650d8d0b 100644 --- a/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/DynamicResultProviderTest.java +++ b/framework/framework-core/src/test/java/org/daisy/pipeline/script/impl/DynamicResultProviderTest.java @@ -1,35 +1,36 @@ -package org.daisy.pipeline.job.impl; +package org.daisy.pipeline.script.impl; import java.io.File; import java.net.URI; -import org.daisy.pipeline.job.impl.DynamicResultProvider; -import org.daisy.pipeline.job.URIMapper; +import javax.xml.transform.Result; + +import org.daisy.pipeline.job.impl.Mock; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import javax.xml.transform.Result; - public class DynamicResultProviderTest { + DynamicResultProvider provider; + @Before public void setUp(){ - provider = new DynamicResultProvider(Mock.getResultProvider("/tmp/file.xml"), "irrelevant", "irrelevant", new URIMapper(null, URI.create(""))); + provider = new DynamicResultProvider(Mock.getResultProvider("/tmp/file.xml"), "irrelevant", "irrelevant", new File("/")); } @Test public void testGenerateFirst(){ Result result=provider.get(); - Assert.assertEquals("/tmp/file.xml",result.getSystemId()); + Assert.assertEquals("file:/tmp/file.xml", result.getSystemId()); } @Test public void testGenerateSecond(){ Result result=provider.get(); result=provider.get(); - Assert.assertEquals("/tmp/file-1.xml",result.getSystemId()); + Assert.assertEquals("file:/tmp/file-1.xml", result.getSystemId()); } @Test(expected=UnsupportedOperationException.class) diff --git a/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/XProcDecoratorTest.java b/framework/framework-core/src/test/java/org/daisy/pipeline/script/impl/XProcDecoratorTest.java similarity index 76% rename from framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/XProcDecoratorTest.java rename to framework/framework-core/src/test/java/org/daisy/pipeline/script/impl/XProcDecoratorTest.java index e652e8ef5d..ac3951cab4 100644 --- a/framework/framework-core/src/test/java/org/daisy/pipeline/job/impl/XProcDecoratorTest.java +++ b/framework/framework-core/src/test/java/org/daisy/pipeline/script/impl/XProcDecoratorTest.java @@ -1,5 +1,6 @@ -package org.daisy.pipeline.job.impl; +package org.daisy.pipeline.script.impl; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -12,14 +13,18 @@ import javax.xml.transform.sax.SAXSource; import javax.xml.transform.Source; +import org.apache.commons.io.FileUtils; + import org.daisy.common.xproc.XProcInput; import org.daisy.common.xproc.XProcOutput; +import org.daisy.pipeline.job.impl.IOHelper; +import org.daisy.pipeline.job.impl.Mock; import org.daisy.pipeline.job.JobResources; -import org.daisy.pipeline.job.URIMapper; import org.daisy.pipeline.script.ScriptInput; import org.daisy.pipeline.script.XProcOptionMetadata; import org.daisy.pipeline.script.XProcScript; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -30,22 +35,38 @@ import com.google.common.collect.Lists; public class XProcDecoratorTest { - URIMapper mapper; - String testFile="dir/myfile.xml"; - String testFile2="dir/myfile2.xml"; + + String testFile = "dir/myfile.xml"; + String testFile2 = "dir/myfile2.xml"; + File tmpDir = new File(System.getProperty("java.io.tmpdir")); + File contextDir = new File(tmpDir, "inputs"); + File resultDir = new File(tmpDir, "outputs"); JobResources resources = new JobResources() { public Iterable getNames() { return Lists.newArrayList(testFile, testFile2); } public Supplier getResource(String name) { - throw new UnsupportedOperationException(); + return () -> new ByteArrayInputStream("".getBytes()); }}; - String testDir="dir"; + String testDir = "dir"; + @Before - public void setUp() throws IOException { + public void setup() { + try { + IOHelper.makeDirs(resultDir); + } catch (IOException e) { + throw new RuntimeException(e); + } + } - URI tmp=new File(System.getProperty("java.io.tmpdir")).toURI(); - mapper = new URIMapper(tmp.resolve("inputs/"),tmp.resolve("outputs/")); + @After + public void tearDown() { + try { + FileUtils.deleteDirectory(contextDir); + FileUtils.deleteDirectory(resultDir); + } catch (IOException e) { + throw new RuntimeException(e); + } } @Test @@ -68,20 +89,18 @@ public void testResolveInputPorts() throws IOException { ScriptInput input = new ScriptInput.Builder(resources) .withInput(optName, src) .withInput(optName, src2) - .build(); - + .build() + .storeToDisk(contextDir); XProcInput.Builder builder = new XProcInput.Builder(); - XProcDecorator trans=XProcDecorator.from(mscript, mapper, null); + XProcDecorator trans = XProcDecorator.from(mscript, resultDir, null); trans.decorateInputPorts(mscript,input,builder); - XProcInput newInput = builder.build(); List> sources = Lists.newLinkedList(newInput.getInputs(optName)); URI res1 = URI.create(sources.get(0).get().getSystemId()); - URI expected=URI.create(mapper.getInputBase().toString()+testFile); + URI expected = URI.create(contextDir.toURI().toString()+testFile); Assert.assertEquals(res1,expected); - URI res2 = URI.create(sources.get(1).get().getSystemId()); - URI expected2=URI.create(mapper.getInputBase().toString()+testFile2); + URI expected2=URI.create(contextDir.toURI().toString()+testFile2); Assert.assertEquals(res2,expected2); } @@ -94,22 +113,19 @@ public void testResolveInputPortGenerated() throws IOException { ScriptInput input = new ScriptInput.Builder() .withInput(optName, new SAXSource(new InputSource(new StringReader("foo")))) .withInput(optName, new SAXSource(new InputSource(new StringReader("bar")))) - .build(); - + .build() + .storeToDisk(contextDir); XProcInput.Builder builder = new XProcInput.Builder(); - XProcDecorator trans=XProcDecorator.from(mscript, mapper, null); + XProcDecorator trans = XProcDecorator.from(mscript, resultDir, null); trans.decorateInputPorts(mscript,input,builder); - XProcInput newInput = builder.build(); List> sources = Lists.newLinkedList(newInput.getInputs(optName)); URI res1 = URI.create(sources.get(0).get().getSystemId()); - URI expected=URI.create(mapper.getInputBase()+optName+"-0.xml"); + URI expected=URI.create(contextDir.toURI() + optName + "-0.xml"); Assert.assertEquals(res1,expected); - URI res2 = URI.create(sources.get(1).get().getSystemId()); - URI expected2=URI.create(mapper.getInputBase()+optName+"-1.xml"); + URI expected2=URI.create(contextDir.toURI() + optName + "-1.xml"); Assert.assertEquals(res2,expected2); - } @Test @@ -118,14 +134,16 @@ public void testResolveOptionsInput() throws IOException { XProcScript mscript = new Mock.ScriptGenerator.Builder().withOptionInputs(1).build().generate(); //adding a value to the input option QName optName=Mock.ScriptGenerator.getOptionInputName(0); - ScriptInput input = new ScriptInput.Builder(resources).withInput(optName.getLocalPart(), Mock.getSource(testFile)).build(); + ScriptInput input = new ScriptInput.Builder(resources) + .withInput(optName.getLocalPart(), Mock.getSource(testFile)) + .build() + .storeToDisk(contextDir); XProcInput.Builder builder = new XProcInput.Builder(); - XProcDecorator trans=XProcDecorator.from(mscript, mapper, null); + XProcDecorator trans = XProcDecorator.from(mscript, resultDir, null); trans.decorateInputOptions(mscript, input, builder); - XProcInput newInput = builder.build(); URI res1 = URI.create((String)newInput.getOptions().get(optName)); - URI expected=URI.create(mapper.getInputBase()+testFile); + URI expected=URI.create(contextDir.toURI() + testFile); Assert.assertEquals(expected, res1); } @@ -138,123 +156,101 @@ public void testResolveOptionsInputSequence() throws IOException { ScriptInput input = new ScriptInput.Builder(resources) .withInput(optName.getLocalPart(), Mock.getSource(testFile)) .withInput(optName.getLocalPart(), Mock.getSource(testFile2)) - .build(); - + .build() + .storeToDisk(contextDir); XProcInput.Builder builder = new XProcInput.Builder(); - XProcDecorator trans=XProcDecorator.from(mscript, mapper, null); + XProcDecorator trans = XProcDecorator.from(mscript, resultDir, null); trans.decorateInputOptions(mscript, input, builder); - XProcInput newInput = builder.build(); String res= (String)newInput.getOptions().get(optName); - String expected1=URI.create(mapper.getInputBase()+testFile).toString(); - String expected2=URI.create(mapper.getInputBase()+testFile2).toString(); + String expected1=URI.create(contextDir.toURI() + testFile).toString(); + String expected2=URI.create(contextDir.toURI() + testFile2).toString(); Assert.assertEquals(res, expected1 + XProcOptionMetadata.DEFAULT_SEPARATOR + expected2); } @Test public void testResolveOptionsInputEmpty() throws IOException { //it should just ignore them, rite? - XProcScript script = new Mock.ScriptGenerator.Builder().withOptionInputs(1).build().generate(); //no settings for the input QName optName=Mock.ScriptGenerator.getOptionInputName(0); ScriptInput input = new ScriptInput.Builder().build(); - XProcInput.Builder builder = new XProcInput.Builder(); - XProcDecorator trans=XProcDecorator.from(script, mapper, null); + XProcDecorator trans = XProcDecorator.from(script, resultDir, null); trans.decorateInputOptions(script, input, builder); - XProcInput newInput = builder.build(); - Assert.assertEquals("",newInput.getOptions().get(optName)); } @Test public void testResolveOptionsOutputsCopy() throws IOException { - XProcScript script = new Mock.ScriptGenerator.Builder().withOptionOther(1).build().generate(); QName optName=Mock.ScriptGenerator.getRegularOptionName(0); //adding a value to the input option ScriptInput input = new ScriptInput.Builder().withOption(optName.getLocalPart(), "cosa").build(); - XProcInput.Builder builder = new XProcInput.Builder(); - XProcDecorator trans=XProcDecorator.from(script, mapper, null); + XProcDecorator trans = XProcDecorator.from(script, resultDir, null); trans.decorateInputOptions(script, input, builder); - XProcInput newInput = builder.build(); - Assert.assertEquals("cosa",newInput.getOptions().get(optName)); } @Test public void testResolveOptionsOutputsFile() throws IOException { - XProcScript script = new Mock.ScriptGenerator.Builder().withOptionOutputsFile(1).build().generate(); QName optName=Mock.ScriptGenerator.getOptionOutputFileName(0); // adding a value to the input does not have an effect ScriptInput input = new ScriptInput.Builder().withOption(optName.getLocalPart(), testFile).build(); - XProcInput.Builder builder = new XProcInput.Builder(); - XProcDecorator trans=XProcDecorator.from(script, mapper, null); + XProcDecorator trans = XProcDecorator.from(script, resultDir, null); trans.decorateOutputOptions(script, input, builder); - XProcInput newInput = builder.build(); - - URI expected = URI.create(mapper.getOutputBase() + "option-output-file-0.xml"); + URI expected = URI.create(resultDir.toURI() + "option-output-file-0.xml"); URI reslut=URI.create( (String)newInput.getOptions().get(optName) ); Assert.assertEquals(expected,reslut); } @Test public void testResolveOptionsOutputsDir() throws IOException { - XProcScript script = new Mock.ScriptGenerator.Builder().withOptionOutputsDir(1).build().generate(); QName optName=Mock.ScriptGenerator.getOptionOutputDirName(0); // adding a value to the input does not have an effect ScriptInput input = new ScriptInput.Builder().withOption(optName.getLocalPart(), testDir).build(); XProcInput.Builder builder = new XProcInput.Builder(); - XProcDecorator trans=XProcDecorator.from(script, mapper, null); + XProcDecorator trans = XProcDecorator.from(script, resultDir, null); trans.decorateOutputOptions(script, input, builder); - XProcInput newInput = builder.build(); - - URI expected = URI.create(mapper.getOutputBase() + "option-output-dir-0/"); + URI expected = URI.create(resultDir.toURI() + "option-output-dir-0/"); URI reslut=URI.create( (String)newInput.getOptions().get(optName) ); Assert.assertEquals(expected,reslut); } @Test public void testResolveOptionsOutputsGeneratedFile() throws IOException { - XProcScript script = new Mock.ScriptGenerator.Builder().withOptionOutputsFile(1).build().generate(); QName optName=Mock.ScriptGenerator.getOptionOutputFileName(0); //adding a value to the input option ScriptInput input = new ScriptInput.Builder().withOption(optName.getLocalPart(), "").build(); - XProcInput.Builder builder = new XProcInput.Builder(); - XProcDecorator trans=XProcDecorator.from(script, mapper, null); + XProcDecorator trans = XProcDecorator.from(script, resultDir, null); trans.decorateOutputOptions(script, input, builder); - XProcInput newInput = builder.build(); - URI expected = URI.create(mapper.getOutputBase() + "option-output-file-0.xml"); + URI expected = URI.create(resultDir.toURI() + "option-output-file-0.xml"); URI reslut=URI.create( (String)newInput.getOptions().get(optName) ); Assert.assertEquals(expected,reslut); } @Test public void testResolveOptionsOutputsGeneratedDir() throws IOException { - XProcScript script = new Mock.ScriptGenerator.Builder().withOptionOutputsDir(1).build().generate(); QName optName=Mock.ScriptGenerator.getOptionOutputDirName(0); //adding a value to the input option ScriptInput input = new ScriptInput.Builder().withOption(optName.getLocalPart(), "").build(); - XProcInput.Builder builder = new XProcInput.Builder(); - XProcDecorator trans=XProcDecorator.from(script, mapper, null); + XProcDecorator trans = XProcDecorator.from(script, resultDir, null); trans.decorateOutputOptions(script, input, builder); - XProcInput newInput = builder.build(); - URI expected = URI.create(mapper.getOutputBase() + "option-output-dir-0/"); + URI expected = URI.create(resultDir.toURI() + "option-output-dir-0/"); URI reslut=URI.create( (String)newInput.getOptions().get(optName) ); Assert.assertEquals(expected,reslut); } @@ -272,72 +268,57 @@ public void translateInputs() throws Exception { QName optReg = Mock.ScriptGenerator.getRegularOptionName(0); QName optOutFile = Mock.ScriptGenerator.getOptionOutputFileName(0); QName optOutDir = Mock.ScriptGenerator.getOptionOutputDirName(0); - ScriptInput input = new ScriptInput.Builder(resources) .withInput(optIn.getLocalPart(), Mock.getSource(testFile)) .withOption(optReg.getLocalPart(), "value") .withOption(optOutFile.getLocalPart(), "dir/output.xml") // no effect .withOption(optOutDir.getLocalPart(), "outs") // no effect - .build(); - - XProcDecorator trans=XProcDecorator.from(script, mapper, null); + .build() + .storeToDisk(contextDir); + XProcDecorator trans = XProcDecorator.from(script, resultDir, null); XProcInput iTrans=trans.decorate(input); - - Assert.assertEquals(iTrans.getOptions().get(optIn), mapper.getInputBase() + testFile); - Assert.assertEquals(iTrans.getOptions().get(optOutFile), mapper.getOutputBase() + "option-output-file-0.xml"); - Assert.assertEquals(iTrans.getOptions().get(optOutDir), mapper.getOutputBase() + "option-output-dir-0/"); + Assert.assertEquals(iTrans.getOptions().get(optIn), contextDir.toURI() + testFile); + Assert.assertEquals(iTrans.getOptions().get(optOutFile), resultDir.toURI() + "option-output-file-0.xml"); + Assert.assertEquals(iTrans.getOptions().get(optOutDir), resultDir.toURI() + "option-output-dir-0/"); Assert.assertEquals(iTrans.getOptions().get(optReg),"value"); - } @Test public void ouputPortFile() throws Exception{ XProcScript script = new Mock.ScriptGenerator.Builder().withOutputPorts(1).build().generate(); String outName = Mock.ScriptGenerator.getOutputName(0); - XProcOutput outs = new XProcOutput.Builder().withOutput(outName,Mock.getResultProvider("dir/file.xml")).build(); - XProcDecorator trans=XProcDecorator.from(script, mapper, null); + XProcDecorator trans = XProcDecorator.from(script, resultDir, null); XProcOutput decorated=trans.decorate(outs); - Supplier res=decorated.getResultProvider(outName); - String expected=(mapper.getOutputBase()+"dir/file.xml"); - + String expected=(resultDir.toURI() + "dir/file.xml"); Assert.assertEquals(expected.toString(),res.get().getSystemId()); - - } @Test public void ouputSeqPortFiles() throws Exception{ XProcScript script = new Mock.ScriptGenerator.Builder().withOutputPorts(1).build().generate(); String outName = Mock.ScriptGenerator.getOutputName(0); - XProcOutput outs = new XProcOutput.Builder().withOutput(outName,Mock.getResultProvider("dir/file.xml")).build(); - XProcDecorator trans=XProcDecorator.from(script, mapper, null); + XProcDecorator trans = XProcDecorator.from(script, resultDir, null); XProcOutput decorated=trans.decorate(outs); - Supplier res=decorated.getResultProvider(outName); - String expected2=(mapper.getOutputBase()+"dir/file-1.xml"); - + String expected2=(resultDir.toURI() + "dir/file-1.xml"); //Assert.assertEquals(expected.toString(),res.provide().getSystemId()); //discard one res.get(); Assert.assertEquals(expected2.toString(),res.get().getSystemId()); - } @Test public void ouputPortDir() throws Exception{ XProcScript script = new Mock.ScriptGenerator.Builder().withOutputPorts(1).build().generate(); String outName = Mock.ScriptGenerator.getOutputName(0); - XProcOutput outs = new XProcOutput.Builder().withOutput(outName,Mock.getResultProvider("dir/")).build(); - XProcDecorator trans=XProcDecorator.from(script, mapper, null); + XProcDecorator trans = XProcDecorator.from(script, resultDir, null); XProcOutput decorated=trans.decorate(outs); - Supplier res=decorated.getResultProvider(outName); - String expected=(mapper.getOutputBase()+"dir/"+outName+".xml"); - + String expected=(resultDir.toURI() + "dir/" + outName + ".xml"); Assert.assertEquals(expected.toString(),res.get().getSystemId()); } @@ -345,13 +326,11 @@ public void ouputPortDir() throws Exception{ public void ouputSeqPortDir() throws Exception{ XProcScript script = new Mock.ScriptGenerator.Builder().withOutputPorts(1).build().generate(); String outName = Mock.ScriptGenerator.getOutputName(0); - XProcOutput outs = new XProcOutput.Builder().withOutput(outName,Mock.getResultProvider("dir/")).build(); - XProcDecorator trans=XProcDecorator.from(script, mapper, null); + XProcDecorator trans = XProcDecorator.from(script, resultDir, null); XProcOutput decorated=trans.decorate(outs); - Supplier res=decorated.getResultProvider(outName); - String expected=(mapper.getOutputBase()+"dir/"+outName+"-1.xml"); + String expected=(resultDir.toURI() + "dir/" + outName + "-1.xml"); //discard first res.get(); Assert.assertEquals(expected.toString(),res.get().getSystemId()); @@ -361,14 +340,11 @@ public void ouputSeqPortDir() throws Exception{ public void ouputPortEmptyString() throws Exception{ XProcScript script = new Mock.ScriptGenerator.Builder().withOutputPorts(1).build().generate(); String outName = Mock.ScriptGenerator.getOutputName(0); - XProcOutput outs = new XProcOutput.Builder().withOutput(outName,null).build(); - XProcDecorator trans=XProcDecorator.from(script, mapper, null); + XProcDecorator trans = XProcDecorator.from(script, resultDir, null); XProcOutput decorated=trans.decorate(outs); - Supplier res=decorated.getResultProvider(outName); - String expected=(mapper.getOutputBase()+outName+"/"+outName+".xml"); - + String expected=(resultDir.toURI() + outName + "/" +outName + ".xml"); Assert.assertEquals(expected.toString(),res.get().getSystemId()); } @@ -376,14 +352,11 @@ public void ouputPortEmptyString() throws Exception{ public void ouputPortEmptyNull() throws Exception{ XProcScript script = new Mock.ScriptGenerator.Builder().withOutputPorts(1).build().generate(); String outName = Mock.ScriptGenerator.getOutputName(0); - XProcOutput outs = new XProcOutput.Builder().build(); - XProcDecorator trans=XProcDecorator.from(script, mapper, null); + XProcDecorator trans = XProcDecorator.from(script, resultDir, null); XProcOutput decorated=trans.decorate(outs); - Supplier res=decorated.getResultProvider(outName); - String expected=(mapper.getOutputBase()+outName+"/"+outName+".xml"); - + String expected=(resultDir.toURI() + outName + "/" + outName + ".xml"); Assert.assertEquals(expected.toString(),res.get().getSystemId()); } @@ -391,13 +364,11 @@ public void ouputPortEmptyNull() throws Exception{ public void ouputSeqPortEmptyNull() throws Exception{ XProcScript script = new Mock.ScriptGenerator.Builder().withOutputPorts(1).build().generate(); String outName = Mock.ScriptGenerator.getOutputName(0); - XProcOutput outs = new XProcOutput.Builder().build(); - XProcDecorator trans=XProcDecorator.from(script, mapper, null); + XProcDecorator trans = XProcDecorator.from(script, resultDir, null); XProcOutput decorated=trans.decorate(outs); - Supplier res=decorated.getResultProvider(outName); - String expected=(mapper.getOutputBase()+outName+"/"+outName+"-1.xml"); + String expected=(resultDir.toURI() + outName + "/" + outName + "-1.xml"); res.get(); Assert.assertEquals(expected.toString(),res.get().getSystemId()); } @@ -406,13 +377,11 @@ public void ouputSeqPortEmptyNull() throws Exception{ public void ouputPortSeqEmptyString() throws Exception{ XProcScript script = new Mock.ScriptGenerator.Builder().withOutputPorts(1).build().generate(); String outName = Mock.ScriptGenerator.getOutputName(0); - XProcOutput outs = new XProcOutput.Builder().withOutput(outName,null).build(); - XProcDecorator trans=XProcDecorator.from(script, mapper, null); + XProcDecorator trans = XProcDecorator.from(script, resultDir, null); XProcOutput decorated=trans.decorate(outs); - Supplier res=decorated.getResultProvider(outName); - String expected=(mapper.getOutputBase()+outName+"/"+outName+"-1.xml"); + String expected=(resultDir.toURI() + outName + "/" + outName + "-1.xml"); res.get(); Assert.assertEquals(expected.toString(),res.get().getSystemId()); } diff --git a/framework/framework-core/src/test/resources/OSGI-INF/xproc-engine.xml b/framework/framework-core/src/test/resources/OSGI-INF/xproc-engine.xml new file mode 100644 index 0000000000..a882776994 --- /dev/null +++ b/framework/framework-core/src/test/resources/OSGI-INF/xproc-engine.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/framework/framework-persistence/pom.xml b/framework/framework-persistence/pom.xml index ed0fd18a6f..26e74f6b27 100644 --- a/framework/framework-persistence/pom.xml +++ b/framework/framework-persistence/pom.xml @@ -4,11 +4,11 @@ org.daisy.pipeline framework-parent - 1.14.21-SNAPSHOT + 1.14.22-SNAPSHOT ../parent framework-persistence - 2.1.11-SNAPSHOT + 2.1.12-SNAPSHOT bundle UTF-8 diff --git a/framework/framework-persistence/src/main/java/org/daisy/pipeline/persistence/impl/job/PersistentJob.java b/framework/framework-persistence/src/main/java/org/daisy/pipeline/persistence/impl/job/PersistentJob.java index abfe5949f7..7251af0024 100644 --- a/framework/framework-persistence/src/main/java/org/daisy/pipeline/persistence/impl/job/PersistentJob.java +++ b/framework/framework-persistence/src/main/java/org/daisy/pipeline/persistence/impl/job/PersistentJob.java @@ -61,7 +61,7 @@ public class PersistentJob extends AbstractJob implements Serializable { * Constructs a new instance. */ private PersistentJob() { - super(null, null, null, null, true); + super(null, null, true); } @Enumerated(EnumType.ORDINAL) diff --git a/framework/framework-persistence/src/main/java/org/daisy/pipeline/persistence/impl/job/PersistentJobContext.java b/framework/framework-persistence/src/main/java/org/daisy/pipeline/persistence/impl/job/PersistentJobContext.java index 64f7347785..fddb9ddc95 100644 --- a/framework/framework-persistence/src/main/java/org/daisy/pipeline/persistence/impl/job/PersistentJobContext.java +++ b/framework/framework-persistence/src/main/java/org/daisy/pipeline/persistence/impl/job/PersistentJobContext.java @@ -27,7 +27,6 @@ import org.daisy.pipeline.job.JobIdFactory; import org.daisy.pipeline.job.JobMonitorFactory; import org.daisy.pipeline.job.JobResultSet; -import org.daisy.pipeline.job.URIMapper; import org.daisy.pipeline.persistence.impl.webservice.PersistentClient; import org.daisy.pipeline.persistence.impl.webservice.PersistentClientStorage; import org.daisy.pipeline.script.ScriptInput; @@ -55,7 +54,6 @@ public final class PersistentJobContext extends AbstractJobContext { private String scriptId = null; - //embedded mapper @Embedded private PersistentMapper pMapper; @ManyToOne @@ -96,7 +94,7 @@ public final class PersistentJobContext extends AbstractJobContext { super(ctxt); // Map complex objects to their Persistent representation logger.debug("coping the objects to the model "); - this.pMapper = new PersistentMapper(this.uriMapper); + this.pMapper = new PersistentMapper(this.resultDir, this.input.getResources()); this.inputPorts = ContextHydrator.dehydrateInputPorts(this.getId(), this.getScript(), this.input); this.options = ContextHydrator.dehydrateOptions(this.getId(), this.getScript(), this.input); if (this.getClient() instanceof PersistentClient) @@ -130,9 +128,9 @@ private PersistentJobContext() { private void postLoad(){ logger.debug("Post loading jobcontext"); //we have all the model but we have to hidrate the actual objects - this.uriMapper = this.pMapper.getMapper(); + this.resultDir = new File(URI.create(this.pMapper.outputBase)); File contextDir = null; { - URI u = uriMapper.getInputBase(); + URI u = URI.create(this.pMapper.inputBase); if (u != null && !"".equals(u.toString())) { try { contextDir = new File(u); @@ -251,7 +249,11 @@ ScriptInput getInput() { return input; } - URIMapper getResultMapper() { - return uriMapper; + File getResultDir() { + return resultDir; + } + + File getContextDir() { + return new File(URI.create(pMapper.inputBase)); } } diff --git a/framework/framework-persistence/src/main/java/org/daisy/pipeline/persistence/impl/job/PersistentMapper.java b/framework/framework-persistence/src/main/java/org/daisy/pipeline/persistence/impl/job/PersistentMapper.java index 4fd82e0211..d0737cf95b 100644 --- a/framework/framework-persistence/src/main/java/org/daisy/pipeline/persistence/impl/job/PersistentMapper.java +++ b/framework/framework-persistence/src/main/java/org/daisy/pipeline/persistence/impl/job/PersistentMapper.java @@ -1,10 +1,11 @@ package org.daisy.pipeline.persistence.impl.job; -import java.net.URI; +import java.io.File; import javax.persistence.Embeddable; -import org.daisy.pipeline.job.URIMapper; +import org.daisy.pipeline.job.JobResources; +import org.daisy.pipeline.job.JobResourcesDir; @Embeddable public class PersistentMapper { @@ -12,22 +13,17 @@ public class PersistentMapper { String inputBase; String outputBase; - /** - * Constructs a new instance. - */ public PersistentMapper() { - } - - public PersistentMapper(URIMapper mapper){ - setMapper(mapper); - } - - public URIMapper getMapper(){ - return new URIMapper(URI.create(this.inputBase),URI.create(this.outputBase)); } - public void setMapper(URIMapper mapper){ - this.inputBase=mapper.getInputBase().toString(); - this.outputBase=mapper.getOutputBase().toString(); + public PersistentMapper(File resultDir, JobResources resources) { + this.outputBase = resultDir.toURI().toString(); + if (resources == null) + this.inputBase = ""; + else { + if (!(resources instanceof JobResourcesDir)) + throw new IllegalArgumentException(); // could happen if ScriptInput.storeToDisk was not called + this.inputBase = ((JobResourcesDir)resources).getBaseDir().toURI().toString(); + } } } diff --git a/framework/framework-persistence/src/test/java/org/daisy/pipeline/persistence/impl/job/Mocks.java b/framework/framework-persistence/src/test/java/org/daisy/pipeline/persistence/impl/job/Mocks.java index a1adb86976..152f9c7127 100644 --- a/framework/framework-persistence/src/test/java/org/daisy/pipeline/persistence/impl/job/Mocks.java +++ b/framework/framework-persistence/src/test/java/org/daisy/pipeline/persistence/impl/job/Mocks.java @@ -26,7 +26,6 @@ import org.daisy.pipeline.job.JobResult; import org.daisy.pipeline.job.JobResultSet; import org.daisy.pipeline.job.StatusNotifier; -import org.daisy.pipeline.job.URIMapper; import org.daisy.pipeline.job.impl.IOHelper; import org.daisy.pipeline.persistence.impl.webservice.PersistentClient; import org.daisy.pipeline.script.Script; @@ -53,7 +52,7 @@ public class Mocks { public static final String value1 = "value1"; public static final File result1; public static final File result2; - public static final URI out = URI.create("file:/tmp/out/"); + public static final File out = new File("/tmp/out/"); public static final String portResult="res"; static { @@ -121,7 +120,8 @@ public void setSystemId(String systemId) { } public static XProcScript buildScript(){ - XProcScript.Builder builder = new XProcScript.Builder(Mocks.scriptId, "", URI.create(Mocks.scriptUri), null); + XProcScript.Builder builder = new XProcScript.Builder(Mocks.scriptId, "", URI.create(Mocks.scriptUri), + null, null, null); builder = builder.withInputPort(XProcPortInfo.newInputPort("source", true, false, true), new XProcPortMetadata("", "", "")); builder = builder.withOutputPort(XProcPortInfo.newOutputPort(portResult, true, true), @@ -138,31 +138,31 @@ public static AbstractJob buildJob() { } public static AbstractJob buildJob(File contextDir) { - return new AbstractJob(buildContext(contextDir), Priority.MEDIUM, null, null, true) {}; + return new AbstractJob(buildContext(contextDir), Priority.MEDIUM, true) {}; } public static AbstractJob buildJob(Priority priority) { - return new AbstractJob(buildContext(), priority, null, null, true) {}; + return new AbstractJob(buildContext(), priority, true) {}; } public static AbstractJob buildJob(Priority priority, File contextDir) { - return new AbstractJob(buildContext(contextDir), priority, null, null, true) {}; + return new AbstractJob(buildContext(contextDir), priority, true) {}; } public static AbstractJob buildJob(Client client) { - return new AbstractJob(buildContext(client), Priority.MEDIUM, null, null, true) {}; + return new AbstractJob(buildContext(client), Priority.MEDIUM, true) {}; } public static AbstractJob buildJob(Client client, File contextDir) { - return new AbstractJob(buildContext(client, null, contextDir), Priority.MEDIUM, null, null, true) {}; + return new AbstractJob(buildContext(client, null, contextDir), Priority.MEDIUM, true) {}; } public static AbstractJob buildJob(Client client, JobBatchId batchId) { - return new AbstractJob(buildContext(client, batchId), Priority.MEDIUM, null, null, true) {}; + return new AbstractJob(buildContext(client, batchId), Priority.MEDIUM, true) {}; } public static AbstractJob buildJob(Client client, JobBatchId batchId, File contextDir) { - return new AbstractJob(buildContext(client, batchId, contextDir), Priority.MEDIUM, null, null, true) {}; + return new AbstractJob(buildContext(client, batchId, contextDir), Priority.MEDIUM, true) {}; } public static AbstractJobContext buildContext() { @@ -191,7 +191,7 @@ public Supplier getResource(String name) { return () -> new ByteArrayInputStream("foo".getBytes()); } }; - final ScriptInput input; + ScriptInput input; try { input = new ScriptInput.Builder(resources).withInput("source", new Mocks.SimpleSourceProvider(file1)) .withInput("source", new Mocks.SimpleSourceProvider(file2)) @@ -201,10 +201,9 @@ public Supplier getResource(String name) { throw new RuntimeException(e); } final JobId id = JobIdFactory.newId(); - final URIMapper mapper= new URIMapper(contextDir != null ? contextDir.toURI() : URI.create(""), out); if (contextDir != null) try { - IOHelper.dump(resources, mapper); + input = input.storeToDisk(contextDir); } catch (IOException e) { throw new RuntimeException(e); } @@ -218,11 +217,11 @@ public Supplier getResource(String name) { DatabaseProvider.getDatabase().addObject(client); } //inception! - return new MyHiddenContext(rSet,script,input,mapper,client,id,batchId); + return new MyHiddenContext(rSet,script,input,out,client,id,batchId); } static class MyHiddenContext extends AbstractJobContext{ - public MyHiddenContext(JobResultSet set, Script script, ScriptInput input, URIMapper mapper, Client client, JobId id, JobBatchId batchId){ + public MyHiddenContext(JobResultSet set, Script script, ScriptInput input, File resultDir, Client client, JobId id, JobBatchId batchId){ super(); this.client = client; this.id = id; @@ -231,7 +230,7 @@ public MyHiddenContext(JobResultSet set, Script script, ScriptInput input, URIMa this.niceName = "hidden"; this.script = script; this.input = input; - this.uriMapper = mapper; + this.resultDir = resultDir; this.results = set; this.monitor = new JobMonitor() { @Override diff --git a/framework/framework-persistence/src/test/java/org/daisy/pipeline/persistence/impl/job/PersistentJobContextTest.java b/framework/framework-persistence/src/test/java/org/daisy/pipeline/persistence/impl/job/PersistentJobContextTest.java index 51dbec2854..8ad3498548 100644 --- a/framework/framework-persistence/src/test/java/org/daisy/pipeline/persistence/impl/job/PersistentJobContextTest.java +++ b/framework/framework-persistence/src/test/java/org/daisy/pipeline/persistence/impl/job/PersistentJobContextTest.java @@ -18,7 +18,6 @@ import org.daisy.pipeline.job.JobId; import org.daisy.pipeline.job.JobIdFactory; import org.daisy.pipeline.job.JobResult; -import org.daisy.pipeline.job.URIMapper; import org.daisy.pipeline.persistence.impl.Database; import org.daisy.pipeline.script.ScriptInput; import org.daisy.pipeline.script.ScriptRegistry; @@ -101,7 +100,8 @@ public void optionTest(){ @Test public void mapperTest(){ PersistentJobContext jCtxt= db.getEntityManager().find(PersistentJobContext.class, id.toString()); - Assert.assertEquals(jCtxt.getResultMapper(), new URIMapper(tempDir.toURI(), Mocks.out)); + Assert.assertEquals(jCtxt.getResultDir(), Mocks.out); + Assert.assertEquals(jCtxt.getContextDir(), tempDir); } @Test diff --git a/framework/logging-appender/pom.xml b/framework/logging-appender/pom.xml index 54b6cfb1e5..7315d8e26b 100644 --- a/framework/logging-appender/pom.xml +++ b/framework/logging-appender/pom.xml @@ -6,12 +6,12 @@ org.daisy.pipeline framework-parent - 1.14.21-SNAPSHOT + 1.14.22-SNAPSHOT ../parent logging-appender - 2.1.5-SNAPSHOT + 2.1.6-SNAPSHOT bundle DAISY Pipeline 2 :: Logging Appender diff --git a/framework/modules-registry/pom.xml b/framework/modules-registry/pom.xml index bf20eb81a7..1fb629694e 100644 --- a/framework/modules-registry/pom.xml +++ b/framework/modules-registry/pom.xml @@ -6,12 +6,12 @@ org.daisy.pipeline framework-parent - 1.14.21-SNAPSHOT + 1.14.22-SNAPSHOT ../parent modules-registry - 5.0.0-SNAPSHOT + 5.0.1-SNAPSHOT bundle DAISY Pipeline 2 :: Module Registry diff --git a/framework/parent/pom.xml b/framework/parent/pom.xml index 9adff035f4..b5e0cb5169 100644 --- a/framework/parent/pom.xml +++ b/framework/parent/pom.xml @@ -12,7 +12,7 @@ org.daisy.pipeline framework-parent - 1.14.21-SNAPSHOT + 1.14.22-SNAPSHOT pom DAISY Pipeline 2 :: Framework Parent POM @@ -110,7 +110,7 @@ org.daisy.pipeline framework-bom - 1.14.21-SNAPSHOT + 1.14.22-SNAPSHOT pom import @@ -158,7 +158,7 @@ org.daisy.pipeline ds-to-spi-runtime - 1.2.1-SNAPSHOT + 1.2.1 org.daisy.pipeline.build @@ -246,7 +246,7 @@ org.daisy.pipeline.build ds-to-spi-maven-plugin - 1.1.2 + 1.1.5 diff --git a/framework/persistence-derby/pom.xml b/framework/persistence-derby/pom.xml index ab12fcffd2..7b9d7296ca 100644 --- a/framework/persistence-derby/pom.xml +++ b/framework/persistence-derby/pom.xml @@ -6,7 +6,7 @@ org.daisy.pipeline framework-parent - 1.14.21-SNAPSHOT + 1.14.22-SNAPSHOT ../parent @@ -209,6 +209,7 @@ + MockXProcEngine, org.daisy.pipeline.script.impl.XProcScript_my_script diff --git a/framework/persistence-derby/src/test/java/MockXProcEngine.java b/framework/persistence-derby/src/test/java/MockXProcEngine.java new file mode 100644 index 0000000000..b8c28ab898 --- /dev/null +++ b/framework/persistence-derby/src/test/java/MockXProcEngine.java @@ -0,0 +1,22 @@ +import java.net.URI; + +import org.daisy.common.xproc.XProcEngine; +import org.daisy.common.xproc.XProcErrorException; +import org.daisy.common.xproc.XProcInput; +import org.daisy.common.xproc.XProcPipeline; +import org.daisy.common.xproc.XProcPipelineInfo; +import org.daisy.common.xproc.XProcResult; + +import org.osgi.service.component.annotations.Component; + +@Component( + name = "mock-xproc-engine", + service = { XProcEngine.class } +) +public class MockXProcEngine implements XProcEngine { + + public XProcPipeline load(URI uri) { throw new UnsupportedOperationException(); } + public XProcPipelineInfo getInfo(URI uri) { throw new UnsupportedOperationException(); } + public XProcResult run(URI uri, XProcInput data) throws XProcErrorException { throw new UnsupportedOperationException(); } + +} diff --git a/framework/persistence-derby/src/test/java/TestBase.java b/framework/persistence-derby/src/test/java/TestBase.java index 36bf0b50f9..e319d09c45 100644 --- a/framework/persistence-derby/src/test/java/TestBase.java +++ b/framework/persistence-derby/src/test/java/TestBase.java @@ -69,7 +69,8 @@ protected Properties systemProperties() { @ProbeBuilder public TestProbeBuilder probeConfiguration(TestProbeBuilder probe) { // FIXME: can not delete this yet because it can not be generated with maven-bundle-plugin - probe.setHeader("Service-Component", "OSGI-INF/script.xml"); + probe.setHeader("Service-Component", "OSGI-INF/script.xml," + + "OSGI-INF/xproc-engine.xml"); return probe; } } diff --git a/framework/persistence-derby/src/test/resources/OSGI-INF/xproc-engine.xml b/framework/persistence-derby/src/test/resources/OSGI-INF/xproc-engine.xml new file mode 100644 index 0000000000..a882776994 --- /dev/null +++ b/framework/persistence-derby/src/test/resources/OSGI-INF/xproc-engine.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/framework/persistence-mysql/pom.xml b/framework/persistence-mysql/pom.xml index 181c16c2de..3ac836a993 100644 --- a/framework/persistence-mysql/pom.xml +++ b/framework/persistence-mysql/pom.xml @@ -4,7 +4,7 @@ org.daisy.pipeline framework-parent - 1.14.21-SNAPSHOT + 1.14.22-SNAPSHOT ../parent org.daisy.pipeline diff --git a/framework/pom.xml b/framework/pom.xml index 761498c791..29d1894c4a 100644 --- a/framework/pom.xml +++ b/framework/pom.xml @@ -12,7 +12,7 @@ org.daisy.pipeline framework-aggregator - 1.14.21-SNAPSHOT + 1.14.22-SNAPSHOT pom DAISY Pipeline 2 :: Framework Aggregator diff --git a/framework/saxon-adapter/pom.xml b/framework/saxon-adapter/pom.xml index ee1f51f72d..9b6980c57e 100644 --- a/framework/saxon-adapter/pom.xml +++ b/framework/saxon-adapter/pom.xml @@ -6,12 +6,12 @@ org.daisy.pipeline framework-parent - 1.14.21-SNAPSHOT + 1.14.22-SNAPSHOT ../parent saxon-adapter - 5.5.0-SNAPSHOT + 5.5.1-SNAPSHOT bundle DAISY Pipeline 2 :: Adapter for Saxon diff --git a/framework/utils/clientlib-java-jaxb/pom.xml b/framework/utils/clientlib-java-jaxb/pom.xml index 11a13e9b3e..6474fc7a97 100644 --- a/framework/utils/clientlib-java-jaxb/pom.xml +++ b/framework/utils/clientlib-java-jaxb/pom.xml @@ -4,7 +4,7 @@ org.daisy.pipeline framework-parent - 1.14.21-SNAPSHOT + 1.14.22-SNAPSHOT ../../parent org.daisy.pipeline diff --git a/framework/utils/xproc-engine-daisy-pipeline-logging/pom.xml b/framework/utils/xproc-engine-daisy-pipeline-logging/pom.xml index 88514c9f60..36c3bd58ac 100644 --- a/framework/utils/xproc-engine-daisy-pipeline-logging/pom.xml +++ b/framework/utils/xproc-engine-daisy-pipeline-logging/pom.xml @@ -6,7 +6,7 @@ org.daisy.pipeline framework-parent - 1.14.21-SNAPSHOT + 1.14.22-SNAPSHOT ../../parent diff --git a/framework/utils/xproc-engine-daisy-pipeline/pom.xml b/framework/utils/xproc-engine-daisy-pipeline/pom.xml index bd30e65545..8e41a82915 100644 --- a/framework/utils/xproc-engine-daisy-pipeline/pom.xml +++ b/framework/utils/xproc-engine-daisy-pipeline/pom.xml @@ -6,7 +6,7 @@ org.daisy.pipeline framework-parent - 1.14.21-SNAPSHOT + 1.14.22-SNAPSHOT ../../parent diff --git a/framework/webservice/pom.xml b/framework/webservice/pom.xml index ec2ea52f04..0c3c837537 100644 --- a/framework/webservice/pom.xml +++ b/framework/webservice/pom.xml @@ -6,12 +6,12 @@ org.daisy.pipeline framework-parent - 1.14.21-SNAPSHOT + 1.14.22-SNAPSHOT ../parent webservice - 3.5.1-SNAPSHOT + 3.5.2-SNAPSHOT bundle DAISY Pipeline 2 :: Web Service diff --git a/framework/webservice/src/main/java/org/daisy/pipeline/webservice/restlet/impl/PipelineWebService.java b/framework/webservice/src/main/java/org/daisy/pipeline/webservice/restlet/impl/PipelineWebService.java index 1dee1c6025..52a95bd6a8 100644 --- a/framework/webservice/src/main/java/org/daisy/pipeline/webservice/restlet/impl/PipelineWebService.java +++ b/framework/webservice/src/main/java/org/daisy/pipeline/webservice/restlet/impl/PipelineWebService.java @@ -61,7 +61,9 @@ */ @org.osgi.service.component.annotations.Component( name = "org.daisy.pipeline.webservice", - immediate = true + immediate = true, + service = { PipelineWebService.class } // this is to ensure object created by SPIHelper.createWebService() + // is an instance of PipelineWebService ) public class PipelineWebService extends Application { @@ -146,7 +148,7 @@ public void init() { webserviceStorage = new VolatileWebserviceStorage(); if (!checkAuthenticationSanity()){ try { - this.halt(); + close(); } catch (Exception e) { logger.error("Error shutting down:"+e.getMessage()); } @@ -177,7 +179,7 @@ public void init() { } catch (Exception e) { logger.error("Shutting down the framework because of:"+e.getMessage()); try{ - this.halt(); + close(); }catch (Exception innerException) { logger.error("Error shutting down:"+e.getMessage()); } @@ -272,10 +274,11 @@ private void halt() { @Deactivate public void close() { try { - pushNotifier.close(); - if (this.component!=null) - this.component.stop(); - this.stop(); + if (pushNotifier != null) + pushNotifier.close(); + if (component != null) + component.stop(); + stop(); logger.info("Webservice stopped."); } catch (Exception e) { logger.error("Error stopping the web service:" + e.getMessage()); diff --git a/framework/woodstox-osgi-adapter/pom.xml b/framework/woodstox-osgi-adapter/pom.xml index bbb98f25a0..1f77ad346c 100644 --- a/framework/woodstox-osgi-adapter/pom.xml +++ b/framework/woodstox-osgi-adapter/pom.xml @@ -6,7 +6,7 @@ org.daisy.pipeline framework-parent - 1.14.21-SNAPSHOT + 1.14.22-SNAPSHOT ../parent diff --git a/framework/xproc-api/pom.xml b/framework/xproc-api/pom.xml index fdf9684a99..4e973dbe40 100644 --- a/framework/xproc-api/pom.xml +++ b/framework/xproc-api/pom.xml @@ -6,7 +6,7 @@ org.daisy.pipeline framework-parent - 1.14.21-SNAPSHOT + 1.14.22-SNAPSHOT ../parent