Changes for Java 21 - a LTS Java release

Required changes due to deprecated objects and methods.
java.net.URL all constructors were deprecated.

This was harder than one would expect. There is no direct replacement
in the java.net library for the URL constructors. Nothing tolerates
jar-file URIs as of Java 20+.

Add xsi:type sensitive comparison for float and double.

Tolerate regex match behavior change: seems lookahead is no longer
captured into individual groups.

Now uses Java 11 release for Java code compilation with Java 21.
This eliminates warnings about Java 8 being deprecated.

Removed redundancy of XMLUtils and NodeInfo for converting strings to float/double.

DAFFODIL-2757, DAFFODIL-2402

build.sbt

@@ -211,6 +211,12 @@ lazy val testStdLayout = Project("daffodil-test-stdLayout", file("test-stdLayout
   .dependsOn(tdmlProc % "test")
   .settings(commonSettings, nopublish)
 
+// Choices here are Java LTS versions, 8, 11, 17, 21,...
+// However 8 is deprecated as of Java 21, so will be phased out.
+val minSupportedJavaVersion: String =
+  if (scala.util.Properties.isJavaAtLeast("21")) "11"
+  else "8"
+
 lazy val commonSettings = Seq(
   organization := "org.apache.daffodil",
   version := "3.6.0-SNAPSHOT",
@@ -245,7 +251,7 @@ lazy val commonSettings = Seq(
 
 def buildScalacOptions(scalaVersion: String) = {
   val commonOptions = Seq(
-    "-target:jvm-1.8",
+    s"-release:$minSupportedJavaVersion", // scala 2.12 can only do Java 8, regardless of this setting.
     "-feature",
     "-deprecation",
     "-language:experimental.macros",
@@ -268,13 +274,19 @@ def buildScalacOptions(scalaVersion: String) = {
     case _ => Seq.empty
   }
 
-  val javaVersionSpecificOptions =
-    if (scala.util.Properties.isJavaAtLeast("9"))
-      Seq("-release", "8") // ensure Java backwards compatibility (DAFFODIL-2579)
-    else
-      Seq.empty
+  commonOptions ++ scalaVersionSpecificOptions
+}
+
+val javaVersionSpecificOptions = {
+  val releaseOption = // as of Java 11, they no longer accept "-release". Must use "--release".
+    if (scala.util.Properties.isJavaAtLeast("11")) "--release" else "-release"
 
-  commonOptions ++ scalaVersionSpecificOptions ++ javaVersionSpecificOptions
+  // Java 21 deprecates Java 8 and warns about it.
+  // So if you are using Java 21, Java code compilation will specify a newer Java version
+  // to avoid warnings.
+  if (scala.util.Properties.isJavaAtLeast("11")) Seq(releaseOption, minSupportedJavaVersion)
+  else if (scala.util.Properties.isJavaAtLeast("9")) Seq(releaseOption, "8")
+  else Nil // for Java 8 compilation
 }
 
 // Workaround issue that some options are valid for javac, not javadoc.
@@ -285,12 +297,6 @@ def buildJavacOptions() = {
     "-Xlint:deprecation",
   )
 
-  val javaVersionSpecificOptions =
-    if (scala.util.Properties.isJavaAtLeast("9"))
-      Seq("--release", "8") // ensure Java backwards compatibility (DAFFODIL-2579)
-    else
-      Seq.empty
-
   commonOptions ++ javaVersionSpecificOptions
 }
 

daffodil-cli/src/main/scala/org/apache/daffodil/cli/Main.scala

@@ -28,6 +28,7 @@ import java.nio.channels.Channels
 import java.nio.file.Paths
 import java.util.Scanner
 import java.util.concurrent.Executors
+import javax.xml.parsers.SAXParserFactory
 import javax.xml.transform.TransformerException
 import javax.xml.transform.TransformerFactory
 import javax.xml.transform.stream.StreamResult
@@ -92,7 +93,6 @@ import org.rogach.scallop.exceptions.GenericScallopException
 import org.slf4j.event.Level
 import org.xml.sax.InputSource
 import org.xml.sax.SAXParseException
-import org.xml.sax.helpers.XMLReaderFactory
 
 class ScallopExitException(val exitCode: Int) extends Exception
 
@@ -1813,8 +1813,10 @@ class Main(
           case (false, true) => { // Encoding
             val exiResult = new EXIResult(exiFactory.get)
             exiResult.setOutputStream(output)
-
-            val reader = XMLReaderFactory.createXMLReader()
+            val factory = SAXParserFactory.newInstance()
+            factory.setNamespaceAware(true)
+            val saxParser = factory.newSAXParser()
+            val reader = saxParser.getXMLReader
             reader.setContentHandler(exiResult.getHandler)
             reader.setErrorHandler(new EXIErrorHandler)
             try {

daffodil-core/src/test/scala/org/apache/daffodil/core/api/TestForHeapDump.scala

@@ -150,8 +150,6 @@ class TestForHeapDump {
   }
 
   def gcAndAllowHeapDump(): Unit = {
-    System.gc()
-    System.runFinalization()
     System.gc()
     System.out.println("Take a Heap Dump Now! (You have 10 seconds)")
     Thread.sleep(10000)

daffodil-io/src/test/scala/org/apache/daffodil/layers/TestJavaIOStreams.scala

@@ -129,10 +129,16 @@ ZyBzb2x1dGlvbnMuCg=="""
     val scanner = new Scanner(is, StandardCharsets.ISO_8859_1.name())
     is.skip(3)
     is.mark(2)
-    val matchString = scanner.findWithinHorizon("(.*?)(?=(\\Q;\\E))", 2)
+    // Prior to changes for Java21, this test used non-capturing lookahead for the ";"
+    // and group(2) was containing the match of the lookahead.
+    // as of Java21 testing, the lookahead match is no longer made into a group it seems.
+    // The lookahead is included in the "whole match" aka group(0)
+    val matchString = scanner.findWithinHorizon("(.*?)(\\Q;\\E)", 2)
     is.reset()
-    assertEquals("l", matchString)
-    assertEquals(";", scanner.`match`().group(2))
+    val m = scanner.`match`()
+    assertEquals("l;", m.group(0))
+    assertEquals("l", m.group(1))
+    assertEquals(";", m.group(2))
   }
 
   /**

daffodil-lib/src/main/scala/org/apache/daffodil/lib/util/Misc.scala

@@ -20,9 +20,8 @@ package org.apache.daffodil.lib.util
 import java.io.ByteArrayInputStream
 import java.io.ByteArrayOutputStream
 import java.io.File
-import java.io.InputStream
+import java.io.IOException
 import java.net.URI
-import java.net.URL
 import java.net.URLClassLoader
 import java.nio.ByteBuffer
 import java.nio.CharBuffer
@@ -103,38 +102,102 @@ object Misc {
     // more time is wasted by people forgetting that the initial "/" is needed
     // to get classpath relative behavior... Let's make sure there is a leading "/"
     val resPath = if (resourcePath.startsWith("/")) resourcePath else "/" + resourcePath
-    val res = this.getClass().getResource(resPath)
-    if (res == null) {
-      (None, resPath)
-    } else (Some(res.toURI), resPath)
+    val res = Option(this.getClass().getResource(resPath))
+    (res.map(_.toURI), resPath)
   }
 
   /**
    * Gets a resource on the classpath, or relative to another URI
    */
-  def getResourceRelativeOption(rawResName: String, optContextURI: Option[URI]): Option[URI] = {
+  private def getResourceAbsoluteOrRelativeOption(
+    rawResName: String,
+    optContextURI: Option[URI],
+  ): Option[URI] = {
     val resName = rawResName.replaceAll("""\s""", "%20")
     val (maybeRes, _) = Misc.getResourceOption(resName)
     if (maybeRes.isDefined) {
       maybeRes // found directly on the classpath.
     } else {
       optContextURI.flatMap { contextURI =>
-        //
-        // try relative to enclosing context uri
-        //
-        // Done using URL constructor because the URI.resolve(uri) method
-        // doesn't work against so called opaque URIs, and jar URIs of the
-        // sort we get here if the resource is in a jar, are opaque.
-        // Some discussion of this issue is https://issues.apache.org/jira/browse/XMLSCHEMA-3
-        //
-        val contextURL = contextURI.toURL
-        val completeURL = new URL(contextURL, resName)
-        val res = tryURL(completeURL)
-        res
+        getResourceRelativeOnlyOption(resName, contextURI)
       }
     }
   }
 
+  /**
+   * Get resource relative to the context URI.
+   *
+   * Does NOT try the string as an absolute location first
+   * or anything like that.
+   *
+   * @param relPath
+   * @param contextURI
+   * @return Some uri if the relative resource exists.
+   */
+  def getResourceRelativeOnlyOption(relPath: String, contextURI: URI): Option[URI] = {
+    Assert.usage(relPath ne null)
+    Assert.usage(contextURI ne null)
+    if (contextURI.isOpaque) {
+      //
+      // We used to call new URL(jarURI, relativePathString)
+      // but that is deprecated now (as of Java 20)
+      //
+      optRelativeJarFileURI(contextURI, relPath)
+    } else {
+      // context URI is not opaque. It's probably a file URI
+      if (contextURI.getScheme == "file") {
+        val relURI = contextURI.resolve(relPath)
+        if (Paths.get(relURI).toFile.exists())
+          Some(relURI)
+        else None
+      } else {
+        // not a file nor an opaque resource URI. What is it?
+        throw new IllegalArgumentException(s"Unrecognized URI type: $contextURI")
+      }
+    }
+  }
+
+  /**
+   * Java 20 deprecated the 2-arg URL constructor which worked to create relative URIs
+   * within the same Jar file.
+   *
+   * This is a bit harder to achieve now. You are not allowed to resolve relative to a jar file URI.
+   * That is URI.resolve(relPath) doesn't work if the URI is a jar file URI.
+   *
+   * Now we have to hack the jar:file: URI as a string because URI.resolve won't work
+   *
+   * jar file URIs look like this:
+   *
+   *    `jar:file:/..absolute path to jar file.jar!/absolute path from root inside jar to file``
+   *
+   * We split at the !/, make a relative path on just the inside-jar-file part, then glue
+   * back together.
+   *
+   *
+   * @param contextURI
+   * @param relPath
+   * @return Some(uri) for an existing relative path within the same jar file, or None if it does not exist.
+   */
+  def optRelativeJarFileURI(contextURI: URI, relPath: String): Option[URI] = {
+    val parts = contextURI.toString.split("\\!\\/")
+    Assert.invariant(parts.length == 2)
+    val jarPart = parts(0)
+    val pathPart = parts(1)
+    Assert.invariant(pathPart ne null)
+    val contextURIPathOnly = URI.create(pathPart)
+    val resolvedURIPathOnly = contextURIPathOnly.resolve(relPath)
+    val newJarPathURI = URI.create(jarPart + "!/" + resolvedURIPathOnly.toString)
+    try {
+      newJarPathURI.toURL.openStream().close()
+      // that worked, so we can open it so it exists.
+      Some(newJarPathURI)
+    } catch {
+      case io: IOException =>
+        // failed. So that jar file doesn't exist
+        None
+    }
+  }
+
   /**
    * Search for a resource name, trying a handful of heuristics.
    *
@@ -155,7 +218,7 @@ object Misc {
       if (resAsURI.getScheme != null) Paths.get(resAsURI) else Paths.get(resName)
     val resolvedURI =
       if (Files.exists(resPath)) Some(resPath.toFile().toURI())
-      else Misc.getResourceRelativeOption(resName, relativeTo)
+      else Misc.getResourceAbsoluteOrRelativeOption(resName, relativeTo)
     val res = resolvedURI.orElse {
       // try ignoring the directory part
       val parts = resName.split("/")
@@ -170,21 +233,6 @@ object Misc {
     res
   }
 
-  private def tryURL(url: URL): Option[URI] = {
-    var is: InputStream = null
-    val res =
-      try {
-        is = url.openStream()
-        // worked! We found it.
-        Some(url.toURI)
-      } catch {
-        case e: java.io.IOException => None
-      } finally {
-        if (is != null) is.close()
-      }
-    res
-  }
-
   lazy val classPath = {
     val cl = this.getClass().getClassLoader()
     val urls = cl match {