check tmp dir for noexec flag and warn user (#275)

* check tmp dir for noexec flag and warn user

zstd requires a tmp dir without the `noexec` flag - probe for that and
inform the user about their options

* fmt

* readme

README.md

@@ -490,4 +490,14 @@ linked above). They exist only at compile time. Hence, it's safe to cast a given
 
 Yes, properties are no longer fields of stored nodes. Hence the debugger cannot find them.
 
-But despair not! We have attached the `_debugChildren()` method to the GNode class. In order to see anything useful, you need to tell your debugger to use that in its object inspector. So in intellij, you need to add a custom java type renderer, make it apply to all `flatgraph.GNode` instances, and then tell it to use the expression `_debugChildren()` when expanding a node. See e.g. https://www.jetbrains.com/help/idea/customizing-views.html#renderers .
+But despair not! We have attached the `_debugChildren()` method to the GNode class. In order to see anything useful, you need to tell your debugger to use that in its object inspector. So in intellij, you need to add a custom java type renderer, make it apply to all `flatgraph.GNode` instances, and then tell it to use the expression `_debugChildren()` when expanding a node. See e.g. https://www.jetbrains.com/help/idea/customizing-views.html#renderers .
+
+## I got this strange `UnsatisfiedLinkError` with zstd, what's that all about?
+We use https://github.com/luben/zstd-jni for storage compression, and that requires that you have not mounted your `tmp` partition with the `noexec` flag. If you really need that flag on your system tmp partition, you can workaround this by defining the java system property `-Djava.io.tmpdir=/path/to/tmp`. Before invoking zstd, flatgraph performs some basic checks and tries to help the user, should this go wrong. Here's how it looks like if it does (listed here for search engine indices): 
+```
+flatgraph.storage.ZstdWrapper$JniInvocationException: Error while trying to invoke zstd, i.e. cannot compress or decompress, which is required for flatgraph's storage
+...
+Cause: java.lang.ExceptionInInitializerError: Exception java.lang.UnsatisfiedLinkError: /tmp/libzstd-jni-1.5.6-77867291841665399714.so: /tmp/libzstd-jni-1.5.6-77867291841665399714.so: failed to map segment from shared object
+[info] no zstd-jni-1.5.6-7 in java.library.path: /usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib
+[info] Unsupported OS/arch, cannot find /linux/amd64/libzstd-jni-1.5.6-7.so or load zstd-jni-1.5.6-7 from system libraries.
+```

core/src/main/scala/flatgraph/storage/Deserialization.scala

@@ -172,19 +172,22 @@ object Deserialization {
   }
 
   private def readPool(manifest: GraphItem, fileChannel: FileChannel): Array[String] = {
-    val stringPoolLength = Zstd
-      .decompress(
-        fileChannel.map(FileChannel.MapMode.READ_ONLY, manifest.stringPoolLength.startOffset, manifest.stringPoolLength.compressedLength),
-        manifest.stringPoolLength.decompressedLength
-      )
-      .order(ByteOrder.LITTLE_ENDIAN)
-    val stringPoolBytes = Zstd
-      .decompress(
-        fileChannel
-          .map(FileChannel.MapMode.READ_ONLY, manifest.stringPoolBytes.startOffset, manifest.stringPoolBytes.compressedLength),
-        manifest.stringPoolBytes.decompressedLength
-      )
-      .order(ByteOrder.LITTLE_ENDIAN)
+    val stringPoolLength = ZstdWrapper(
+      Zstd
+        .decompress(
+          fileChannel.map(FileChannel.MapMode.READ_ONLY, manifest.stringPoolLength.startOffset, manifest.stringPoolLength.compressedLength),
+          manifest.stringPoolLength.decompressedLength
+        )
+        .order(ByteOrder.LITTLE_ENDIAN)
+    )
+    val stringPoolBytes = ZstdWrapper(
+      Zstd
+        .decompress(
+          fileChannel.map(FileChannel.MapMode.READ_ONLY, manifest.stringPoolBytes.startOffset, manifest.stringPoolBytes.compressedLength),
+          manifest.stringPoolBytes.decompressedLength
+        )
+        .order(ByteOrder.LITTLE_ENDIAN)
+    )
     val poolBytes = new Array[Byte](manifest.stringPoolBytes.decompressedLength)
     stringPoolBytes.get(poolBytes)
     val pool    = new Array[String](manifest.stringPoolLength.decompressedLength >> 2)
@@ -214,9 +217,9 @@ object Deserialization {
 
   private def readArray(channel: FileChannel, ptr: OutlineStorage, nodes: Array[Array[GNode]], stringPool: Array[String]): Array[?] = {
     if (ptr == null) return null
-    val dec = Zstd
-      .decompress(channel.map(FileChannel.MapMode.READ_ONLY, ptr.startOffset, ptr.compressedLength), ptr.decompressedLength)
-      .order(ByteOrder.LITTLE_ENDIAN)
+    val dec = ZstdWrapper(
+      Zstd.decompress(channel.map(FileChannel.MapMode.READ_ONLY, ptr.startOffset, ptr.compressedLength), ptr.decompressedLength)
+    ).order(ByteOrder.LITTLE_ENDIAN)
     ptr.typ match {
       case StorageType.Bool =>
         val bytes = new Array[Byte](dec.limit())

core/src/main/scala/flatgraph/storage/Serialization.scala

@@ -180,7 +180,7 @@ object Serialization {
 
   private[flatgraph] def write(bytes: Array[Byte], res: OutlineStorage, filePtr: AtomicLong, fileChannel: FileChannel): OutlineStorage = {
     res.decompressedLength = bytes.length
-    val compressed = Zstd.compress(bytes)
+    val compressed = ZstdWrapper(Zstd.compress(bytes))
 
     var outPos = filePtr.getAndAdd(compressed.length)
     res.startOffset = outPos

core/src/main/scala/flatgraph/storage/ZstdWrapper.scala

@@ -0,0 +1,50 @@
+package flatgraph.storage
+
+import org.slf4j.LoggerFactory
+
+import java.nio.file.{Files, Paths}
+import scala.jdk.CollectionConverters.*
+import scala.util.{Properties, Try}
+
+object ZstdWrapper {
+  private val logger = LoggerFactory.getLogger(getClass)
+
+  /** zstd-jni ships system libraries that are being unpacked, loaded and executed from the system tmp directory. If that fails we get a
+    * rather obscure error message - this wrapper adds a check if the tmp dir is executable, and enhances the error message if the zstd
+    * invocation fails.
+    *
+    * This is where zstd-jni loads the system library:
+    * https://github.com/luben/zstd-jni/blob/9b08f1d0cdcf3b12b7a307cbba3d9f195149250b/src/main/java/com/github/luben/zstd/util/Native.java#L71
+    */
+  def apply[A](fun: => A): A = {
+    probeTmpMountOptions()
+
+    try {
+      fun
+    } catch {
+      case e =>
+        throw new JniInvocationException(
+          "Error while trying to invoke zstd, i.e. cannot compress or decompress, which is required for flatgraph's storage",
+          Option(e)
+        )
+    }
+  }
+
+  private def probeTmpMountOptions(): Unit = {
+    val tmpDirPath = System.getProperty("java.io.tmpdir")
+    lazy val warnMessage = s"the configured temp directory ($tmpDirPath) is mounted with `noexec` flag - " +
+      "this will likely lead to an error when trying to invoke zstd. Please either remount it without `noexec` or " +
+      "configure a different tmp directory, e.g. via java system property `-Djava.io.tmpdir=/path/to/tmp`"
+    Try {
+      if (Properties.isLinux || Properties.isMac) {
+        val mounts = Files.readAllLines(Paths.get("/proc/mounts"))
+        if (mounts.asScala.exists { mountInfoLine => mountInfoLine.contains(s" $tmpDirPath ") && mountInfoLine.contains("noexec") })
+          logger.warn(warnMessage)
+      }
+    }
+    // we're just probing here to warn the user and give some hints about fixing the situation
+    // it's fairly brittle as well, so if this fails we won't bother
+  }
+
+  class JniInvocationException(message: String, cause: Option[Throwable]) extends RuntimeException(message, cause.orNull)
+}