[java] Download 3rd-party Java sources from Maven

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## master (unreleased)
 
+* [#310](https://github.com/clojure-emacs/orchard/pull/310): Java: add functions for downloading 3rd-party Java sources from Maven.
 * [#309](https://github.com/clojure-emacs/orchard/pull/309): **BREAKING:** Remove deprecated functions from orchard.java (`jdk-find`, `jdk-sources`, `jdk-tools`, `ensure-jdk-sources`).
 * [#309](https://github.com/clojure-emacs/orchard/pull/309): **BREAKING:** Remove `orchard.java/cache-initializer`.
 * [#311](https://github.com/clojure-emacs/orchard/pull/311): Trace: fix the printing inside the wrapped function to be truncated.

README.md

@@ -107,8 +107,23 @@ Currently, Orchard is able to find Java source files in the following places:
 - In the `src.zip` archive that comes together with most JDK distributions.
 - For clases that come from Maven-downloaded dependencies — in the special
   `-sources.jar` artifact that resides next to the main artifact in the `~/.m2`
-  directory. The sources artifact has to be downloaded ahead of time, for
-  example, by [enrich-classpath](https://github.com/clojure-emacs/enrich-classpath).
+  directory. The sources artifact has to be downloaded ahead of time.
+
+Orchard provides
+`orchard.java.source-files/download-sources-jar-for-coordinates` function to
+download the sources by invoking a subprocess with one of the supported build
+tools (Clojure CLI or Leiningen). You can call this function at any point of
+time on your own. Alternatively, you can bind the dynamic variable
+`orchard.java.source-files/*download-sources-jar-fn*` to a function which
+accepts a Class object, and Orchard will call this function automatically when
+it fails to locate a Java source file for the class. Usage example:
+
+```clj
+(binding [src-files/*download-sources-jar-fn*
+          #(src-files/download-sources-jar-for-coordinates
+            (src-files/infer-maven-coordinates-for-class %))]
+  (class->source-file-url <class-arg>))
+```
 
 If the source file can be located, this is usually enough for basic "jump to
 source" functionality. For a more precise "jump to definition" and for

src/orchard/java.clj

@@ -275,30 +275,31 @@
     (let [package (some-> c package symbol)
           relative-source-path (src-files/class->sourcefile-path c)
           source-file-url (src-files/class->source-file-url c)
-          {:keys [members] :as result} (misc/deep-merge (reflect-info (reflection-for c))
-                                                        (when source-file-url
-                                                          {:file relative-source-path
-                                                           :file-url source-file-url})
-                                                        (when (and *analyze-sources* source-file-url)
-                                                          (source-info c source-file-url))
-                                                        {:name       (-> c .getSimpleName symbol)
-                                                         :class      (-> c .getName symbol)
-                                                         :package    package
-                                                         :super      (-> c .getSuperclass typesym)
-                                                         :interfaces (map typesym (.getInterfaces c))
-                                                         :javadoc    (javadoc-url class)})]
-      (assoc result
-             :members (into {}
-                            (map (fn [[method arities]]
-                                   [method (into {}
-                                                 (map (fn [[k arity]]
-                                                        [k (let [static? (:static (:modifiers arity))]
-                                                             (-> arity
-                                                                 (assoc :annotated-arglists
-                                                                        (extract-annotated-arglists static? package arity))
-                                                                 (dissoc :non-generic-argtypes)))]))
-                                                 arities)]))
-                            members)))))
+          result (misc/deep-merge (reflect-info (reflection-for c))
+                                  (when source-file-url
+                                    {:file relative-source-path
+                                     :file-url source-file-url})
+                                  (when (and *analyze-sources* source-file-url)
+                                    (source-info c source-file-url))
+                                  {:name       (-> c .getSimpleName symbol)
+                                   :class      (-> c .getName symbol)
+                                   :package    package
+                                   :super      (-> c .getSuperclass typesym)
+                                   :interfaces (map typesym (.getInterfaces c))
+                                   :javadoc    (javadoc-url class)})]
+      (update result :members
+              (fn [members]
+                (into {}
+                      (map (fn [[method arities]]
+                             [method (into {}
+                                           (map (fn [[k arity]]
+                                                  [k (let [static? (:static (:modifiers arity))]
+                                                       (-> arity
+                                                           (assoc :annotated-arglists
+                                                                  (extract-annotated-arglists static? package arity))
+                                                           (dissoc :non-generic-argtypes)))]))
+                                           arities)]))
+                      members))))))
 
 #_(class-info* `Thread)
 #_(class-info* 'clojure.lang.PersistentList)
@@ -327,27 +328,21 @@
   overloads."
   [class]
   (let [cached (.get cache class)
-        info (if cached
-               (:info cached)
-               (class-info* class))
-        resource (:resource-url info)
-        last-modified (if (or (nil? resource)
-                              (util.io/url-to-file-within-archive? resource))
-                        0
-                        (util.io/last-modified-time resource))
-        stale (not= last-modified (:last-modified cached))
-        ;; If last-modified in cache mismatches last-modified of the file,
-        ;; regenerate class-info.
-        info (if (and cached stale)
-               (class-info* class)
-               info)]
-    (when (and
-           ;; Only cache full values that possibly were slowly computed.
-           ;; It would be a mistake to cache the fast values, letting them shadow a full computation:
-           *analyze-sources*
-           (or (not cached) stale))
-      (.put cache class {:info info, :last-modified last-modified}))
-    info))
+        file-url (-> cached :info :file-url)
+        last-modified (some-> file-url util.io/last-modified-time)
+        ;; Cache is valid only if the cached info discovered a valid file-url,
+        ;; and the modified date of that file matches the remembered date.
+        cache-valid? (and file-url (= last-modified (:last-modified cached)))]
+    (if cache-valid?
+      (:info cached)
+
+      (let [{:keys [file-url] :as info} (class-info* class)]
+        ;; Only cache value if we discovered the source file and analyzed it.
+        (when (and *analyze-sources* file-url)
+          (.put cache class
+                {:info info
+                 :last-modified (util.io/last-modified-time file-url)}))
+        info))))
 
 ;;; ## Class/Member Info
 ;;

src/orchard/java/source_files.clj

@@ -9,6 +9,12 @@
   (:import (java.io File IOException)
            (java.net URL)))
 
+(def ^:dynamic *download-sources-jar-fn*
+  "When not nil, this function will be called on the Class object if none of the
+  resolving methods could discover the source file for that class. The bound
+  function should try to download the JAR and return true if it did."
+  nil)
+
 (defn- readable-file [^File f]
   (when (.canRead f) f))
 
@@ -102,31 +108,142 @@
         sources-jar (io/file parent (str fname "-sources.jar"))]
     (readable-file sources-jar)))
 
+(defn- find-jar-file-for-class [^Class klass]
+  ;; Get the classloader that loaded `klass` and locate the resource that the
+  ;; class was loaded from. If that resource is a JAR file, return it.
+  (when-let [cl (.getClassLoader klass)]
+    (let [class-file (class->classfile-path klass)
+          res (.getResource cl class-file)]
+      (some-> res parse-jar-path-from-url))))
+
 (defn- locate-source-url-near-class-jar
   "If `klass` comes from a third-party JAR that presumedly resides in `.m2`
   directory, try to look for a sources JAR near the class JAR and return it
   together with the implied source filename."
   [^Class klass]
-  (when-let [cl (some-> klass .getClassLoader)]
-    ;; Get the classloader that loaded the `klass` and locate the resource that
-    ;; the the class was loaded from. If that resource is a JAR file, search for
-    ;; a sources JAR near it.
-    (let [class-file (class->classfile-path klass)
-          res (.getResource cl class-file)
-          sources-jar-file (some-> res parse-jar-path-from-url infer-sources-jar-file)
-          source-filename (classfile-path->sourcefile-path class-file)]
-      (when sources-jar-file
-        (-> (combine-archive-url sources-jar-file source-filename)
-            verify-url-readable)))))
+  (let [class-jar-file (find-jar-file-for-class klass)
+        sources-jar-file (some-> class-jar-file infer-sources-jar-file)
+        class-file (class->classfile-path klass)
+        source-filename (classfile-path->sourcefile-path class-file)]
+    (when sources-jar-file
+      (-> (combine-archive-url sources-jar-file source-filename)
+          verify-url-readable))))
 
 #_(locate-source-url-near-class-jar clojure.lang.PersistentVector)
 
-(defn class->source-file-url ^URL [klass]
+(defn class->source-file-url
+  "Resolve the URL path to the sources given `klass` using any of the supported
+  methods. If no method worked and `*download-sources-jar-fn*` is bound, invoke
+  it and try to resolve again."
+  ^URL [klass]
   (or (locate-source-url-on-classpath klass)
       (locate-source-url-in-jdk-sources klass)
-      (locate-source-url-near-class-jar klass)))
+      (locate-source-url-near-class-jar klass)
+      (and *download-sources-jar-fn*
+           (*download-sources-jar-fn* klass)
+           (locate-source-url-near-class-jar klass))))
 
 #_(class->source-file-url mx.cider.orchard.LruMap)
 #_(class->source-file-url java.lang.Thread)
 #_(class->source-file-url clojure.lang.PersistentVector)
 #_(class->source-file-url clojure.core.Eduction)
+
+;;; ## Downloading Java sources from Maven
+
+(defn infer-maven-coordinates-for-class
+  "Given a class, attempt to parse its Maven coordinates (group id, artifact id,
+  version) from the JAR path it resides in."
+  [^Class klass]
+  (when-let [^File jar-file (find-jar-file-for-class klass)]
+    ;; Leap of faith here. We really infer this data from the filesystem path.
+    ;; The correct way would be to parse pom.xml inside the jar and get the
+    ;; coordinates from there, but that is more work.
+    (let [jar-file'   (.getParentFile jar-file)
+          version     (.getName jar-file')
+          jar-file''  (.getParentFile jar-file')
+          artifact    (.getName jar-file'')
+          group-parts (loop [acc (), f jar-file'']
+                        (let [parent (.getParentFile f)
+                              name   (some-> parent .getName)]
+                          (if (and name (not= name "repository"))
+                            (recur (cons name acc) parent)
+                            acc)))]
+      {:artifact artifact
+       :group    (string/join "." group-parts)
+       :version  version})))
+
+#_(infer-maven-coordinates-for-class clojure.lang.PersistentVector)
+
+(defn- download-sources-using-invoke-tool
+  "Download source JAR for given library coordinates using Clojure 1.12
+  `invoke-tool` wrapper."
+  [{:keys [group artifact version]}]
+  (let [procurer (or ((requiring-resolve 'clojure.java.basis/current-basis))
+                     {:mvn/repos {"central" {:url "https://repo1.maven.org/maven2/"}
+                                  "clojars" {:url "https://repo.clojars.org/"}}})
+        sources-artifact (symbol (format "%s/%s$sources" group artifact))
+        lib-coords {sources-artifact {:mvn/version version}}
+        res ((requiring-resolve 'clojure.tools.deps.interop/invoke-tool)
+             {:tool-alias :deps
+              :fn 'clojure.tools.deps/resolve-added-libs
+              :args {:add lib-coords, :procurer procurer}})]
+    (vec (mapcat :paths (vals (:added res))))))
+
+#_(download-sources-using-invoke-tool (infer-maven-coordinates-for-class clojure.lang.PersistentVector))
+
+(defn- run-subprocess [args]
+  (println "[Orchard] Invoking subprocess:" (string/join " " args))
+  (let [process (.start (ProcessBuilder. ^java.util.List args))]
+    (.waitFor process)
+    (let [out (slurp (.getInputStream process))
+          err (slurp (.getErrorStream process))]
+      (when-not (string/blank? out)
+        (println out))
+      (when-not (string/blank? err)
+        (binding [*out* *err*] (println err))))
+    (.exitValue process)))
+
+(defn- download-sources-using-clojure-cli
+  "Download source JAR for given library coordinates by invoking `clojure` CLI
+  subprocess."
+  [{:keys [group artifact version]}]
+  (when (System/getProperty "clojure.basis")
+    (run-subprocess ["clojure" "-P" "-Sdeps"
+                     (format "{:deps {%s/%s$sources {:mvn/version \"%s\"}}}"
+                             group artifact version)])))
+
+#_(download-sources-using-clojure-cli (infer-maven-coordinates-for-class clojure.lang.PersistentVector))
+
+(defn- download-sources-using-lein
+  "Download source JAR for given library coordinates by invoking `lein`
+  subprocess."
+  [{:keys [group artifact version]}]
+  (run-subprocess
+   ;; "update-in :dependencies empty" is needed to drop the existing deps so
+   ;; that they don't mess with the sources JAR version we need.
+   ["lein" "update-in" ":dependencies" "empty" "--"
+    "update-in" ":dependencies" "conj"
+    (format "[%s/%s \"%s\" :classifier \"sources\"]" group artifact version)
+    "--" "deps"]))
+
+#_(download-sources-using-lein (infer-maven-coordinates-for-class clojure.lang.PersistentVector))
+
+(defn download-sources-jar-for-coordinates
+  "Download source JAR for given library coordinates using either tools.deps or
+  Leiningen, depending how the Clojure process was started. Returns non-nil if
+  any of the methods at least attempted to download the artifact, or nil if
+  downloading didn't happen."
+  [maven-coordinates]
+  (let [clj-1-12+? (try (require 'clojure.tools.deps.interop)
+                        (catch Exception _))
+        tools-deps? (System/getProperty "clojure.basis")]
+    (cond (and tools-deps? clj-1-12+?)
+          (download-sources-using-invoke-tool maven-coordinates)
+
+          tools-deps?
+          (download-sources-using-clojure-cli maven-coordinates)
+
+          (System/getenv "LEIN_HOME") ;; Indicator of Leiningen-started process.
+          (download-sources-using-lein maven-coordinates))))
+
+#_(download-sources-jar-for-coordinates (infer-maven-coordinates-for-class clojure.lang.PersistentVector))

test/orchard/java/source_files_test.clj

@@ -1,9 +1,52 @@
 (ns orchard.java.source-files-test
-  (:require [clojure.test :refer [deftest is]]
+  (:require [clojure.test :refer [deftest is testing]]
             [orchard.java.source-files :as src-files]))
 
 (deftest class->source-file-url-test
   (is (src-files/class->source-file-url mx.cider.orchard.LruMap)) ;; classpath
   (is (src-files/class->source-file-url Thread)) ;; JDK
   (is (src-files/class->source-file-url clojure.lang.PersistentVector)) ;; Clojure
   (is (nil? (src-files/class->source-file-url clojure.core.Eduction)))) ;; record
+
+;; Download sources testing
+
+(deftest test-infer-maven-coordinates
+  (is (= (src-files/infer-maven-coordinates-for-class clojure.lang.Ref)
+         {:artifact "clojure", :group "org.clojure", :version (clojure-version)}))
+
+  (is (nil? (src-files/infer-maven-coordinates-for-class String))
+      "JDK classes don't resolve to a Maven coordinate."))
+
+(defn- sources-jar-file ^java.io.File [klass]
+  (with-redefs [src-files/readable-file identity]
+    (#'src-files/infer-sources-jar-file
+     (#'src-files/find-jar-file-for-class klass))))
+
+;; TODO: test for Clojure CLI and invoke tool at some point.
+(deftest test-download-sources-jar-using-lein
+  (let [f (sources-jar-file clojure.lang.Ref)]
+    (.delete f)
+    (is (not (.exists f))))
+
+  (is (#'src-files/download-sources-using-lein
+       (src-files/infer-maven-coordinates-for-class clojure.lang.Ref)))
+
+  (is (.exists (sources-jar-file clojure.lang.Ref)))
+
+  (testing "downloaded source jars contain actual source files"
+    (is (> (count (slurp (src-files/class->source-file-url clojure.lang.Ref)))
+           200))))
+
+(deftest test-download-sources-jar
+  (let [f (sources-jar-file clojure.lang.Ref)]
+    (.delete f)
+    (is (not (.exists f))))
+
+  (is (src-files/download-sources-jar-for-coordinates
+       (src-files/infer-maven-coordinates-for-class clojure.lang.Ref)))
+
+  (is (.exists (sources-jar-file clojure.lang.Ref)))
+
+  (testing "downloaded source jars contain actual source files"
+    (is (> (count (slurp (src-files/class->source-file-url clojure.lang.Ref)))
+           200))))