diff --git a/CHANGELOG.md b/CHANGELOG.md index ecc29f0f..6eb4ff6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## master (unreleased) +* [#310](https://github.com/clojure-emacs/orchard/pull/310): Java: add functions for downloading 3rd-party Java sources from Maven. * [#309](https://github.com/clojure-emacs/orchard/pull/309): **BREAKING:** Remove deprecated functions from orchard.java (`jdk-find`, `jdk-sources`, `jdk-tools`, `ensure-jdk-sources`). * [#309](https://github.com/clojure-emacs/orchard/pull/309): **BREAKING:** Remove `orchard.java/cache-initializer`. * [#311](https://github.com/clojure-emacs/orchard/pull/311): Trace: fix the printing inside the wrapped function to be truncated. diff --git a/README.md b/README.md index a9cb36f7..4e8b2234 100644 --- a/README.md +++ b/README.md @@ -107,8 +107,23 @@ Currently, Orchard is able to find Java source files in the following places: - In the `src.zip` archive that comes together with most JDK distributions. - For clases that come from Maven-downloaded dependencies — in the special `-sources.jar` artifact that resides next to the main artifact in the `~/.m2` - directory. The sources artifact has to be downloaded ahead of time, for - example, by [enrich-classpath](https://github.com/clojure-emacs/enrich-classpath). + directory. The sources artifact has to be downloaded ahead of time. + +Orchard provides +`orchard.java.source-files/download-sources-jar-for-coordinates` function to +download the sources by invoking a subprocess with one of the supported build +tools (Clojure CLI or Leiningen). You can call this function at any point of +time on your own. Alternatively, you can bind the dynamic variable +`orchard.java.source-files/*download-sources-jar-fn*` to a function which +accepts a Class object, and Orchard will call this function automatically when +it fails to locate a Java source file for the class. Usage example: + +```clj +(binding [src-files/*download-sources-jar-fn* + #(src-files/download-sources-jar-for-coordinates + (src-files/infer-maven-coordinates-for-class %))] + (class->source-file-url )) +``` If the source file can be located, this is usually enough for basic "jump to source" functionality. For a more precise "jump to definition" and for diff --git a/src/orchard/java.clj b/src/orchard/java.clj index 37070713..cfb9e351 100644 --- a/src/orchard/java.clj +++ b/src/orchard/java.clj @@ -275,30 +275,31 @@ (let [package (some-> c package symbol) relative-source-path (src-files/class->sourcefile-path c) source-file-url (src-files/class->source-file-url c) - {:keys [members] :as result} (misc/deep-merge (reflect-info (reflection-for c)) - (when source-file-url - {:file relative-source-path - :file-url source-file-url}) - (when (and *analyze-sources* source-file-url) - (source-info c source-file-url)) - {:name (-> c .getSimpleName symbol) - :class (-> c .getName symbol) - :package package - :super (-> c .getSuperclass typesym) - :interfaces (map typesym (.getInterfaces c)) - :javadoc (javadoc-url class)})] - (assoc result - :members (into {} - (map (fn [[method arities]] - [method (into {} - (map (fn [[k arity]] - [k (let [static? (:static (:modifiers arity))] - (-> arity - (assoc :annotated-arglists - (extract-annotated-arglists static? package arity)) - (dissoc :non-generic-argtypes)))])) - arities)])) - members))))) + result (misc/deep-merge (reflect-info (reflection-for c)) + (when source-file-url + {:file relative-source-path + :file-url source-file-url}) + (when (and *analyze-sources* source-file-url) + (source-info c source-file-url)) + {:name (-> c .getSimpleName symbol) + :class (-> c .getName symbol) + :package package + :super (-> c .getSuperclass typesym) + :interfaces (map typesym (.getInterfaces c)) + :javadoc (javadoc-url class)})] + (update result :members + (fn [members] + (into {} + (map (fn [[method arities]] + [method (into {} + (map (fn [[k arity]] + [k (let [static? (:static (:modifiers arity))] + (-> arity + (assoc :annotated-arglists + (extract-annotated-arglists static? package arity)) + (dissoc :non-generic-argtypes)))])) + arities)])) + members)))))) #_(class-info* `Thread) #_(class-info* 'clojure.lang.PersistentList) @@ -327,27 +328,21 @@ overloads." [class] (let [cached (.get cache class) - info (if cached - (:info cached) - (class-info* class)) - resource (:resource-url info) - last-modified (if (or (nil? resource) - (util.io/url-to-file-within-archive? resource)) - 0 - (util.io/last-modified-time resource)) - stale (not= last-modified (:last-modified cached)) - ;; If last-modified in cache mismatches last-modified of the file, - ;; regenerate class-info. - info (if (and cached stale) - (class-info* class) - info)] - (when (and - ;; Only cache full values that possibly were slowly computed. - ;; It would be a mistake to cache the fast values, letting them shadow a full computation: - *analyze-sources* - (or (not cached) stale)) - (.put cache class {:info info, :last-modified last-modified})) - info)) + file-url (-> cached :info :file-url) + last-modified (some-> file-url util.io/last-modified-time) + ;; Cache is valid only if the cached info discovered a valid file-url, + ;; and the modified date of that file matches the remembered date. + cache-valid? (and file-url (= last-modified (:last-modified cached)))] + (if cache-valid? + (:info cached) + + (let [{:keys [file-url] :as info} (class-info* class)] + ;; Only cache value if we discovered the source file and analyzed it. + (when (and *analyze-sources* file-url) + (.put cache class + {:info info + :last-modified (util.io/last-modified-time file-url)})) + info)))) ;;; ## Class/Member Info ;; diff --git a/src/orchard/java/source_files.clj b/src/orchard/java/source_files.clj index 4e4fd6f1..5244b75a 100644 --- a/src/orchard/java/source_files.clj +++ b/src/orchard/java/source_files.clj @@ -9,6 +9,12 @@ (:import (java.io File IOException) (java.net URL))) +(def ^:dynamic *download-sources-jar-fn* + "When not nil, this function will be called on the Class object if none of the + resolving methods could discover the source file for that class. The bound + function should try to download the JAR and return true if it did." + nil) + (defn- readable-file [^File f] (when (.canRead f) f)) @@ -102,31 +108,142 @@ sources-jar (io/file parent (str fname "-sources.jar"))] (readable-file sources-jar))) +(defn- find-jar-file-for-class [^Class klass] + ;; Get the classloader that loaded `klass` and locate the resource that the + ;; class was loaded from. If that resource is a JAR file, return it. + (when-let [cl (.getClassLoader klass)] + (let [class-file (class->classfile-path klass) + res (.getResource cl class-file)] + (some-> res parse-jar-path-from-url)))) + (defn- locate-source-url-near-class-jar "If `klass` comes from a third-party JAR that presumedly resides in `.m2` directory, try to look for a sources JAR near the class JAR and return it together with the implied source filename." [^Class klass] - (when-let [cl (some-> klass .getClassLoader)] - ;; Get the classloader that loaded the `klass` and locate the resource that - ;; the the class was loaded from. If that resource is a JAR file, search for - ;; a sources JAR near it. - (let [class-file (class->classfile-path klass) - res (.getResource cl class-file) - sources-jar-file (some-> res parse-jar-path-from-url infer-sources-jar-file) - source-filename (classfile-path->sourcefile-path class-file)] - (when sources-jar-file - (-> (combine-archive-url sources-jar-file source-filename) - verify-url-readable))))) + (let [class-jar-file (find-jar-file-for-class klass) + sources-jar-file (some-> class-jar-file infer-sources-jar-file) + class-file (class->classfile-path klass) + source-filename (classfile-path->sourcefile-path class-file)] + (when sources-jar-file + (-> (combine-archive-url sources-jar-file source-filename) + verify-url-readable)))) #_(locate-source-url-near-class-jar clojure.lang.PersistentVector) -(defn class->source-file-url ^URL [klass] +(defn class->source-file-url + "Resolve the URL path to the sources given `klass` using any of the supported + methods. If no method worked and `*download-sources-jar-fn*` is bound, invoke + it and try to resolve again." + ^URL [klass] (or (locate-source-url-on-classpath klass) (locate-source-url-in-jdk-sources klass) - (locate-source-url-near-class-jar klass))) + (locate-source-url-near-class-jar klass) + (and *download-sources-jar-fn* + (*download-sources-jar-fn* klass) + (locate-source-url-near-class-jar klass)))) #_(class->source-file-url mx.cider.orchard.LruMap) #_(class->source-file-url java.lang.Thread) #_(class->source-file-url clojure.lang.PersistentVector) #_(class->source-file-url clojure.core.Eduction) + +;;; ## Downloading Java sources from Maven + +(defn infer-maven-coordinates-for-class + "Given a class, attempt to parse its Maven coordinates (group id, artifact id, + version) from the JAR path it resides in." + [^Class klass] + (when-let [^File jar-file (find-jar-file-for-class klass)] + ;; Leap of faith here. We really infer this data from the filesystem path. + ;; The correct way would be to parse pom.xml inside the jar and get the + ;; coordinates from there, but that is more work. + (let [jar-file' (.getParentFile jar-file) + version (.getName jar-file') + jar-file'' (.getParentFile jar-file') + artifact (.getName jar-file'') + group-parts (loop [acc (), f jar-file''] + (let [parent (.getParentFile f) + name (some-> parent .getName)] + (if (and name (not= name "repository")) + (recur (cons name acc) parent) + acc)))] + {:artifact artifact + :group (string/join "." group-parts) + :version version}))) + +#_(infer-maven-coordinates-for-class clojure.lang.PersistentVector) + +(defn- download-sources-using-invoke-tool + "Download source JAR for given library coordinates using Clojure 1.12 + `invoke-tool` wrapper." + [{:keys [group artifact version]}] + (let [procurer (or ((requiring-resolve 'clojure.java.basis/current-basis)) + {:mvn/repos {"central" {:url "https://repo1.maven.org/maven2/"} + "clojars" {:url "https://repo.clojars.org/"}}}) + sources-artifact (symbol (format "%s/%s$sources" group artifact)) + lib-coords {sources-artifact {:mvn/version version}} + res ((requiring-resolve 'clojure.tools.deps.interop/invoke-tool) + {:tool-alias :deps + :fn 'clojure.tools.deps/resolve-added-libs + :args {:add lib-coords, :procurer procurer}})] + (vec (mapcat :paths (vals (:added res)))))) + +#_(download-sources-using-invoke-tool (infer-maven-coordinates-for-class clojure.lang.PersistentVector)) + +(defn- run-subprocess [args] + (println "[Orchard] Invoking subprocess:" (string/join " " args)) + (let [process (.start (ProcessBuilder. ^java.util.List args))] + (.waitFor process) + (let [out (slurp (.getInputStream process)) + err (slurp (.getErrorStream process))] + (when-not (string/blank? out) + (println out)) + (when-not (string/blank? err) + (binding [*out* *err*] (println err)))) + (.exitValue process))) + +(defn- download-sources-using-clojure-cli + "Download source JAR for given library coordinates by invoking `clojure` CLI + subprocess." + [{:keys [group artifact version]}] + (when (System/getProperty "clojure.basis") + (run-subprocess ["clojure" "-P" "-Sdeps" + (format "{:deps {%s/%s$sources {:mvn/version \"%s\"}}}" + group artifact version)]))) + +#_(download-sources-using-clojure-cli (infer-maven-coordinates-for-class clojure.lang.PersistentVector)) + +(defn- download-sources-using-lein + "Download source JAR for given library coordinates by invoking `lein` + subprocess." + [{:keys [group artifact version]}] + (run-subprocess + ;; "update-in :dependencies empty" is needed to drop the existing deps so + ;; that they don't mess with the sources JAR version we need. + ["lein" "update-in" ":dependencies" "empty" "--" + "update-in" ":dependencies" "conj" + (format "[%s/%s \"%s\" :classifier \"sources\"]" group artifact version) + "--" "deps"])) + +#_(download-sources-using-lein (infer-maven-coordinates-for-class clojure.lang.PersistentVector)) + +(defn download-sources-jar-for-coordinates + "Download source JAR for given library coordinates using either tools.deps or + Leiningen, depending how the Clojure process was started. Returns non-nil if + any of the methods at least attempted to download the artifact, or nil if + downloading didn't happen." + [maven-coordinates] + (let [clj-1-12+? (try (require 'clojure.tools.deps.interop) + (catch Exception _)) + tools-deps? (System/getProperty "clojure.basis")] + (cond (and tools-deps? clj-1-12+?) + (download-sources-using-invoke-tool maven-coordinates) + + tools-deps? + (download-sources-using-clojure-cli maven-coordinates) + + (System/getenv "LEIN_HOME") ;; Indicator of Leiningen-started process. + (download-sources-using-lein maven-coordinates)))) + +#_(download-sources-jar-for-coordinates (infer-maven-coordinates-for-class clojure.lang.PersistentVector)) diff --git a/test/orchard/java/source_files_test.clj b/test/orchard/java/source_files_test.clj index a542c299..937fd9fe 100644 --- a/test/orchard/java/source_files_test.clj +++ b/test/orchard/java/source_files_test.clj @@ -1,5 +1,5 @@ (ns orchard.java.source-files-test - (:require [clojure.test :refer [deftest is]] + (:require [clojure.test :refer [deftest is testing]] [orchard.java.source-files :as src-files])) (deftest class->source-file-url-test @@ -7,3 +7,46 @@ (is (src-files/class->source-file-url Thread)) ;; JDK (is (src-files/class->source-file-url clojure.lang.PersistentVector)) ;; Clojure (is (nil? (src-files/class->source-file-url clojure.core.Eduction)))) ;; record + +;; Download sources testing + +(deftest test-infer-maven-coordinates + (is (= (src-files/infer-maven-coordinates-for-class clojure.lang.Ref) + {:artifact "clojure", :group "org.clojure", :version (clojure-version)})) + + (is (nil? (src-files/infer-maven-coordinates-for-class String)) + "JDK classes don't resolve to a Maven coordinate.")) + +(defn- sources-jar-file ^java.io.File [klass] + (with-redefs [src-files/readable-file identity] + (#'src-files/infer-sources-jar-file + (#'src-files/find-jar-file-for-class klass)))) + +;; TODO: test for Clojure CLI and invoke tool at some point. +(deftest test-download-sources-jar-using-lein + (let [f (sources-jar-file clojure.lang.Ref)] + (.delete f) + (is (not (.exists f)))) + + (is (#'src-files/download-sources-using-lein + (src-files/infer-maven-coordinates-for-class clojure.lang.Ref))) + + (is (.exists (sources-jar-file clojure.lang.Ref))) + + (testing "downloaded source jars contain actual source files" + (is (> (count (slurp (src-files/class->source-file-url clojure.lang.Ref))) + 200)))) + +(deftest test-download-sources-jar + (let [f (sources-jar-file clojure.lang.Ref)] + (.delete f) + (is (not (.exists f)))) + + (is (src-files/download-sources-jar-for-coordinates + (src-files/infer-maven-coordinates-for-class clojure.lang.Ref))) + + (is (.exists (sources-jar-file clojure.lang.Ref))) + + (testing "downloaded source jars contain actual source files" + (is (> (count (slurp (src-files/class->source-file-url clojure.lang.Ref))) + 200))))