Merge pull request #278 from guusdk/github-stats

Add GitHub statistics

src/main/java/org/jivesoftware/site/GitHubAPI.java

@@ -0,0 +1,175 @@
+package org.jivesoftware.site;
+
+import org.jivesoftware.webservices.RestClient;
+import org.json.JSONArray;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.*;
+
+public class GitHubAPI extends HttpServlet
+{
+    private static final Logger Log = LoggerFactory.getLogger(GitHubAPI.class);
+
+    private static long CACHE_PERIOD = 30 * 60 * 1000; // 30 minutes
+
+    private static long lastUpdate = 0;
+
+    private static Map<String, Long> counts = new Hashtable<>();
+
+    private static RestClient restClient;
+
+    private static final String TOTAL = "TOTAL";
+
+    public void init(ServletConfig servletConfig) throws ServletException
+    {
+        super.init(servletConfig);
+
+        restClient = new RestClient();
+    }
+
+    public static Long getTotalCommitCountLastWeek()
+    {
+        collectTotals();
+        if (counts.containsKey(TOTAL)) {
+            return counts.get(TOTAL);
+        }
+        return null;
+    }
+
+    public static Long getCommitCountLastWeek(final String repoName)
+    {
+        collectTotals();
+        if (counts.containsKey(repoName.toLowerCase())) {
+            return counts.get(repoName.toLowerCase());
+        }
+        return null;
+    }
+
+    public static Long getCommitCountLastWeekPartialName(final String partialRepoName)
+    {
+        collectTotals();
+
+        long result = 0L;
+        for (final Map.Entry<String, Long> entry : counts.entrySet()) {
+            if (entry.getKey().toLowerCase().contains(partialRepoName.toLowerCase()) && !entry.getKey().equals(TOTAL) ) {
+                result += entry.getValue();
+            }
+        }
+        return result;
+    }
+
+    /**
+     * Retrieves last week's commit count of a repository on GitHub.
+     *
+     * @param repo The name of the repository without the .git extension. The name is not case sensitive.
+     * @return the query result, or null if an exception occurred.
+     * @see <a href="https://docs.github.com/en/rest/metrics/statistics?apiVersion=2022-11-28#get-the-weekly-commit-count">https://docs.github.com/en/rest/metrics/statistics?apiVersion=2022-11-28#get-the-weekly-commit-count</a>
+     */
+    private static Long getLastWeekCommitCount(final String repo) {
+        final Map<String, String> headers = new HashMap<>();
+        headers.put("Accept", "application/vnd.github+json");
+        headers.put("X-GitHub-Api-Version", "2022-11-28");
+
+        // TODO The current implementation may retrieve _last_ weeks activity, instead of this weeks (the last 7 days). Find a way to count commits in the last 7 days.
+        try {
+            final JSONArray all = restClient.get("https://api.github.com/repos/igniterealtime/" + repo + "/stats/participation", headers).getJSONArray("all");
+            return all.getLong(all.length() - 1);
+        } catch (Throwable t) {
+            Log.warn("Unable to interact with GitHub's API.", t);
+            return null;
+        }
+    }
+
+    /**
+     * Returns the names of all public repositories in our GitHub organisation.
+     *
+     * @return names for all public repositories.
+     * @see <a href="https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-organization-repositories">https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-organization-repositories</a>
+     */
+    private static Set<String> getRepositoryNames() {
+        final Map<String, String> headers = new HashMap<>();
+        headers.put("Accept", "application/vnd.github+json");
+        headers.put("X-GitHub-Api-Version", "2022-11-28");
+
+        try {
+            final Set<String> results = new HashSet<>();
+            for (int page = 1; page <= 10; page++) {
+                final JSONArray pagedData = restClient.getAsArray("https://api.github.com/orgs/igniterealtime/repos?type=public&sort=pushed&per_page=100&page=" + page, headers);
+                if (pagedData == null || pagedData.isEmpty()) {
+                    break;
+                }
+                for (int i = 0; i < pagedData.length(); i++) {
+                    results.add(pagedData.getJSONObject(i).getString("name"));
+
+                    // TODO Authenticate these requests to unlock better rate limits, allowing us to pull from _all_ repos! We'll now settle for the last few active repos, but that's not ideal.
+                    if (results.size() >= 10) {
+                        break;
+                    }
+                }
+            }
+            return results;
+        } catch (Throwable t) {
+            Log.warn("Unable to interact with GitHub's API.", t);
+            return null;
+        }
+    }
+
+    /**
+     * Collects all of the totals from the API. Has a rudimentary caching mechanism
+     * so that the queries are only run every CACHE_PERIOD milliseconds.
+     */
+    private synchronized static void collectTotals() {
+        // See if we need to update the totals
+        if ((lastUpdate + CACHE_PERIOD) > System.currentTimeMillis()) {
+            return;
+        }
+        lastUpdate = System.currentTimeMillis();
+
+        // Collect the new totals on a background thread since they could take a while
+        Thread collectorThread = new Thread(new GitHubAPI.DownloadStatsRunnable(counts));
+        collectorThread.start();
+        if (counts.isEmpty()) {
+            // Need to wait for the collectorThread to finish since the counts are not initialized yet
+            try {
+                collectorThread.join();
+            }
+            catch (Exception e) { Log.info( "An exception occurred while collecting GitHub stats.", e); }
+        }
+    }
+
+    private static class DownloadStatsRunnable implements Runnable {
+        private Map<String, Long> counts;
+
+        public DownloadStatsRunnable(Map<String, Long> counts) {
+            this.counts = counts;
+        }
+
+        public void run() {
+            Log.debug("Retrieving GitHub statistics...");
+
+            Instant start = Instant.now();
+            final Map<String, Long> results = new HashMap<>();
+            final Set<String> repoNames = getRepositoryNames();
+            if (repoNames != null) {
+                repoNames.forEach(repo -> results.put(repo.toLowerCase(), getLastWeekCommitCount(repo)));
+            }
+            final Long total = results.values().stream().filter(Objects::nonNull).mapToLong(Long::longValue).sum();
+            results.put(TOTAL, total);
+
+            Log.info("Queried all GitHub stats in {}", Duration.between(start, Instant.now()));
+
+            // Replace all values in the object used by the website in one go.
+            counts.clear();
+            counts.putAll(results);
+
+            Log.debug("Retrieved GitHub statistics:");
+            results.forEach((key, value) -> Log.debug("- {} : {}", key, value));
+        }
+    }
+}

src/main/java/org/jivesoftware/webservices/RestClient.java

@@ -6,6 +6,7 @@
 import org.apache.hc.core5.http.*;
 import org.apache.hc.core5.http.io.entity.EntityUtils;
 import org.apache.hc.core5.http.io.support.ClassicRequestBuilder;
+import org.json.JSONArray;
 import org.json.JSONException;
 import org.json.JSONObject;
 import org.slf4j.Logger;
@@ -24,24 +25,55 @@ public class RestClient {
         .build();
 
     public JSONObject get(String url) {
-        JSONObject result = null;
+        return get(url, null);
+    }
+
+    public JSONObject get(String url, Map<String, String> headers)
+    {
+        try {
+            final String result = getAsString(url, headers);
+            if (result == null) {
+                return null;
+            }
+            return new JSONObject(result);
+        } catch (JSONException e) {
+            Log.warn("Invalid content while querying '{}' for a JSON Object", url, e);
+            return null;
+        }
+    }
+
+    public JSONArray getAsArray(String url, Map<String, String> headers)
+    {
+        try {
+            final String result = getAsString(url, headers);
+            if (result == null) {
+                return null;
+            }
+            return new JSONArray(result);
+        } catch (JSONException e) {
+            Log.warn("Invalid content while querying '{}' for a JSON Array", url, e);
+            return null;
+        }
+    }
+
+    public String getAsString(String url, Map<String, String> headers) {
 
         try (final CloseableHttpClient httpclient = CachingHttpClients.custom().setCacheConfig(cacheConfig).build())
         {
-            final ClassicHttpRequest httpGet = ClassicRequestBuilder.get(url).build();
-            result = httpclient.execute(httpGet, response -> {
-                try {
-                    return new JSONObject(EntityUtils.toString(response.getEntity()));
-                } catch (JSONException e) {
-                    Log.warn("Invalid content while querying '{}'", url, e);
-                    return null;
+            final ClassicRequestBuilder builder = ClassicRequestBuilder.get(url);
+            if (headers != null) {
+                for (Map.Entry<String, String> header : headers.entrySet()) {
+                    builder.addHeader(header.getKey(), header.getValue());
                 }
-            });
+            }
+
+            final ClassicHttpRequest httpGet = builder.build();
+            return httpclient.execute(httpGet, response -> EntityUtils.toString(response.getEntity()));
         } catch (IOException e) {
             Log.warn("Fatal transport error while querying '{}'", url, e);
         }
 
-        return result;
+        return null;
     }
 
     public JSONObject post(String url, Map<String, String> headers, Map<String, String> parameters)

src/main/webapp/WEB-INF/web.xml

@@ -108,6 +108,11 @@ http://java.sun.com/xml/ns/j2ee/web-app_2_4.xsd"
         <servlet-class>org.jivesoftware.site.DiscourseAPI</servlet-class>
         <load-on-startup>1</load-on-startup>
     </servlet>
+    <servlet>
+        <servlet-name>GitHubAPI</servlet-name>
+        <servlet-class>org.jivesoftware.site.GitHubAPI</servlet-class>
+        <load-on-startup>1</load-on-startup>
+    </servlet>
     <servlet>
         <servlet-name>DownloadServlet</servlet-name>
         <servlet-class>org.jivesoftware.site.DownloadServlet</servlet-class>

src/main/webapp/includes/sidebar_7daySnapshot.jspf

@@ -2,10 +2,12 @@
 <%@ taglib prefix="fmt" uri="http://java.sun.com/jsp/jstl/fmt" %>
 <%@ page import="org.jivesoftware.site.DownloadStats" %>
 <%@ page import="org.jivesoftware.site.DiscourseAPI" %>
+<%@ page import="org.jivesoftware.site.GitHubAPI" %>
 <%
     request.setAttribute("downloadsLast7Days", DownloadStats.getTotalDownloadsLast7Days());
     request.setAttribute("activeMembers", DiscourseAPI.getActiveMembersLast7Days());
     request.setAttribute("newPosts", DiscourseAPI.getNewPostsLast7Days());
+    request.setAttribute("commits", GitHubAPI.getTotalCommitCountLastWeek());
 %>
 <div class="ignite_sidebar_bluebox_photo">
     <div class="ignite_sidebar_top"></div>
@@ -31,6 +33,12 @@
 <%--            <div class="ignite_sidebar_body_stat"><span>Blog Entries</span>--%>
 <%--                <strong><%= blogService48.getBlogPostCount() %></strong>--%>
 <%--            </div>--%>
+        <cache:cache time="30" key="/github/statistics/usage">
+            <c:if test="${not empty commits}">
+                <div class="ignite_sidebar_body_stat"><span>Code Commits</span>
+                    <strong><fmt:formatNumber value="${commits}"/></strong>
+                </div>
+            </c:if>
         </cache:cache>
         <div class="ignite_sidebar_body_stat">
             <em>Activity in last 7 days</em>