diff --git a/src/main/java/org/jivesoftware/site/GitHubAPI.java b/src/main/java/org/jivesoftware/site/GitHubAPI.java new file mode 100644 index 00000000..6a84173e --- /dev/null +++ b/src/main/java/org/jivesoftware/site/GitHubAPI.java @@ -0,0 +1,175 @@ +package org.jivesoftware.site; + +import org.jivesoftware.webservices.RestClient; +import org.json.JSONArray; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.ServletConfig; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import java.time.Duration; +import java.time.Instant; +import java.util.*; + +public class GitHubAPI extends HttpServlet +{ + private static final Logger Log = LoggerFactory.getLogger(GitHubAPI.class); + + private static long CACHE_PERIOD = 30 * 60 * 1000; // 30 minutes + + private static long lastUpdate = 0; + + private static Map counts = new Hashtable<>(); + + private static RestClient restClient; + + private static final String TOTAL = "TOTAL"; + + public void init(ServletConfig servletConfig) throws ServletException + { + super.init(servletConfig); + + restClient = new RestClient(); + } + + public static Long getTotalCommitCountLastWeek() + { + collectTotals(); + if (counts.containsKey(TOTAL)) { + return counts.get(TOTAL); + } + return null; + } + + public static Long getCommitCountLastWeek(final String repoName) + { + collectTotals(); + if (counts.containsKey(repoName.toLowerCase())) { + return counts.get(repoName.toLowerCase()); + } + return null; + } + + public static Long getCommitCountLastWeekPartialName(final String partialRepoName) + { + collectTotals(); + + long result = 0L; + for (final Map.Entry entry : counts.entrySet()) { + if (entry.getKey().toLowerCase().contains(partialRepoName.toLowerCase()) && !entry.getKey().equals(TOTAL) ) { + result += entry.getValue(); + } + } + return result; + } + + /** + * Retrieves last week's commit count of a repository on GitHub. + * + * @param repo The name of the repository without the .git extension. The name is not case sensitive. + * @return the query result, or null if an exception occurred. + * @see https://docs.github.com/en/rest/metrics/statistics?apiVersion=2022-11-28#get-the-weekly-commit-count + */ + private static Long getLastWeekCommitCount(final String repo) { + final Map headers = new HashMap<>(); + headers.put("Accept", "application/vnd.github+json"); + headers.put("X-GitHub-Api-Version", "2022-11-28"); + + // TODO The current implementation may retrieve _last_ weeks activity, instead of this weeks (the last 7 days). Find a way to count commits in the last 7 days. + try { + final JSONArray all = restClient.get("https://api.github.com/repos/igniterealtime/" + repo + "/stats/participation", headers).getJSONArray("all"); + return all.getLong(all.length() - 1); + } catch (Throwable t) { + Log.warn("Unable to interact with GitHub's API.", t); + return null; + } + } + + /** + * Returns the names of all public repositories in our GitHub organisation. + * + * @return names for all public repositories. + * @see https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-organization-repositories + */ + private static Set getRepositoryNames() { + final Map headers = new HashMap<>(); + headers.put("Accept", "application/vnd.github+json"); + headers.put("X-GitHub-Api-Version", "2022-11-28"); + + try { + final Set results = new HashSet<>(); + for (int page = 1; page <= 10; page++) { + final JSONArray pagedData = restClient.getAsArray("https://api.github.com/orgs/igniterealtime/repos?type=public&sort=pushed&per_page=100&page=" + page, headers); + if (pagedData == null || pagedData.isEmpty()) { + break; + } + for (int i = 0; i < pagedData.length(); i++) { + results.add(pagedData.getJSONObject(i).getString("name")); + + // TODO Authenticate these requests to unlock better rate limits, allowing us to pull from _all_ repos! We'll now settle for the last few active repos, but that's not ideal. + if (results.size() >= 10) { + break; + } + } + } + return results; + } catch (Throwable t) { + Log.warn("Unable to interact with GitHub's API.", t); + return null; + } + } + + /** + * Collects all of the totals from the API. Has a rudimentary caching mechanism + * so that the queries are only run every CACHE_PERIOD milliseconds. + */ + private synchronized static void collectTotals() { + // See if we need to update the totals + if ((lastUpdate + CACHE_PERIOD) > System.currentTimeMillis()) { + return; + } + lastUpdate = System.currentTimeMillis(); + + // Collect the new totals on a background thread since they could take a while + Thread collectorThread = new Thread(new GitHubAPI.DownloadStatsRunnable(counts)); + collectorThread.start(); + if (counts.isEmpty()) { + // Need to wait for the collectorThread to finish since the counts are not initialized yet + try { + collectorThread.join(); + } + catch (Exception e) { Log.info( "An exception occurred while collecting GitHub stats.", e); } + } + } + + private static class DownloadStatsRunnable implements Runnable { + private Map counts; + + public DownloadStatsRunnable(Map counts) { + this.counts = counts; + } + + public void run() { + Log.debug("Retrieving GitHub statistics..."); + + Instant start = Instant.now(); + final Map results = new HashMap<>(); + final Set repoNames = getRepositoryNames(); + if (repoNames != null) { + repoNames.forEach(repo -> results.put(repo.toLowerCase(), getLastWeekCommitCount(repo))); + } + final Long total = results.values().stream().filter(Objects::nonNull).mapToLong(Long::longValue).sum(); + results.put(TOTAL, total); + + Log.info("Queried all GitHub stats in {}", Duration.between(start, Instant.now())); + + // Replace all values in the object used by the website in one go. + counts.clear(); + counts.putAll(results); + + Log.debug("Retrieved GitHub statistics:"); + results.forEach((key, value) -> Log.debug("- {} : {}", key, value)); + } + } +} diff --git a/src/main/java/org/jivesoftware/webservices/RestClient.java b/src/main/java/org/jivesoftware/webservices/RestClient.java index 94024a09..e83ddfdb 100644 --- a/src/main/java/org/jivesoftware/webservices/RestClient.java +++ b/src/main/java/org/jivesoftware/webservices/RestClient.java @@ -6,6 +6,7 @@ import org.apache.hc.core5.http.*; import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.http.io.support.ClassicRequestBuilder; +import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.slf4j.Logger; @@ -24,24 +25,55 @@ public class RestClient { .build(); public JSONObject get(String url) { - JSONObject result = null; + return get(url, null); + } + + public JSONObject get(String url, Map headers) + { + try { + final String result = getAsString(url, headers); + if (result == null) { + return null; + } + return new JSONObject(result); + } catch (JSONException e) { + Log.warn("Invalid content while querying '{}' for a JSON Object", url, e); + return null; + } + } + + public JSONArray getAsArray(String url, Map headers) + { + try { + final String result = getAsString(url, headers); + if (result == null) { + return null; + } + return new JSONArray(result); + } catch (JSONException e) { + Log.warn("Invalid content while querying '{}' for a JSON Array", url, e); + return null; + } + } + + public String getAsString(String url, Map headers) { try (final CloseableHttpClient httpclient = CachingHttpClients.custom().setCacheConfig(cacheConfig).build()) { - final ClassicHttpRequest httpGet = ClassicRequestBuilder.get(url).build(); - result = httpclient.execute(httpGet, response -> { - try { - return new JSONObject(EntityUtils.toString(response.getEntity())); - } catch (JSONException e) { - Log.warn("Invalid content while querying '{}'", url, e); - return null; + final ClassicRequestBuilder builder = ClassicRequestBuilder.get(url); + if (headers != null) { + for (Map.Entry header : headers.entrySet()) { + builder.addHeader(header.getKey(), header.getValue()); } - }); + } + + final ClassicHttpRequest httpGet = builder.build(); + return httpclient.execute(httpGet, response -> EntityUtils.toString(response.getEntity())); } catch (IOException e) { Log.warn("Fatal transport error while querying '{}'", url, e); } - return result; + return null; } public JSONObject post(String url, Map headers, Map parameters) diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index 7897e76d..c170d7fd 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -108,6 +108,11 @@ http://java.sun.com/xml/ns/j2ee/web-app_2_4.xsd" org.jivesoftware.site.DiscourseAPI 1 + + GitHubAPI + org.jivesoftware.site.GitHubAPI + 1 + DownloadServlet org.jivesoftware.site.DownloadServlet diff --git a/src/main/webapp/includes/sidebar_7daySnapshot.jspf b/src/main/webapp/includes/sidebar_7daySnapshot.jspf index bf9ee53c..7fb11c3c 100644 --- a/src/main/webapp/includes/sidebar_7daySnapshot.jspf +++ b/src/main/webapp/includes/sidebar_7daySnapshot.jspf @@ -2,10 +2,12 @@ <%@ taglib prefix="fmt" uri="http://java.sun.com/jsp/jstl/fmt" %> <%@ page import="org.jivesoftware.site.DownloadStats" %> <%@ page import="org.jivesoftware.site.DiscourseAPI" %> +<%@ page import="org.jivesoftware.site.GitHubAPI" %> <% request.setAttribute("downloadsLast7Days", DownloadStats.getTotalDownloadsLast7Days()); request.setAttribute("activeMembers", DiscourseAPI.getActiveMembersLast7Days()); request.setAttribute("newPosts", DiscourseAPI.getNewPostsLast7Days()); + request.setAttribute("commits", GitHubAPI.getTotalCommitCountLastWeek()); %>
@@ -31,6 +33,12 @@ <%--
Blog Entries--%> <%-- <%= blogService48.getBlogPostCount() %>--%> <%--
--%> + + +
Code Commits + +
+
Activity in last 7 days