diff --git a/go.mod b/go.mod index 00b9fa3..723b93c 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21.6 require ( github.com/Code-Hex/go-generics-cache v1.5.1 - github.com/emanuelef/github-repo-activity-stats v0.2.29 + github.com/emanuelef/github-repo-activity-stats v0.2.30 github.com/gofiber/contrib/otelfiber v1.0.10 github.com/gofiber/fiber/v2 v2.52.5 github.com/joho/godotenv v1.5.1 diff --git a/go.sum b/go.sum index 0706cdc..1992910 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK3 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emanuelef/github-repo-activity-stats v0.2.29 h1:zJNiNWnBv8gBMIxakzy9SimJpGYz9pOJUA6CvK5b2oY= -github.com/emanuelef/github-repo-activity-stats v0.2.29/go.mod h1:Z6q8Ahan1HhKCArQovdXyjZAhHswriqTFGJ14fu7MKA= +github.com/emanuelef/github-repo-activity-stats v0.2.30 h1:Q6/nV+CsV6XhCI648t6VNhbq7/3oygIuZiaWNmgSK7A= +github.com/emanuelef/github-repo-activity-stats v0.2.30/go.mod h1:Z6q8Ahan1HhKCArQovdXyjZAhHswriqTFGJ14fu7MKA= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= diff --git a/main.go b/main.go index 723c5ee..52b15b9 100644 --- a/main.go +++ b/main.go @@ -55,6 +55,10 @@ type IssuesWithStatsResponse struct { Issues []stats.IssuesPerDay `json:"issues"` } +type ForksWithStatsResponse struct { + Forks []stats.ForksPerDay `json:"forks"` +} + func getEnv(key, fallback string) string { value, exists := os.LookupEnv(key) if !exists { @@ -106,9 +110,11 @@ func main() { cacheOverall := cache.New[string, *stats.RepoStats]() cacheStars := cache.New[string, StarsWithStatsResponse]() cacheIssues := cache.New[string, IssuesWithStatsResponse]() + cacheForks := cache.New[string, ForksWithStatsResponse]() onGoingStars := make(map[string]bool) onGoingIssues := make(map[string]bool) + onGoingForks := make(map[string]bool) ghStatClients := make(map[string]*repostats.ClientGQL) @@ -454,6 +460,110 @@ func main() { return c.JSON(res) }) + app.Get("/allForks", func(c *fiber.Ctx) error { + param := c.Query("repo") + randomIndex := rand.Intn(len(maps.Keys(ghStatClients))) + clientKey := c.Query("client", maps.Keys(ghStatClients)[randomIndex]) + forceRefetch := c.Query("forceRefetch", "false") == "true" + + client, ok := ghStatClients[clientKey] + if !ok { + return c.Status(404).SendString("Resource not found") + } + + repo, err := url.QueryUnescape(param) + if err != nil { + return err + } + + // needed because c.Query cannot be used as a map key + repo = fmt.Sprintf("%s", repo) + repo = strings.ToLower(repo) + + ip := c.Get("X-Forwarded-For") + + // If X-Forwarded-For is empty, fallback to RemoteIP + if ip == "" { + ip = c.IP() + } + + userAgent := c.Get("User-Agent") + log.Printf("Forks Request from IP: %s, Repo: %s User-Agent: %s\n", ip, repo, userAgent) + + if strings.Contains(userAgent, "python-requests") { + return c.Status(404).SendString("Custom 404 Error: Resource not found") + } + + span := trace.SpanFromContext(c.UserContext()) + span.SetAttributes(attribute.String("github.repo", repo)) + span.SetAttributes(attribute.String("caller.ip", ip)) + + if forceRefetch { + cacheForks.Delete(repo) + } + + if res, hit := cacheForks.Get(repo); hit { + return c.JSON(res) + } + + // if another request is already getting the data, skip and rely on SSE updates + if _, hit := onGoingForks[repo]; hit { + return c.SendStatus(fiber.StatusNoContent) + } + + onGoingForks[repo] = true + + updateChannel := make(chan int) + var allForks []stats.ForksPerDay + + eg, ctx := errgroup.WithContext(ctx) + + eg.Go(func() error { + allForks, err = client.GetAllForksHistory(ctx, repo, updateChannel) + if err != nil { + return err + } + return nil + }) + + for progress := range updateChannel { + // fmt.Printf("Progress: %d\n", progress) + + wg := &sync.WaitGroup{} + + for _, s := range currentSessions.Sessions { + wg.Add(1) + go func(cs *session.Session) { + defer wg.Done() + if cs.Repo == repo { + cs.StateChannel <- progress + } + }(s) + } + wg.Wait() + } + + if err := eg.Wait(); err != nil { + delete(onGoingForks, repo) + return err + } + + // defer close(updateChannel) + + res := ForksWithStatsResponse{ + Forks: allForks, + } + + now := time.Now() + nextDay := now.UTC().Truncate(24 * time.Hour).Add(DAY_CACHED * 24 * time.Hour) + durationUntilEndOfDay := nextDay.Sub(now) + + cacheForks.Set(repo, res, cache.WithExpiration(durationUntilEndOfDay)) + delete(onGoingForks, repo) + + return c.JSON(res) + }) + app.Get("/limits", func(c *fiber.Ctx) error { client, ok := ghStatClients["PAT"] if !ok { diff --git a/website/src/App.tsx b/website/src/App.tsx index eb3e022..f678d9f 100644 --- a/website/src/App.tsx +++ b/website/src/App.tsx @@ -5,6 +5,7 @@ import MainPage from "./MainPage"; import TimeSeriesChart from "./TimeSeriesChart"; import CompareChart from "./CompareChart"; import IssuesTimeSeriesChart from "./IssuesTimeSeriesChart"; +import ForksTimeSeriesChart from "./ForksTimeSeriesChart"; import CalendarChart from "./CalendarChart"; import InfoPage from "./InfoPage"; @@ -18,6 +19,7 @@ import QueryStatsRoundedIcon from "@mui/icons-material/QueryStatsRounded"; import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; import SsidChartRoundedIcon from "@mui/icons-material/SsidChartRounded"; import BugReportRoundedIcon from "@mui/icons-material/BugReportRounded"; +import AltRouteOutlinedIcon from "@mui/icons-material/AltRouteOutlined"; import { ThemeProvider, createTheme } from "@mui/material/styles"; import CssBaseline from "@mui/material/CssBaseline"; @@ -81,7 +83,8 @@ function App() { useLocation().pathname.includes("/table") || useLocation().pathname.includes("/calendar") || useLocation().pathname.includes("/info") || - useLocation().pathname.includes("/issues") + useLocation().pathname.includes("/issues") || + useLocation().pathname.includes("/forks") ) } > @@ -109,6 +112,17 @@ function App() { > Issues + } + icon={ + + + + } + active={useLocation().pathname.includes("/forks")} + > + Forks + } icon={ @@ -158,6 +172,7 @@ function App() { } /> } /> } /> + } /> diff --git a/website/src/ForksTimeSeriesChart.jsx b/website/src/ForksTimeSeriesChart.jsx new file mode 100644 index 0000000..71a39ee --- /dev/null +++ b/website/src/ForksTimeSeriesChart.jsx @@ -0,0 +1,787 @@ +/* eslint-disable no-case-declarations */ +import { useState, useEffect, useRef } from "react"; +import { useParams, useNavigate, useLocation } from "react-router-dom"; +import TextField from "@mui/material/TextField"; +import FormControl from "@mui/material/FormControl"; +import FormControlLabel from "@mui/material/FormControlLabel"; +import Tooltip from "@mui/material/Tooltip"; +import Checkbox from "@mui/material/Checkbox"; +import Button from "@mui/material/Button"; +import InputLabel from "@mui/material/InputLabel"; +import MenuItem from "@mui/material/MenuItem"; +import Select from "@mui/material/Select"; +import LoadingButton from "@mui/lab/LoadingButton"; +import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined"; +import SendIcon from "@mui/icons-material/Send"; +import FusionCharts from "fusioncharts"; +import TimeSeries from "fusioncharts/fusioncharts.timeseries"; +import ReactFC from "react-fusioncharts"; +import schema from "./schema-forks"; +import { parseISO, intervalToDuration } from "date-fns"; +import { parseGitHubRepoURL } from "./githubUtils"; +import GammelTheme from "fusioncharts/themes/fusioncharts.theme.gammel"; +import CandyTheme from "fusioncharts/themes/fusioncharts.theme.candy"; +import ZuneTheme from "fusioncharts/themes/fusioncharts.theme.zune"; +import UmberTheme from "fusioncharts/themes/fusioncharts.theme.umber"; +import GitHubButton from "react-github-btn"; +import { ToastContainer, toast } from "react-toastify"; +import "react-toastify/dist/ReactToastify.css"; +import { + addRunningMedian, + addRunningAverage, + addLOESS, + calculatePercentiles, +} from "./utils"; + +const HOST = import.meta.env.VITE_HOST; + +const YEARLY_BINNING = { + year: [1], + month: [], + day: [], + week: [], + hour: [], + minute: [], + second: [], +}; + +const MONTHLY_BINNING = { + year: [], + month: [1], + day: [], + week: [], + hour: [], + minute: [], + second: [], +}; + +const WEEKLY_BINNING = { + year: [], + month: [], + day: [], + week: [1], + hour: [], + minute: [], + second: [], +}; + +ReactFC.fcRoot( + FusionCharts, + TimeSeries, + GammelTheme, + CandyTheme, + ZuneTheme, + UmberTheme +); + +const formatDate = (originalDate) => { + const parts = originalDate.split("-"); + return `${parts[2]}-${parts[1]}-${parts[0]}`; +}; + +const FORCE_REFETCH_TOOLTIP = + "Using cached data, force refetching the data from GitHub. This will take a while if the repo has a lot of stars."; + +const INFO_TOOLTIP = + "Stars are fetched until UTC midnight of the previous day. \ + You can zoom inside the graph by scrolling up and down or dragging the selectors in the underline graph. \ + Once fetched the history is kept for 7 days but it's possible to refetch again by checking the Force Refetch checkbox."; + + +const isToday = (dateString) => { + const today = new Date(); + const [day, month, year] = dateString.split("-").map(Number); + return ( + today.getDate() === day && + today.getMonth() + 1 === month && // Adding 1 to month because JavaScript months are 0-indexed + today.getFullYear() === year + ); +}; + +function ForksTimeSeriesChart() { + let defaultRepo = "helm/helm"; + const { user, repository } = useParams(); + if (user && repository) { + defaultRepo = `${user}/${repository}`; + } + + const chart_props = { + type: "timeseries", + width: "100%", + height: "80%", + dataEmptyMessage: "Fetching data...", + styleDefinition: { + colorstyle: { + fill: "#ffff00", //color of the reference line + }, + }, + dataSource: { + tooltip: { + style: { + container: { + "border-color": "#000000", + "background-color": "#75748D", + }, + text: { + color: "#FFFFFF", + }, + }, + }, + yAxis: [ + { + plot: { + value: "Daily Forks", + type: "line", + }, + title: "Daily Forks", + aggregation: "average", + referenceline: [], + type: "", // can be log + }, + { + plot: { + value: "Total Forks", + type: "line", + }, + title: "Total Forks", + }, + ], + xAxis: { + plot: "Time", + timemarker: [], + binning: {}, + }, + chart: { + animation: "0", + theme: "candy", + exportEnabled: "1", + exportMode: "client", + exportFormats: "PNG=Export as PNG|PDF=Export as PDF", + }, + }, + }; + + const location = useLocation(); + const queryParams = new URLSearchParams(location.search); + + const [ds, setds] = useState(chart_props); + + const [totalStars, setTotalStars] = useState(0); + const [creationDate, setCreationDate] = useState("2021-01-01"); + const [age, setAge] = useState(""); + const [currentForksHistory, setCurrentForksHistory] = useState([]); + const [loading, setLoading] = useState(false); + const [showForceRefetch, setShowForceRefetch] = useState(false); + const [forceRefetch, setForceRefetch] = useState(false); + + const [theme, setTheme] = useState("candy"); + + const [transformation, setTransformation] = useState( + queryParams.get("transformation") || "none" + ); + + const [aggregation, setAggregation] = useState("average"); + + const [selectedTimeRange, setSelectedTimeRange] = useState({ + start: queryParams.get("start"), + end: queryParams.get("end"), + }); + + const navigate = useNavigate(); + + const [selectedRepo, setSelectedRepo] = useState(defaultRepo); + const [checkedDateRange, setCheckedDateRange] = useState(false); + + const currentSSE = useRef(null); + + const handleDateRangeCheckChange = (event) => { + setCheckedDateRange(event.target.checked); + }; + + const handleYAxisTypeCheckChange = (event) => { + setCheckedYAxisType(event.target.checked); + const options = { ...ds }; + options.dataSource.yAxis[0].type = event.target.checked ? "log" : ""; + setds(options); + }; + + const handleThemeChange = (event) => { + setTheme(event.target.value); + const options = { ...ds }; + options.dataSource.chart.theme = event.target.value; + setds(options); + }; + + const handleTransformationChange = (event) => { + setTransformation(event.target.value); + }; + + const handleAggregationChange = (event) => { + setAggregation(event.target.value); + const options = { ...ds }; + options.dataSource.yAxis[0].aggregation = event.target.value; + + let text = `${event.target.value} Stars`; + + options.dataSource.yAxis[0].plot.value = + schema[1].name = + options.dataSource.yAxis[0].title = + text; + + setds(options); + }; + + useEffect(() => { + updateGraph(currentForksHistory); + }, [transformation]); + + const handleForceRefetchChange = (event) => { + setForceRefetch(event.target.checked); + }; + + const fetchTotalStars = async (repo) => { + try { + const response = await fetch(`${HOST}/totalStars?repo=${repo}`); + + if (!response.ok) { + setLoading(false); + toast.error("Internal Server Error. Please try again later.", { + position: toast.POSITION.BOTTOM_CENTER, + }); + throw new Error(`HTTP error! Status: ${response.status}`); + } + const data = await response.json(); + return data; + } catch (error) { + console.error(`An error occurred: ${error}`); + setLoading(false); + } + }; + + const fetchStatus = async (repo) => { + try { + const response = await fetch(`${HOST}/status?repo=${repo}`); + + if (!response.ok) { + throw new Error(`HTTP error! Status: ${response.status}`); + } + const data = await response.json(); + return data; + } catch (error) { + console.error(`An error occurred: ${error}`); + } + }; + + const updateGraph = async (starHistory) => { + // check if last element is today + if (starHistory.length > 1) { + const lastElement = starHistory[starHistory.length - 1]; + console.log(lastElement); + console.log(starHistory); + const isLastElementToday = isToday(lastElement[0]); + starHistory.pop(); // remove last element as the current day is not complete + console.log("isLastElementToday", isLastElementToday); + setShowForceRefetch(!isLastElementToday); + setForceRefetch(false); + } else { + console.log("Array is empty."); + } + + let appliedTransformationResult = starHistory; + let binning = {}; + + const options = { ...ds }; + + console.log(starHistory.length); + + options.dataSource.subcaption = ""; + options.dataSource.yAxis[0].referenceline = []; + options.dataSource.yAxis[0].aggregation = "average"; + + let textBinning = ""; + + //console.log(res); + switch (transformation) { + case "none": + options.dataSource.yAxis[0].plot.value = + schema[1].name = + options.dataSource.yAxis[0].title = + "Daily Forks"; + options.dataSource.yAxis[0].plot.type = "line"; + options.dataSource.subcaption = ""; + break; + case "yearlyBinning": + textBinning = `Daily Forks ${aggregation} by Year`; + if (aggregation == "sum") { + textBinning = "Total Forks by Year"; + } + + options.dataSource.yAxis[0].plot.value = + schema[1].name = + options.dataSource.yAxis[0].title = + textBinning; + + binning = YEARLY_BINNING; + options.dataSource.yAxis[0].plot.type = "column"; + options.dataSource.yAxis[0].aggregation = aggregation; + break; + case "monthlyBinning": + textBinning = `Daily Opened ${aggregation} by Month`; + if (aggregation == "sum") { + textBinning = "Total Opened by Year"; + } + + options.dataSource.yAxis[0].plot.value = + schema[1].name = + options.dataSource.yAxis[0].title = + textBinning; + binning = MONTHLY_BINNING; + options.dataSource.yAxis[0].plot.type = "column"; + options.dataSource.yAxis[0].aggregation = aggregation; + break; + case "weeklyBinning": + textBinning = `Daily Opened ${aggregation} by Week`; + if (aggregation == "sum") { + ("Total Opened by Year"); + } + options.dataSource.yAxis[0].plot.value = + schema[1].name = + options.dataSource.yAxis[0].title = + textBinning; + binning = WEEKLY_BINNING; + options.dataSource.yAxis[0].plot.type = "column"; + options.dataSource.yAxis[0].aggregation = aggregation; + break; + case "normalize": + options.dataSource.yAxis[0].plot.value = + schema[1].name = + options.dataSource.yAxis[0].title = + "Normalized"; + + const [median, highPercentile] = calculatePercentiles( + starHistory + .filter((subArray) => subArray[1] > 0) + .map((subArray) => subArray[1]), + 0.5, + 0.98 + ); + + console.log(median, highPercentile); + + appliedTransformationResult = starHistory.map((subArray) => { + if (subArray[1] > highPercentile) { + return [subArray[0], highPercentile, subArray[2]]; + } + return subArray; + }); + options.dataSource.yAxis[0].plot.type = "line"; + + options.dataSource.yAxis[0].referenceline = [ + { + label: "Median", + value: median, + }, + ]; + + break; + case "loess": + options.dataSource.yAxis[0].plot.value = + schema[1].name = + options.dataSource.yAxis[0].title = + "LOESS"; + appliedTransformationResult = addLOESS(starHistory, 0.08); + options.dataSource.yAxis[0].plot.type = "line"; + + /* options.dataSource.xAxis.initialinterval = { + from: "01-01-2022", + to: "01-01-2023", + }; */ + + break; + case "runningAverage": + options.dataSource.yAxis[0].plot.value = + schema[1].name = + options.dataSource.yAxis[0].title = + "Running Average"; + appliedTransformationResult = addRunningAverage(starHistory, 120); + options.dataSource.yAxis[0].plot.type = "line"; + break; + case "runningMedian": + options.dataSource.yAxis[0].plot.value = + schema[1].name = + options.dataSource.yAxis[0].title = + "Running Median"; + appliedTransformationResult = addRunningMedian(starHistory, 120); + options.dataSource.yAxis[0].plot.type = "line"; + break; + default: + break; + } + + const fusionTable = new FusionCharts.DataStore().createDataTable( + appliedTransformationResult, + schema + ); + + options.dataSource.data = fusionTable; + + options.dataSource.xAxis.binning = binning; + options.dataSource.chart.theme = theme; + options.dataSource.chart.exportFileName = `${selectedRepo.replace( + "/", + "_" + )}-stars-history`; + + console.log(options.dataSource.yAxis); + + setds(options); + }; + + const fetchAllForks = async (repo, ignoreForceRefetch = false) => { + console.log(repo); + + setCurrentForksHistory([]); + + let fetchUrl = `${HOST}/allForks?repo=${repo}`; + + if (forceRefetch && !ignoreForceRefetch) { + fetchUrl += "&forceRefetch=true"; + } + + fetch(fetchUrl) + .then((response) => { + // Check if the response status indicates success (e.g., 200 OK) + if (!response.ok) { + throw new Error(`HTTP error! Status: ${response.status}`); + } + // Attempt to parse the response as JSON + return response.json(); + }) + .then((data) => { + setLoading(false); + console.log(data); + const starHistory = data.forks; + setCurrentForksHistory(starHistory); + updateGraph(starHistory); + const options = { ...ds }; + options.dataSource.caption = { text: `Forks ${repo}` }; + + setds(options); + }) + .catch((e) => { + console.error(`An error occurred: ${e}`); + setLoading(false); + }); + }; + + const openCurrentRepoPage = () => { + const repoParsed = parseGitHubRepoURL(selectedRepo); + window.open("https://github.com/" + repoParsed, "_blank"); + }; + + const closeSSE = () => { + if (currentSSE.current) { + console.log("STOP SSE"); + currentSSE.current.close(); + } + }; + + const startSSEUpates = (repo, callsNeeded, onGoing) => { + console.log(repo, callsNeeded, onGoing); + const sse = new EventSource(`${HOST}/sse?repo=${repo}`); + closeSSE(); + currentSSE.current = sse; + + sse.onerror = (err) => { + console.log("on error", err); + }; + + // The onmessage handler is called if no event name is specified for a message. + sse.onmessage = (msg) => { + console.log("on message", msg); + }; + + sse.onopen = (...args) => { + console.log("on open", args); + }; + + sse.addEventListener("current-value", (event) => { + const parsedData = JSON.parse(event.data); + const currentValue = parsedData.data; + + // console.log("currentValue", currentValue, callsNeeded); + + if (currentValue === callsNeeded) { + console.log("CLOSE SSE"); + closeSSE(); + //if (onGoing) { + setTimeout(() => { + fetchAllForks(repo, true); + }, 1600); + //} + setLoading(false); + } + }); + }; + + useEffect(() => { + handleClick(); + }, []); + + const handleClick = async () => { + const repoParsed = parseGitHubRepoURL(selectedRepo); + + if (repoParsed === null) { + return; + } + + setLoading(true); + + const res = await fetchTotalStars(repoParsed); + // console.log(res); + + if (res) { + setTotalStars(res.stars); + setCreationDate(res.createdAt); + + const { years, months, days } = intervalToDuration({ + start: parseISO(res.createdAt), + end: Date.now(), + }); + setAge( + `${years && years !== 0 ? `${years}y ` : ""}${ + months && months !== 0 ? `${months}m ` : "" + }${days && days !== 0 ? `${days}d ` : ""}` + ); + } + + const status = await fetchStatus(repoParsed); + console.log(status); + + if (!status.onGoing) { + fetchAllForks(repoParsed); + } + + const callsNeeded = Math.floor(res.stars / 100); + startSSEUpates(repoParsed, callsNeeded, status.onGoing); + }; + + const handleInputChange = async (event, setStateFunction) => { + const inputText = event.target.value; + setStateFunction(inputText); + }; + + return ( +
+ +
+ handleInputChange(e, setSelectedRepo)} + onKeyDown={(e) => { + if (e.key === "Enter") { + handleClick(); + } + }} + /> + } + loading={loading} + loadingPosition="end" + variant="contained" + > + Fetch + + {showForceRefetch && ( + + + } + label="Force Refetch" + /> + + )} + + + + + + +
+
+
+ + Theme + + +
+ + {/* + + { + + } + + With Date Range */} + +
+ + Star Me + +
+
+{/* + */} +
+ {ds != null && ds != chart_props && ds && ds.dataSource.data && ( + + )} +
+
+ ); +} + +export default ForksTimeSeriesChart; diff --git a/website/src/schema-forks.js b/website/src/schema-forks.js new file mode 100644 index 0000000..127ad6b --- /dev/null +++ b/website/src/schema-forks.js @@ -0,0 +1,17 @@ +const schema = [ + { + name: "Time", + type: "date", + format: "%-d-%-m-%Y", + }, + { + name: "Daily Forks", + type: "number", + }, + { + name: "Total Forks", + type: "number", + }, +]; + +export default schema;