Add reports for issues and comments by day

docs/_data/navigation.yml

@@ -5,6 +5,8 @@
       url: "/"
     - title: "Contributors"
       url: "/users-contributors"
+    - title: "Issues & Comments"
+      url: "/users-communication"
     - title: "Activity"
       url: "/users-activity"
     - title: "Git Versions"

docs/users-communication.html

@@ -0,0 +1,163 @@
+---
+layout: default
+title: Issues & Comments
+permalink: /users-communication
+---
+
+<div id="issues" class="chart-placeholder">
+	<h3>Issues</h3>
+	<canvas
+		data-url="{{ site.dataURL }}/issues.tsv"
+		data-type="history"
+		data-config='{
+			"views":
+			[
+				{
+					"label": "2 m",
+					"tooltip": "Show the last 2 months",
+					"aggregate": false,
+					"series":
+					[
+						"issues created",
+						"issues closed",
+						"issues updated"
+					],
+					"visibleSeries":
+					[
+						"issues created",
+						"issues closed",
+						"issues updated"
+					],
+					"slice": [0, 61]
+				},
+				{
+					"label": "2 y",
+					"tooltip": "Show the last 2 years",
+					"aggregate":
+					{
+						"period": "week",
+						"method": "sum"
+					},
+					"series":
+					[
+						"issues created",
+						"issues closed",
+						"issues updated"
+					],
+					"visibleSeries":
+					[
+						"issues created",
+						"issues closed",
+						"issues updated"
+					],
+					"slice": [0, 106],
+					"default": true
+				},
+				{
+					"label": "all",
+					"tooltip": "Show all data",
+					"aggregate":
+					{
+						"period": "week",
+						"method": "sum"
+					},
+					"series":
+					[
+						"issues created",
+						"issues closed",
+						"issues updated"
+					],
+					"visibleSeries":
+					[
+						"issues created",
+						"issues closed",
+						"issues updated"
+					]
+				}
+			]
+		}'></canvas>
+	<div class="info-box">
+		<p>
+			<i>Issues</i> shows how many GitHub issues were created, updated, or closed. <i>issues updated</i> excludes creating or closing an issue.
+		</p>
+	</div>
+</div>
+
+<div id="comments" class="chart-placeholder">
+	<h3>Comments</h3>
+	<canvas
+		data-url="{{ site.dataURL }}/comments.tsv"
+		data-type="history"
+		data-config='{
+			"views":
+			[
+				{
+					"label": "2 m",
+					"tooltip": "Show the last 2 months",
+					"aggregate": false,
+					"series":
+					[
+						"issue comments",
+						"pull request comments",
+						"commit comments"
+					],
+					"visibleSeries":
+					[
+						"issue comments",
+						"pull request comments",
+						"commit comments"
+					],
+					"slice": [0, 61]
+				},
+				{
+					"label": "2 y",
+					"tooltip": "Show the last 2 years",
+					"aggregate":
+					{
+						"period": "week",
+						"method": "sum"
+					},
+					"series":
+					[
+						"issue comments",
+						"pull request comments",
+						"commit comments"
+					],
+					"visibleSeries":
+					[
+						"issue comments",
+						"pull request comments",
+						"commit comments"
+					],
+					"slice": [0, 106],
+					"default": true
+				},
+				{
+					"label": "all",
+					"tooltip": "Show all data",
+					"aggregate":
+					{
+						"period": "week",
+						"method": "sum"
+					},
+					"series":
+					[
+						"issue comments",
+						"pull request comments",
+						"commit comments"
+					],
+					"visibleSeries":
+					[
+						"issue comments",
+						"pull request comments",
+						"commit comments"
+					]
+				}
+			]
+		}'></canvas>
+	<div class="info-box">
+		<p>
+			<i>Comments</i> shows how many comments ware created for issues, pull requests, and commits. <i>pull request comments</i> includes comments on the PR as well as the ones for reviews.
+		</p>
+	</div>
+</div>

updater/reports/ReportComments.py

@@ -0,0 +1,47 @@
+from .ReportDailySQL import *
+
+# Lists how many issues got created, closed, were commented on, and number of comments per day
+class ReportComments(ReportDailySQL):
+	def name(self):
+		return "comments"
+
+	def updateDailyData(self):
+		self.header, newData = self.parseData(self.executeQuery(self.query(self.timeRangeToUpdate())))
+		self.data.extend(newData)
+		self.truncateData(self.timeRangeTotal())
+		self.sortDataByDate()
+
+	# Collects the issues closed in time range
+	def subquery(self, timeRange, table, extra_from=None, extra_where=None):
+		return '''
+			SELECT
+				DATE_FORMAT(''' + table + '''.created_at, "%Y-%m-%d") AS date,
+				COUNT(*) AS count
+			FROM
+				''' + table + '''
+				JOIN users ON users.id = ''' + table + '''.user_id ''' + (("\n\t\t\t\t\t" + extra_from) if extra_from else '') + '''
+			WHERE
+				CAST(''' + table + '''.created_at AS DATE) BETWEEN
+					"''' + str(timeRange[0]) + '''" AND "''' + str(timeRange[1]) + '''"
+					''' + self.andExcludedUsers("users.login") + (("\n\t\t\t\t\t" + extra_where) if extra_where else '') + '''
+			GROUP BY
+				date_format(''' + table + '''.created_at, "%Y-%m-%d")'''
+
+	# Collects the number of issues created, closed, commented, and number of comments
+	def query(self, timeRange):
+		# `alldays` is used as a basis for LEFT JOIN, to prevent issues when querying multiple days, e.g. for initial run of the new report 
+		return '''
+			SELECT
+			    alldays.date AS date,
+				IFNULL(i_issues.count, 0) AS "issue comments",
+				IFNULL(pr_issues.count, 0) AS "pull request issue comments",
+				IFNULL(reviews.count, 0) AS "pull request review comments",
+				IFNULL(pr_issues.count, 0) + IFNULL(reviews.count, 0) AS "pull request comments",
+				IFNULL(commits.count, 0) AS "commit comments"
+			FROM
+				(''' + self.allDaysToUpdateUnion(timeRange) + ''') AS alldays
+				LEFT JOIN (''' + self.subquery(timeRange, "issue_comments", "JOIN issues ON issues.id = issue_comments.issue_id", "AND issues.pull_request_id IS NULL") + ''') AS i_issues ON alldays.date = i_issues.date
+				LEFT JOIN (''' + self.subquery(timeRange, "issue_comments", "JOIN issues ON issues.id = issue_comments.issue_id", "AND issues.pull_request_id IS NOT NULL") + ''') AS pr_issues ON alldays.date = pr_issues.date
+				LEFT JOIN (''' + self.subquery(timeRange, "commit_comments") + ''') AS commits ON alldays.date = commits.date
+				LEFT JOIN (''' + self.subquery(timeRange, "pull_request_review_comments") + ''') AS reviews ON alldays.date = reviews.date
+			ORDER BY date DESC'''

updater/reports/ReportDailySQL.py

@@ -0,0 +1,17 @@
+from .ReportDaily import *
+
+# A daily report that also updates days in the past, given by timeRangeToUpdate
+class ReportDailySQL(ReportDaily):
+
+	def allDaysToUpdateUnion(self, timeRange):
+		# This method creates a table column of every single date in timeRange.
+		# It enables querying all results for timeRange in one single SQL query.
+		# Furthermore the result will include 0 for days without results, which
+		# makes the graphs look nicer especially on newly provisioned systems.
+		younger = max(timeRange)
+		older = min(timeRange)
+		snipped = "SELECT '" + str(younger) + "' AS date"
+		while (younger > older):
+			younger -= datetime.timedelta(1)
+			snipped += " UNION SELECT '" + str(younger) + "'"
+		return snipped

updater/reports/ReportIssues.py

@@ -0,0 +1,45 @@
+from .ReportDailySQL import *
+
+# Lists how many issues got created, closed, and were updated per day
+class ReportIssues(ReportDailySQL):
+	def name(self):
+		return "issues"
+
+	def updateDailyData(self):
+		self.header, newData = self.parseData(self.executeQuery(self.query(self.timeRangeToUpdate())))
+		self.data.extend(newData)
+		self.truncateData(self.timeRangeTotal())
+		self.sortDataByDate()
+
+	# Collects the issues closed in time range
+	def subquery(self, timeRange, type, extra=None):
+		return '''
+			SELECT
+				DATE_FORMAT(issues.''' + type + ''', "%Y-%m-%d") AS date,
+				COUNT(*) AS count
+			FROM
+				issues
+				JOIN users ON users.id = issues.user_id
+			WHERE
+				issues.pull_request_id IS NULL AND
+				CAST(issues.''' + type + ''' AS DATE) BETWEEN
+					"''' + str(timeRange[0]) + '''" AND "''' + str(timeRange[1]) + '''"
+					''' + self.andExcludedUsers("users.login") + (("\n\t\t\t\t\t" + extra) if extra else '') + '''
+			GROUP BY
+				date_format(issues.''' + type + ''', "%Y-%m-%d")'''
+
+	# Collects the number of issues created, closed, commented, and number of comments
+	def query(self, timeRange):
+		# `alldays` is used as a basis for LEFT JOIN, to prevent issues when querying multiple days, e.g. for initial run of the new report
+		return '''
+			SELECT
+			    alldays.date AS date,
+				IFNULL(created.count, 0) AS "issues created",
+				IFNULL(updated.count, 0) AS "issues updated",
+				IFNULL(closed.count, 0) AS "issues closed"
+			FROM
+				(''' + self.allDaysToUpdateUnion(timeRange) + ''') AS alldays
+				LEFT JOIN (''' + self.subquery(timeRange, "created_at") + ''') AS created ON alldays.date = created.date
+				LEFT JOIN (''' + self.subquery(timeRange, "updated_at", "AND issues.created_at != issues.updated_at AND issues.closed_at != issues.updated_at") + ''') AS updated ON alldays.date = updated.date
+				LEFT JOIN (''' + self.subquery(timeRange, "closed_at") + ''') AS closed ON alldays.date = closed.date
+			ORDER BY date DESC'''

updater/update-stats.py

@@ -36,6 +36,8 @@
 from reports.ReportTeamsTotal import *
 from reports.ReportTokenlessAuth import *
 from reports.ReportUsers import *
+from reports.ReportIssues import *
+from reports.ReportComments import *
 
 def writeMeta(dataDirectory):
 	outputFilePath = os.path.join(dataDirectory, "meta.tsv")
@@ -104,6 +106,8 @@ def main():
 	ReportTeamsTotal(configuration, dataDirectory, metaStats).update()
 	ReportTokenlessAuth(configuration, dataDirectory, metaStats).update()
 	ReportUsers(configuration, dataDirectory, metaStats).update()
+	ReportIssues(configuration, dataDirectory, metaStats).update()
+	ReportComments(configuration, dataDirectory, metaStats).update()
 
 	# Write meta infos
 	writeMeta(dataDirectory)