Skip to content

Commit

Permalink
feat: reproducibility graph (#138)
Browse files Browse the repository at this point in the history 
Adds back html reproducibility graph
  • Loading branch information
@tdejager
tdejager authored Jun 27, 2024
1 parent b09fc08 commit 1aba54f
Show file tree
Hide file tree
Showing 5 changed files with 116 additions and 81 deletions.
26 changes: 24 additions & 2 deletions src/repror/cli/generate_html.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import os
import re
from datetime import datetime, timedelta
from collections import defaultdict
from typing import Optional

Expand All @@ -9,7 +10,7 @@
from pathlib import Path
from jinja2 import Environment, FileSystemLoader

from repror.internals.db import BuildState, get_rebuild_data
from repror.internals.db import BuildState, get_rebuild_data, get_total_successful_builds_and_rebuilds
from repror.internals.git import github_api
from repror.internals.print import print

Expand Down Expand Up @@ -94,9 +95,28 @@ def rerender_html(root_folder: Path, update_remote: bool = False):

builds = get_rebuild_data()

# Statistics for graph
counts_per_platform = {}
# Get the last 10 days
start = datetime.now() - timedelta(days=9)
end_of_day = datetime(start.year, start.month, start.day, 23, 59, 59)
timestamps = [end_of_day + timedelta(days=i) for i in range(0, 10)]
by_platform = defaultdict(list)

for build in builds:

# Do it in the loop so that we do this only once per platform
# and we dont have to hardcode the platforms
if build.platform_name not in counts_per_platform:
counts = [get_total_successful_builds_and_rebuilds(build.platform_name, before_time=time) for time in timestamps]
counts_per_platform[build.platform_name] = {
# Total successful builds
"builds": [count.builds for count in counts],
# Total reproducible builds
"rebuilds": [count.rebuilds for count in counts],
# Total builds = builds + failed builds
"total_builds": [count.total_builds for count in counts]
}

if build.state == BuildState.FAIL:
by_platform[build.platform_name].append(
StatisticData(
Expand Down Expand Up @@ -129,6 +149,8 @@ def rerender_html(root_folder: Path, update_remote: bool = False):

html_content = template.render(
by_platform=by_platform,
dates=[time.strftime("%Y-%m-%d") for time in timestamps],
counts_per_platform=counts_per_platform,
build_state_fa=build_state_fa,
reproducible=reproducible,
failure=failure,
Expand Down
4 changes: 1 addition & 3 deletions src/repror/cli/rebuild_recipe.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,7 @@ def rebuild_recipe(
)

if latest_build.state == BuildState.FAIL:
raise ValueError(
f"Build failed for recipe {recipe.name}. Cannot rebuild."
)
raise ValueError(f"Build failed for recipe {recipe.name}. Cannot rebuild.")

if latest_rebuild and not force:
print("Found latest rebuild. Skipping rebuilding it again")
Expand Down
114 changes: 55 additions & 59 deletions src/repror/cli/templates/index.html.jinja
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,16 @@
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/chart.umd.min.js"></script>
</head>
<script>
function tabsComponent() {
function platformTab() {
return {
activeTab: 1,
selectTab(tab) {
this.activeTab = tab;
},
};
}
// Calculate a human timestamp
function timeAgo(timestamp) {
const now = new Date();
const date = new Date(timestamp);
Expand Down Expand Up @@ -45,18 +49,10 @@
return `${years} year${years !== 1 ? 's' : ''} ago`;
}
}
document.addEventListener("DOMContentLoaded", function () {
const elements = document.querySelectorAll('[data-timestamp]');
elements.forEach(element => {
const timestamp = element.getAttribute('data-timestamp');
element.textContent = timeAgo(timestamp);
});
});
</script>

<body class="bg-gray-100">
<div class="container mx-auto p-4" x-data="tabsComponent()">
<div class="container mx-auto p-4" x-data="platformTab()">
<h1 class="text-4xl font-bold py-6 mx-auto">
How Reproducible is
<span class="font-mono">rattler-build</span>?
Expand All @@ -65,19 +61,20 @@
<h2 class="text-xl font-bold py-4">
Introduction
</h2>
<p>
<span>
This website contains some information about
<span class="font-mono">.conda</span> packages that have been using
<a href="https://github.com/prefix-dev/rattler-build"
class="text-blue-500 underline hover:text-blue-300">
<span class="font-mono">rattler-build</pre>
<span class="font-mono">rattler-build</span>
<i class="fa-brands fa-github"></i>
</a> to build their packages.
</p>

<p>
More information regarding what we mean by reproducibility can be found
at the github repository for this project <a class="text-blue-500 underline hover:text-blue-300"
href="https://github.com/prefix-dev/reproducible-builds">Click Here</a>
href="https://github.com/prefix-dev/reproducible-builds">Click Here&nbsp;<i class="fa-brands fa-github"></i></a>
</p>
</section>

Expand All @@ -97,7 +94,7 @@
{% set reproducibility_percentage = (reproducible_builds / total_builds) * 100 %}
<div class="bg-white p-4 rounded shadow border cursor-pointer"
:class="{ 'border-blue-600 border-4': activeTab === {{loop.index}}, 'border-gray-300 border-1': activeTab !== {{loop.index}} }"
@click="activeTab = {{loop.index}}">
@click="selectTab({{loop.index}})">
<h3 class="text-lg font-semibold">
<span class="{{ platform | platform_fa }}"></span>
{{ platform | capitalize }}
Expand All @@ -118,7 +115,9 @@
</div>

<div>
<canvas id="myChart"></canvas>
{% for platform in by_platform.keys() %}
<canvas id="chart-{{ platform }}" x-show="activeTab == {{ loop.index }}"></canvas>
{% endfor %}
</div>
<div class="inline-flex gap-4 border py-3 px-4 bg-gray-50 rounded-xl">
<div>Reproducible: <i class="{{ reproducible }}"></i></div>
Expand All @@ -143,7 +142,7 @@
</tr>
</thead>
<tbody>
{% for build in builds | sort(attribute='build_state') %}
{% for build in builds | sort(attribute='build_state, recipe_name') %}
<tr class="border-b border-dashed">
<td class="py-2 px-4">
<span class="{{ build_state_fa(build.build_state, build.rebuild_state) }}"></span>
Expand All @@ -156,8 +155,8 @@
{{ build.rebuild_state.value | capitalize if
build.rebuild_state else 'N/A' }}
</td>
<td class="py-2 px-4" data-timestamp="{{build.time}}"></td>
<td class="py-2 px-4">
<td class="py-2 px-4" x-data="{ humanTime: timeAgo('{{ build.time }}') }" x-html="humanTime"></td>
<td class="py-2 px-4"></td>
<pre>{{ build.reason if build.reason else "" }}</pre>
</td>
<td class="py-2 px-4 flex justify-center">
Expand All @@ -167,59 +166,19 @@
<i class="fa-brands fa-github text-xl"></i>
</a>
</td>
<!-- <td class="py-2 px-4 border-b">
<button class="bg-blue-500 text-white py-1 px-3 rounded hover:bg-blue-700" onclick="viewLogs('{{ build.reason }}')">View Logs</button>
</td> -->
</tr>
{% endfor %}
</tbody>
</table>
</div>
{% endfor %}
</div>

<!-- Modal for logs -->
<div id="logModal" class="fixed inset-0 flex items-center justify-center bg-gray-800 bg-opacity-75 hidden">
<div class="bg-white p-4 rounded shadow-lg max-w-xl w-full">
<h2 class="text-xl font-bold mb-2">Logs</h2>
<pre id="logContent" class="bg-gray-200 p-2 rounded h-64 overflow-y-scroll"></pre>
<button class="bg-red-500 text-white py-1 px-3 rounded hover:bg-red-700 mt-2" onclick="closeModal()">
Close
</button>
</div>
</div>
</body>

<script>
const ctx = document.getElementById('myChart');
const DATA_COUNT = 7;
const NUMBER_CFG = { count: DATA_COUNT, min: -100, max: 100 };
const data = {
labels: ['2021-01-01', '2021-01-02'],
datasets: [
{
label: 'Builds',
data: [10, 20],
fill: true,
},
{
label: 'Rebuilds',
data: [5, 10],
fill: true,
},
{
label: 'Total Builds',
data: [12, 24],
fill: false
},
]
};
const config = {
type: 'line',
data: data,
options: {
responsive: true,
plugins: {
Expand Down Expand Up @@ -280,7 +239,44 @@
}
};
new Chart(ctx, config);
{% for platform in by_platform.keys() %}
// Change the title of the chart
const data_{{platform}} = {
labels: {{ dates | tojson }},
datasets: [
{
label: 'Reproducible',
data: {{ counts_per_platform[platform].rebuilds | tojson }},
fill: true,
},
{
label: 'Builds',
data: {{ counts_per_platform[platform].builds | tojson }},
fill: true,
},
{
label: 'Total Recipes',
data: {{ counts_per_platform[platform].total_builds | tojson }},
fill: false
},
]
};
const config_{{ platform }} = {
...config,
options: {
...config.options,
plugins: {
...config.options.plugins,
title: {
...config.options.plugins.title,
text: 'Rebuild Statistics for {{ platform | capitalize }}',
},
},
},
data: data_{{platform}},
};
const chart_{{ platform }} = new Chart(document.getElementById("chart-{{ platform }}"), config_{{ platform }});
{% endfor %}
</script>

</html>
15 changes: 8 additions & 7 deletions src/repror/internals/db.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import os
from pathlib import Path
import tempfile
from typing import Generator, Optional
from typing import Generator, Literal as Lit, Optional
from pydantic import BaseModel
from sqlalchemy import func, text
from typing import Sequence
Expand All @@ -23,7 +23,6 @@
select,
Session as SqlModelSession,
col,
literal,
)

from repror.internals.recipe import clone_remote_recipe
Expand Down Expand Up @@ -325,11 +324,12 @@ def get_total_unique_recipes(session: Optional[SqlModelSession] = None) -> int:
class SuccessfulBuildsAndRebuilds:
builds: int
rebuilds: int
total_builds: int


def get_total_successful_builds_and_rebuilds(
platform_name: Lit["linux", "darwin", "windows"] | str,
before_time: datetime,
after_time: Optional[datetime],
session: Optional[SqlModelSession] = None,
) -> SuccessfulBuildsAndRebuilds:
"""Query to get the total number of successful builds and rebuilds before the given timestamp."""
Expand All @@ -340,10 +340,8 @@ def get_total_successful_builds_and_rebuilds(
Build.id,
)
.where(
col(Build.platform_name) == platform_name,
col(Build.timestamp) <= before_time,
(col(Build.timestamp) > after_time)
if after_time is not None
else literal(True),
)
.group_by(Build.recipe_name)
)
Expand All @@ -361,9 +359,12 @@ def get_total_successful_builds_and_rebuilds(
Rebuild.state == BuildState.SUCCESS,
)

total_builds_query = select(func.count(col(Build.id))).join(subquery, (col(Build.id) == subquery.c.id))

# Execute the queries and count the results
successful_builds_count = session.exec(successful_builds_query).one()
successful_rebuilds_count = session.exec(successful_rebuilds_query).one()
total_builds: int = session.exec(total_builds_query).one()
return SuccessfulBuildsAndRebuilds(
successful_builds_count, successful_rebuilds_count
successful_builds_count, successful_rebuilds_count, total_builds=total_builds
)
Loading

0 comments on commit 1aba54f

Please sign in to comment.