Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: reproducibility graph #138

Merged
merged 4 commits into from
Jun 27, 2024
Merged

feat: reproducibility graph #138

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1956e64
fix: small html changes
tdejager Jun 27, 2024
8b9449e
Merge branch 'main' into feat/html-graph
tdejager Jun 27, 2024
16e4ec8
feat: bring back the html graph
tdejager Jun 27, 2024
6432889
fix: change to more pythonic if-check
tdejager Jun 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions src/repror/cli/generate_html.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import os
import re
from datetime import datetime, timedelta
from collections import defaultdict
from typing import Optional

Expand All @@ -9,7 +10,7 @@
from pathlib import Path
from jinja2 import Environment, FileSystemLoader

from repror.internals.db import BuildState, get_rebuild_data
from repror.internals.db import BuildState, get_rebuild_data, get_total_successful_builds_and_rebuilds
from repror.internals.git import github_api
from repror.internals.print import print

Expand Down Expand Up @@ -94,9 +95,28 @@ def rerender_html(root_folder: Path, update_remote: bool = False):

builds = get_rebuild_data()

# Statistics for graph
counts_per_platform = {}
# Get the last 10 days
start = datetime.now() - timedelta(days=9)
end_of_day = datetime(start.year, start.month, start.day, 23, 59, 59)
timestamps = [end_of_day + timedelta(days=i) for i in range(0, 10)]
by_platform = defaultdict(list)

for build in builds:

# Do it in the loop so that we do this only once per platform
# and we dont have to hardcode the platforms
if build.platform_name not in counts_per_platform:
counts = [get_total_successful_builds_and_rebuilds(build.platform_name, before_time=time) for time in timestamps]
counts_per_platform[build.platform_name] = {
# Total successful builds
"builds": [count.builds for count in counts],
# Total reproducible builds
"rebuilds": [count.rebuilds for count in counts],
# Total builds = builds + failed builds
"total_builds": [count.total_builds for count in counts]
}

if build.state == BuildState.FAIL:
by_platform[build.platform_name].append(
StatisticData(
Expand Down Expand Up @@ -129,6 +149,8 @@ def rerender_html(root_folder: Path, update_remote: bool = False):

html_content = template.render(
by_platform=by_platform,
dates=[time.strftime("%Y-%m-%d") for time in timestamps],
counts_per_platform=counts_per_platform,
build_state_fa=build_state_fa,
reproducible=reproducible,
failure=failure,
Expand Down
4 changes: 1 addition & 3 deletions src/repror/cli/rebuild_recipe.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,7 @@ def rebuild_recipe(
)

if latest_build.state == BuildState.FAIL:
raise ValueError(
f"Build failed for recipe {recipe.name}. Cannot rebuild."
)
raise ValueError(f"Build failed for recipe {recipe.name}. Cannot rebuild.")

if latest_rebuild and not force:
print("Found latest rebuild. Skipping rebuilding it again")
Expand Down
114 changes: 55 additions & 59 deletions src/repror/cli/templates/index.html.jinja
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,16 @@
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/chart.umd.min.js"></script>
</head>
<script>
function tabsComponent() {
function platformTab() {
return {
activeTab: 1,
selectTab(tab) {
this.activeTab = tab;
},
};
}

// Calculate a human timestamp
function timeAgo(timestamp) {
const now = new Date();
const date = new Date(timestamp);
Expand Down Expand Up @@ -45,18 +49,10 @@
return `${years} year${years !== 1 ? 's' : ''} ago`;
}
}

document.addEventListener("DOMContentLoaded", function () {
const elements = document.querySelectorAll('[data-timestamp]');
elements.forEach(element => {
const timestamp = element.getAttribute('data-timestamp');
element.textContent = timeAgo(timestamp);
});
});
</script>

<body class="bg-gray-100">
<div class="container mx-auto p-4" x-data="tabsComponent()">
<div class="container mx-auto p-4" x-data="platformTab()">
<h1 class="text-4xl font-bold py-6 mx-auto">
How Reproducible is
<span class="font-mono">rattler-build</span>?
Expand All @@ -65,19 +61,20 @@
<h2 class="text-xl font-bold py-4">
Introduction
</h2>
<p>
<span>
This website contains some information about
<span class="font-mono">.conda</span> packages that have been using
<a href="https://github.com/prefix-dev/rattler-build"
class="text-blue-500 underline hover:text-blue-300">
<span class="font-mono">rattler-build</pre>
<span class="font-mono">rattler-build</span>
<i class="fa-brands fa-github"></i>
</a> to build their packages.
</p>

<p>
More information regarding what we mean by reproducibility can be found
at the github repository for this project <a class="text-blue-500 underline hover:text-blue-300"
href="https://github.com/prefix-dev/reproducible-builds">Click Here</a>
href="https://github.com/prefix-dev/reproducible-builds">Click Here&nbsp;<i class="fa-brands fa-github"></i></a>
</p>
</section>

Expand All @@ -97,7 +94,7 @@
{% set reproducibility_percentage = (reproducible_builds / total_builds) * 100 %}
<div class="bg-white p-4 rounded shadow border cursor-pointer"
:class="{ 'border-blue-600 border-4': activeTab === {{loop.index}}, 'border-gray-300 border-1': activeTab !== {{loop.index}} }"
@click="activeTab = {{loop.index}}">
@click="selectTab({{loop.index}})">
<h3 class="text-lg font-semibold">
<span class="{{ platform | platform_fa }}"></span>
{{ platform | capitalize }}
Expand All @@ -118,7 +115,9 @@
</div>

<div>
<canvas id="myChart"></canvas>
{% for platform in by_platform.keys() %}
<canvas id="chart-{{ platform }}" x-show="activeTab == {{ loop.index }}"></canvas>
{% endfor %}
</div>
<div class="inline-flex gap-4 border py-3 px-4 bg-gray-50 rounded-xl">
<div>Reproducible: <i class="{{ reproducible }}"></i></div>
Expand All @@ -143,7 +142,7 @@
</tr>
</thead>
<tbody>
{% for build in builds | sort(attribute='build_state') %}
{% for build in builds | sort(attribute='build_state, recipe_name') %}
<tr class="border-b border-dashed">
<td class="py-2 px-4">
<span class="{{ build_state_fa(build.build_state, build.rebuild_state) }}"></span>
Expand All @@ -156,8 +155,8 @@
{{ build.rebuild_state.value | capitalize if
build.rebuild_state else 'N/A' }}
</td>
<td class="py-2 px-4" data-timestamp="{{build.time}}"></td>
<td class="py-2 px-4">
<td class="py-2 px-4" x-data="{ humanTime: timeAgo('{{ build.time }}') }" x-html="humanTime"></td>
<td class="py-2 px-4"></td>
<pre>{{ build.reason if build.reason else "" }}</pre>
</td>
<td class="py-2 px-4 flex justify-center">
Expand All @@ -167,59 +166,19 @@
<i class="fa-brands fa-github text-xl"></i>
</a>
</td>
<!-- <td class="py-2 px-4 border-b">
<button class="bg-blue-500 text-white py-1 px-3 rounded hover:bg-blue-700" onclick="viewLogs('{{ build.reason }}')">View Logs</button>
</td> -->
</tr>
{% endfor %}
</tbody>
</table>
</div>
{% endfor %}
</div>

<!-- Modal for logs -->
<div id="logModal" class="fixed inset-0 flex items-center justify-center bg-gray-800 bg-opacity-75 hidden">
<div class="bg-white p-4 rounded shadow-lg max-w-xl w-full">
<h2 class="text-xl font-bold mb-2">Logs</h2>
<pre id="logContent" class="bg-gray-200 p-2 rounded h-64 overflow-y-scroll"></pre>
<button class="bg-red-500 text-white py-1 px-3 rounded hover:bg-red-700 mt-2" onclick="closeModal()">
Close
</button>
</div>
</div>
</body>

<script>

const ctx = document.getElementById('myChart');
const DATA_COUNT = 7;
const NUMBER_CFG = { count: DATA_COUNT, min: -100, max: 100 };

const data = {
labels: ['2021-01-01', '2021-01-02'],
datasets: [
{
label: 'Builds',
data: [10, 20],
fill: true,
},
{
label: 'Rebuilds',
data: [5, 10],
fill: true,
},
{
label: 'Total Builds',
data: [12, 24],
fill: false
},
]
};

const config = {
type: 'line',
data: data,
options: {
responsive: true,
plugins: {
Expand Down Expand Up @@ -280,7 +239,44 @@
}
};

new Chart(ctx, config);
{% for platform in by_platform.keys() %}
// Change the title of the chart
const data_{{platform}} = {
labels: {{ dates | tojson }},
datasets: [
{
label: 'Reproducible',
data: {{ counts_per_platform[platform].rebuilds | tojson }},
fill: true,
},
{
label: 'Builds',
data: {{ counts_per_platform[platform].builds | tojson }},
fill: true,
},
{
label: 'Total Recipes',
data: {{ counts_per_platform[platform].total_builds | tojson }},
fill: false
},
]
};
const config_{{ platform }} = {
...config,
options: {
...config.options,
plugins: {
...config.options.plugins,
title: {
...config.options.plugins.title,
text: 'Rebuild Statistics for {{ platform | capitalize }}',
},
},
},
data: data_{{platform}},
};
const chart_{{ platform }} = new Chart(document.getElementById("chart-{{ platform }}"), config_{{ platform }});
{% endfor %}
</script>

</html>
15 changes: 8 additions & 7 deletions src/repror/internals/db.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import os
from pathlib import Path
import tempfile
from typing import Generator, Optional
from typing import Generator, Literal as Lit, Optional
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why Literal as Lit?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because there is another Literal import beneath it.

from pydantic import BaseModel
from sqlalchemy import func, text
from typing import Sequence
Expand All @@ -23,7 +23,6 @@
select,
Session as SqlModelSession,
col,
literal,
)

from repror.internals.recipe import clone_remote_recipe
Expand Down Expand Up @@ -325,11 +324,12 @@ def get_total_unique_recipes(session: Optional[SqlModelSession] = None) -> int:
class SuccessfulBuildsAndRebuilds:
builds: int
rebuilds: int
total_builds: int


def get_total_successful_builds_and_rebuilds(
platform_name: Lit["linux", "darwin", "windows"] | str,
before_time: datetime,
after_time: Optional[datetime],
session: Optional[SqlModelSession] = None,
) -> SuccessfulBuildsAndRebuilds:
"""Query to get the total number of successful builds and rebuilds before the given timestamp."""
Expand All @@ -340,10 +340,8 @@ def get_total_successful_builds_and_rebuilds(
Build.id,
)
.where(
col(Build.platform_name) == platform_name,
col(Build.timestamp) <= before_time,
(col(Build.timestamp) > after_time)
if after_time is not None
else literal(True),
)
.group_by(Build.recipe_name)
)
Expand All @@ -361,9 +359,12 @@ def get_total_successful_builds_and_rebuilds(
Rebuild.state == BuildState.SUCCESS,
)

total_builds_query = select(func.count(col(Build.id))).join(subquery, (col(Build.id) == subquery.c.id))

# Execute the queries and count the results
successful_builds_count = session.exec(successful_builds_query).one()
successful_rebuilds_count = session.exec(successful_rebuilds_query).one()
total_builds: int = session.exec(total_builds_query).one()
return SuccessfulBuildsAndRebuilds(
successful_builds_count, successful_rebuilds_count
successful_builds_count, successful_rebuilds_count, total_builds=total_builds
)
Loading
Loading