Skip to content

Commit e555788

Browse files
ctruedenctrueden
committed
Convert html-report.sh logic to Python
With this change, the HTML generation runs >4x faster than in shell, since subprocesses are not spawned and external I/O is only done once. For now, newest-releases.sh (invoked once), and version-timestamps.sh (invoked once per G:A) still exist. Future work could potentially integrate these scripts into Python as well for further speedups.
1 parent 10e7378 commit e555788

File tree

6 files changed

+236
-274
lines changed

6 files changed

+236
-274
lines changed

‎.github/build.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/sh
22

33
# Regenerate the HTML content.
4-
./html-report.sh > index-new.html &&
4+
python html-report.py > index-new.html &&
55

66
# Push it to the gh-pages branch.
77
git clone --depth=1 --branch=gh-pages git@github.com:scijava/status.scijava.org site &&

‎.github/workflows/build.yml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ jobs:
1717
- uses: webfactory/ssh-agent@v0.5.3
1818
with:
1919
ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
20-
20+
- name: Set up Python
21+
uses: actions/setup-python@v1
22+
with:
23+
python-version: 3.8
2124
- name: Execute the build
2225
run: .github/build.sh

‎html-report.py‎

Lines changed: 231 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,231 @@
1+
#!/usr/bin/env python
2+
#
3+
# This is free and unencumbered software released into the public domain.
4+
# See the UNLICENSE file for details.
5+
#
6+
# ------------------------------------------------------------------------
7+
# html-report.py
8+
# ------------------------------------------------------------------------
9+
# Generates an HTML report of the release status of all
10+
# components in the SciJava BOM (org.scijava:pom-scijava).
11+
12+
import logging, re, subprocess
13+
14+
# -- Constants --
15+
16+
checkMark = "✔"
17+
xMark = "✕"
18+
repoBase = "https://maven.scijava.org"
19+
20+
# -- Data --
21+
22+
def file2map(filepath, sep=' '):
23+
with open(filepath) as f:
24+
pairs = [line.strip().split(sep, 1) for line in f.readlines()]
25+
return {pair[0]: pair[1] for pair in pairs}
26+
27+
badge_overrides = file2map('ci-badges.txt')
28+
timestamps = file2map('timestamps.txt')
29+
project_urls = file2map('projects.txt')
30+
31+
group_orgs = {
32+
'graphics.scenery': 'scenerygraphics',
33+
'io.scif': 'scifio',
34+
'net.imagej': 'imagej',
35+
'net.imglib2': 'imglib',
36+
'org.openmicroscopy': 'ome',
37+
'org.scijava': 'scijava',
38+
'sc.fiji': 'fiji',
39+
'sc.iview': 'scenerygraphics',
40+
}
41+
42+
# -- Functions --
43+
44+
def newest_releases():
45+
"""
46+
Dumps the component list for the given BOM (pom-scijava by default).
47+
The format of each component is groupId:artifactId,bomVersion,newestVersion
48+
"""
49+
return [token.decode() for token in subprocess.check_output(['./newest-releases.sh']).split()]
50+
51+
def project_url(ga):
52+
"""
53+
Gets the URL of a project from its G:A.
54+
"""
55+
if ga in project_urls:
56+
return project_urls[ga]
57+
58+
g, a = ga.split(':', 1)
59+
60+
if g == "sc.fiji":
61+
if a.startswith('bigdataviewer'):
62+
return f'https://github.com/bigdataviewer/{a}'
63+
if a.endswith('_'):
64+
a = a[:-1]
65+
66+
if g in group_orgs:
67+
return f'https://github.com/{group_orgs[g]}/{a}'
68+
69+
return ''
70+
71+
def version_timestamps(g, a, v):
72+
"""
73+
Gets timestamps for the last time a component was released.
74+
Returns a (releaseTimestamp, lastDeployed) pair.
75+
"""
76+
gav, releaseTimestamp, lastDeployed = subprocess.check_output(['./version-timestamps.sh', f'{g}:{a}:{v}']).decode().strip('\n\r').split(' ')
77+
releaseTimestamp = int(releaseTimestamp) if releaseTimestamp else 0
78+
lastDeployed = int(lastDeployed) if lastDeployed else 0
79+
return releaseTimestamp, lastDeployed
80+
81+
def badge(url):
82+
slug = None
83+
if url.startswith('https://github.com/'):
84+
slug = url[len('https://github.com/'):]
85+
if slug in badge_overrides:
86+
return badge_overrides[slug]
87+
if slug:
88+
return f"<td class=\"badge\"><a href=\"https://github.com/{slug}/actions\"><img src=\"https://github.com/{slug}/actions/workflows/build-main.yml/badge.svg\"></a></td>"
89+
return "<td>-</td>"
90+
91+
def timestamp_override(g, a):
92+
"""
93+
Gets the timestamp when a project was last vetted.
94+
95+
Sometimes, we know a new snapshot has no release-worthy changes since
96+
the last release. In this scenario, we can record the last time we
97+
checked a given component in the timestamps.txt file.
98+
99+
This also gives us a hedge against problems with the Maven metadata,
100+
such as that particular G:A:V not be present in the remote repository
101+
for any reason (in which case, this function returns 0).
102+
"""
103+
return int(timestamps.get(f'{g}:{a}', 0))
104+
105+
def release_link(g, a, v):
106+
return f"<a href=\"{repoBase}/#nexus-search;gav~{g}~{a}~{v}~~\">{v}</a>"
107+
108+
# -- Main --
109+
110+
# Emit the HTML header matter.
111+
print('<html>')
112+
print('<head>')
113+
print('<title>SciJava software status</title>')
114+
print('<link type="text/css" rel="stylesheet" href="status.css">')
115+
print('<link rel="icon" type="image/png" href="favicon.png">')
116+
print('<script type="text/javascript" src="sorttable.js"></script>')
117+
print('<script type="text/javascript" src="sortable-badges.js"></script>')
118+
print('</head>')
119+
print('<body onload="makeBadgesSortable()">')
120+
print('<!-- Generated via https://codepo8.github.io/css-fork-on-github-ribbon/ -->')
121+
print('<span id="forkongithub"><a href="https://github.com/scijava/status.scijava.org">Fix me on GitHub</a></span>')
122+
print('<table class="sortable">')
123+
print('<tr>')
124+
print('<th>Artifact</th>')
125+
print('<th>Release</th>')
126+
print('<th>OK</th>')
127+
print('<th>Last vetted</th>')
128+
print('<th>Last updated</th>')
129+
print('<th>OK</th>')
130+
print('<th>Action</th>')
131+
print('<th>Build</th>')
132+
print('</tr>')
133+
134+
# List components of the BOM, and loop over them.
135+
logging.info("Generating list of components")
136+
for line in newest_releases():
137+
ga, bomVersion, newestRelease = line.split(',')
138+
g, a = ga.split(':')
139+
140+
logging.info(f"Processing {ga}")
141+
142+
# Get project URL
143+
url = project_url(ga)
144+
145+
# Compute badges
146+
ciBadge = badge(url)
147+
148+
# Check BOM version vs. newest release
149+
if bomVersion == newestRelease:
150+
bomStatus = "bom-ok"
151+
bomOK = checkMark
152+
else:
153+
bomStatus = "bom-behind"
154+
bomOK = xMark
155+
156+
# Discern timestamps for this component.
157+
#
158+
# Each component is "vetted" either by:
159+
# A) being released; or
160+
# B) adding an override to timestamps.txt.
161+
# Our goal here is to detect whether the component has changed since
162+
# the most recent release (not the release listed in the BOM).
163+
releaseTimestamp, lastUpdated = version_timestamps(g, a, newestRelease)
164+
timestampOverride = timestamp_override(g, a)
165+
lastVetted = max(releaseTimestamp, timestampOverride)
166+
167+
# Compute time difference; >24 hours means a new release is needed.
168+
if lastUpdated - lastVetted > 1000000:
169+
# A SNAPSHOT was deployed more recently than the newest release.
170+
releaseStatus = "release-needed"
171+
releaseOK = xMark
172+
else:
173+
# No SNAPSHOT has happened more than 24 hours after newest release.
174+
releaseStatus = "release-ok"
175+
releaseOK = checkMark
176+
177+
if lastUpdated < timestampOverride:
178+
# NB: Manually vetted more recently than last update; no bump needed.
179+
bomStatus = "bom-ok"
180+
bomOK = checkMark
181+
182+
# Compute action items.
183+
if url and releaseOK == xMark:
184+
action = "Cut"
185+
actionKey = 1
186+
elif bomOK == xMark:
187+
action = "Bump"
188+
actionKey = 2
189+
else:
190+
action = "None"
191+
actionKey = 3
192+
193+
# Emit the HTML table row.
194+
gc = re.sub('[^0-9A-Za-z]', '-', g)
195+
ac = re.sub('[^0-9A-Za-z]', '-', a)
196+
print(f"<tr class=\"g-{gc} a-{ac} {bomStatus} {releaseStatus}\">")
197+
if url:
198+
print(f"<td><a href=\"{url}\">{g} : {a}</td>")
199+
else:
200+
print(f"<td>{g} : {a}</td>")
201+
202+
if bomVersion == newestRelease:
203+
print(f"<td>{release_link(g, a, newestRelease)}</td>")
204+
else:
205+
print(f"<td>{release_link(g, a, bomVersion)} &rarr; {release_link(g, a, newestRelease)}</td>")
206+
207+
print(f"<td>{bomOK}</td>")
208+
if lastVetted == 0:
209+
# Unknown status!
210+
print("<td class=\"unknown\">???</td>")
211+
elif lastVetted == timestampOverride:
212+
# Last vetted manually via timestamps.txt.
213+
print(f"<td class=\"overridden\">{lastVetted}</td>")
214+
elif timestampOverride == 0:
215+
# Last vetted automatically via release artifact; no timestamp override.
216+
print(f"<td>{lastVetted}</td>")
217+
else:
218+
# Last vetted automatically via release artifact; old timestamp override.
219+
print(f"<td class=\"wasOverridden\">{lastVetted}</td>")
220+
print(f"<td>{lastUpdated}</td>")
221+
print(f"<td>{releaseOK}</td>")
222+
print(f"<td sorttable_customkey=\"{actionKey}\">{action}</td>")
223+
print(ciBadge)
224+
print('</tr>')
225+
226+
# Emit the HTML footer matter.
227+
print('</table>')
228+
with open('footer.html') as f:
229+
print(f.read().strip())
230+
print('</body>')
231+
print('</html>')

0 commit comments

Comments
 (0)