Skip to content

Commit 009c0f1

Browse files
brendanx67claude
brendanx67
and
claude
committed
Added Retrain All feature and improved menu styling
* Added RetrainAllAction with Reset/Incremental modes for bulk training data management * Reset mode rebuilds training data from scratch using recent clean runs * Incremental mode uses rolling window to replace oldest runs with newer ones * Added Min/Max runs parameters (defaults 5/20) with 1.5x lookback period * Redesigned menu with active tab highlighting (rounded corners) and gold hover * All pages now set activeTab attribute for consistent menu state Co-Authored-By: Claude <noreply@anthropic.com>
1 parent e2b6798 commit 009c0f1

File tree

11 files changed

+428
-44
lines changed

11 files changed

+428
-44
lines changed

‎testresults/src/org/labkey/testresults/TestResultsController.java‎

Lines changed: 257 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,8 @@
111111
import java.util.Date;
112112
import java.util.Enumeration;
113113
import java.util.HashMap;
114+
import java.util.HashSet;
115+
import java.util.Set;
114116
import java.util.List;
115117
import java.util.Locale;
116118
import java.util.Map;
@@ -1037,6 +1039,261 @@ public Object execute(Object o, BindException errors)
10371039
}
10381040
}
10391041

1042+
@RequiresSiteAdmin
1043+
public static class RetrainAllAction extends MutatingApiAction<Object>
1044+
{
1045+
@Override
1046+
public Object execute(Object o, BindException errors)
1047+
{
1048+
var req = getViewContext().getRequest();
1049+
String mode = req.getParameter("mode");
1050+
if (mode == null || mode.isEmpty())
1051+
mode = "reset";
1052+
boolean incremental = mode.equalsIgnoreCase("incremental");
1053+
1054+
int maxRuns = 20;
1055+
String maxRunsParam = req.getParameter("maxRuns");
1056+
if (maxRunsParam == null || maxRunsParam.isEmpty())
1057+
maxRunsParam = req.getParameter("targetRuns"); // backwards compatibility
1058+
if (maxRunsParam != null && !maxRunsParam.isEmpty())
1059+
{
1060+
try { maxRuns = Integer.parseInt(maxRunsParam); }
1061+
catch (NumberFormatException ignored) { }
1062+
}
1063+
if (maxRuns < 1) maxRuns = 1;
1064+
if (maxRuns > 100) maxRuns = 100;
1065+
1066+
int minRuns = 5;
1067+
String minRunsParam = req.getParameter("minRuns");
1068+
if (minRunsParam != null && !minRunsParam.isEmpty())
1069+
{
1070+
try { minRuns = Integer.parseInt(minRunsParam); }
1071+
catch (NumberFormatException ignored) { }
1072+
}
1073+
if (minRuns < 1) minRuns = 1;
1074+
if (minRuns > maxRuns) minRuns = maxRuns;
1075+
1076+
Container c = getContainer();
1077+
String containerPath = c.getPath();
1078+
int expectedDuration = containerPath.toLowerCase().contains("perf") ? 720 : 540;
1079+
1080+
// Only look back 1.5x maxRuns days to avoid ancient data
1081+
int lookbackDays = (int) Math.ceil(maxRuns * 1.5);
1082+
java.sql.Timestamp cutoffDate = new java.sql.Timestamp(
1083+
System.currentTimeMillis() - (long) lookbackDays * 24 * 60 * 60 * 1000);
1084+
1085+
DbScope scope = TestResultsSchema.getSchema().getScope();
1086+
try (DbScope.Transaction transaction = scope.ensureTransaction())
1087+
{
1088+
// Get unique user IDs from recent testruns for this container
1089+
SQLFragment allUsersSql = new SQLFragment();
1090+
allUsersSql.append(
1091+
"SELECT DISTINCT userid FROM " + TestResultsSchema.getTableInfoTestRuns() +
1092+
" WHERE container = ? AND posttime >= ?");
1093+
allUsersSql.add(c.getEntityId());
1094+
allUsersSql.add(cutoffDate);
1095+
List<Integer> allUserIds = new ArrayList<>();
1096+
new SqlSelector(scope, allUsersSql).forEach(rs ->
1097+
allUserIds.add(rs.getInt("userid")));
1098+
1099+
// Get current trainrun counts per user
1100+
Map<Integer, Integer> userTrainCounts = new HashMap<>();
1101+
for (int userId : allUserIds)
1102+
userTrainCounts.put(userId, 0);
1103+
1104+
if (incremental)
1105+
{
1106+
// In incremental mode, get existing counts
1107+
SQLFragment countsSql = new SQLFragment();
1108+
countsSql.append(
1109+
"SELECT r.userid, COUNT(tr.runid) as traincount" +
1110+
" FROM " + TestResultsSchema.getTableInfoTrain() + " tr" +
1111+
" JOIN " + TestResultsSchema.getTableInfoTestRuns() + " r ON tr.runid = r.id" +
1112+
" WHERE r.container = ?" +
1113+
" GROUP BY r.userid");
1114+
countsSql.add(c.getEntityId());
1115+
new SqlSelector(scope, countsSql).forEach(rs ->
1116+
userTrainCounts.put(rs.getInt("userid"), rs.getInt("traincount")));
1117+
}
1118+
else
1119+
{
1120+
// Reset mode: delete all trainruns for this container
1121+
SQLFragment deleteTrainSql = new SQLFragment();
1122+
deleteTrainSql.append(
1123+
"DELETE FROM " + TestResultsSchema.getTableInfoTrain() +
1124+
" WHERE runid IN (SELECT id FROM " + TestResultsSchema.getTableInfoTestRuns() +
1125+
" WHERE container = ?)");
1126+
deleteTrainSql.add(c.getEntityId());
1127+
new SqlExecutor(scope).execute(deleteTrainSql);
1128+
1129+
// Delete all userdata for this container
1130+
SQLFragment deleteUserDataSql = new SQLFragment();
1131+
deleteUserDataSql.append(
1132+
"DELETE FROM " + TestResultsSchema.getTableInfoUserData() +
1133+
" WHERE container = ?");
1134+
deleteUserDataSql.add(c.getEntityId());
1135+
new SqlExecutor(scope).execute(deleteUserDataSql);
1136+
}
1137+
1138+
int usersRetrained = 0;
1139+
int totalTrainRuns = 0;
1140+
final int minRunsRequired = minRuns;
1141+
1142+
for (int userId : allUserIds)
1143+
{
1144+
// Get existing trainrun IDs for this user
1145+
SQLFragment existingRunsSql = new SQLFragment();
1146+
existingRunsSql.append(
1147+
"SELECT tr.runid FROM " + TestResultsSchema.getTableInfoTrain() + " tr" +
1148+
" JOIN " + TestResultsSchema.getTableInfoTestRuns() + " r ON tr.runid = r.id" +
1149+
" WHERE r.userid = ? AND r.container = ?");
1150+
existingRunsSql.add(userId);
1151+
existingRunsSql.add(c.getEntityId());
1152+
Set<Integer> existingRunIds = new HashSet<>();
1153+
new SqlSelector(scope, existingRunsSql).forEach(rs ->
1154+
existingRunIds.add(rs.getInt("runid")));
1155+
1156+
// Find all clean runs within lookback period (for both modes)
1157+
SQLFragment cleanRunsSql = new SQLFragment();
1158+
cleanRunsSql.append(
1159+
"SELECT tr.id FROM " + TestResultsSchema.getTableInfoTestRuns() + " tr" +
1160+
" WHERE tr.userid = ? AND tr.container = ?" +
1161+
" AND tr.posttime >= ?" +
1162+
" AND tr.failedtests = 0 AND tr.leakedtests = 0" +
1163+
" AND tr.passedtests > 0 AND tr.flagged = false" +
1164+
" AND tr.duration >= ?" +
1165+
" AND NOT EXISTS (SELECT 1 FROM " + TestResultsSchema.getTableInfoHangs() +
1166+
" h WHERE h.testrunid = tr.id)" +
1167+
" ORDER BY tr.posttime DESC");
1168+
cleanRunsSql.add(userId);
1169+
cleanRunsSql.add(c.getEntityId());
1170+
cleanRunsSql.add(cutoffDate);
1171+
cleanRunsSql.add(expectedDuration);
1172+
1173+
List<Integer> recentCleanRunIds = new ArrayList<>();
1174+
new SqlSelector(scope, cleanRunsSql).forEach(rs ->
1175+
recentCleanRunIds.add(rs.getInt("id")));
1176+
1177+
// Determine final set of trainrun IDs
1178+
final List<Integer> finalRunIds = new ArrayList<>();
1179+
final int maxRunsLimit = maxRuns;
1180+
if (incremental && !existingRunIds.isEmpty())
1181+
{
1182+
// Incremental: combine existing + new recent clean runs, keep most recent maxRuns
1183+
// Get all existing trainruns with posttimes for sorting
1184+
SQLFragment allCandidatesSql = new SQLFragment();
1185+
allCandidatesSql.append(
1186+
"SELECT id, posttime FROM " + TestResultsSchema.getTableInfoTestRuns() +
1187+
" WHERE userid = ? AND container = ?" +
1188+
" AND (id = ANY(?) OR id = ANY(?))" +
1189+
" ORDER BY posttime DESC");
1190+
allCandidatesSql.add(userId);
1191+
allCandidatesSql.add(c.getEntityId());
1192+
allCandidatesSql.add(existingRunIds.toArray(new Integer[0]));
1193+
allCandidatesSql.add(recentCleanRunIds.toArray(new Integer[0]));
1194+
1195+
new SqlSelector(scope, allCandidatesSql).forEach(rs -> {
1196+
if (finalRunIds.size() < maxRunsLimit)
1197+
finalRunIds.add(rs.getInt("id"));
1198+
});
1199+
}
1200+
else
1201+
{
1202+
// Reset mode: just use recent clean runs up to maxRuns
1203+
if (recentCleanRunIds.size() > maxRuns)
1204+
finalRunIds.addAll(recentCleanRunIds.subList(0, maxRuns));
1205+
else
1206+
finalRunIds.addAll(recentCleanRunIds);
1207+
}
1208+
1209+
// Skip if total runs is below minimum threshold
1210+
if (finalRunIds.size() < minRunsRequired)
1211+
continue;
1212+
1213+
// Determine runs to add and remove
1214+
Set<Integer> finalRunSet = new HashSet<>(finalRunIds);
1215+
List<Integer> runsToAdd = new ArrayList<>();
1216+
List<Integer> runsToRemove = new ArrayList<>();
1217+
1218+
for (int runId : finalRunIds)
1219+
{
1220+
if (!existingRunIds.contains(runId))
1221+
runsToAdd.add(runId);
1222+
}
1223+
for (int runId : existingRunIds)
1224+
{
1225+
if (!finalRunSet.contains(runId))
1226+
runsToRemove.add(runId);
1227+
}
1228+
1229+
// Remove old trainruns
1230+
for (int runId : runsToRemove)
1231+
{
1232+
SQLFragment deleteTrainSql = new SQLFragment();
1233+
deleteTrainSql.append(
1234+
"DELETE FROM " + TestResultsSchema.getTableInfoTrain() + " WHERE runid = ?");
1235+
deleteTrainSql.add(runId);
1236+
new SqlExecutor(scope).execute(deleteTrainSql);
1237+
}
1238+
1239+
// Insert new trainruns
1240+
for (int runId : runsToAdd)
1241+
{
1242+
SQLFragment insertTrainSql = new SQLFragment();
1243+
insertTrainSql.append(
1244+
"INSERT INTO " + TestResultsSchema.getTableInfoTrain() + " (runid) VALUES (?)");
1245+
insertTrainSql.add(runId);
1246+
new SqlExecutor(scope).execute(insertTrainSql);
1247+
}
1248+
1249+
// Set active=true only if user has at least maxRuns
1250+
boolean isActive = finalRunIds.size() >= maxRuns;
1251+
1252+
// Calculate stats and upsert into userdata
1253+
SQLFragment upsertStatsSql = new SQLFragment();
1254+
upsertStatsSql.append(
1255+
"INSERT INTO " + TestResultsSchema.getTableInfoUserData() +
1256+
" (userid, container, meantestsrun, meanmemory, stddevtestsrun, stddevmemory, active)" +
1257+
" SELECT ?, ?, avg(passedtests), avg(averagemem)," +
1258+
" stddev_pop(passedtests), stddev_pop(averagemem), ?" +
1259+
" FROM " + TestResultsSchema.getTableInfoTestRuns() +
1260+
" WHERE id = ANY(?)" +
1261+
" ON CONFLICT(userid, container) DO UPDATE SET" +
1262+
" meantestsrun = excluded.meantestsrun," +
1263+
" meanmemory = excluded.meanmemory," +
1264+
" stddevtestsrun = excluded.stddevtestsrun," +
1265+
" stddevmemory = excluded.stddevmemory," +
1266+
" active = excluded.active");
1267+
upsertStatsSql.add(userId);
1268+
upsertStatsSql.add(c.getEntityId());
1269+
upsertStatsSql.add(isActive);
1270+
upsertStatsSql.add(finalRunIds.toArray(new Integer[0]));
1271+
new SqlExecutor(scope).execute(upsertStatsSql);
1272+
1273+
usersRetrained++;
1274+
totalTrainRuns += runsToAdd.size();
1275+
}
1276+
1277+
transaction.commit();
1278+
1279+
ApiSimpleResponse response = new ApiSimpleResponse();
1280+
response.put("success", true);
1281+
response.put("usersRetrained", usersRetrained);
1282+
response.put("totalTrainRuns", totalTrainRuns);
1283+
response.put("mode", mode);
1284+
return response;
1285+
}
1286+
catch (Exception e)
1287+
{
1288+
_log.error("Error in RetrainAllAction", e);
1289+
ApiSimpleResponse response = new ApiSimpleResponse();
1290+
response.put("success", false);
1291+
response.put("error", e.getMessage());
1292+
return response;
1293+
}
1294+
}
1295+
}
1296+
10401297
/**
10411298
* action for posting test output as an xml file
10421299
*/

‎testresults/src/org/labkey/testresults/view/errorFiles.jsp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
Container c = getContainer();
2525
%>
2626

27+
<% request.setAttribute("activeTab", "errors"); %>
2728
<%@include file="menu.jsp" %>
2829

2930
<p>All the files listed below at one point or another failed to post. When a run is successfully posted through this page it gets removed from the list.</p>

‎testresults/src/org/labkey/testresults/view/failureDetail.jsp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,7 @@
188188
}
189189
%>
190190

191+
<% request.setAttribute("activeTab", ""); %>
191192
<%@include file="menu.jsp" %>
192193

193194
<style>

‎testresults/src/org/labkey/testresults/view/flagged.jsp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
TestsDataBean data = (TestsDataBean)me.getModelBean();
2626
%>
2727

28+
<% request.setAttribute("activeTab", "flags"); %>
2829
<%@include file="menu.jsp" %>
2930

3031
<p>Runs which are flagged will not show up in the Overview breakdown, Long Term, and Failure pages. This includes graphs, charts, and any other sort of data visualization.</p>

‎testresults/src/org/labkey/testresults/view/longTerm.jsp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@
3131
}
3232
%>
3333

34+
<% request.setAttribute("activeTab", "longterm"); %>
3435
<%@include file="menu.jsp" %>
35-
<br />
3636
<form action="<%=h(new ActionURL(TestResultsController.LongTermAction.class, c))%>">
3737
View Type: <select id="view-type-combobox" name="viewType">
3838
<option disabled selected> -- select an option -- </option>

‎testresults/src/org/labkey/testresults/view/menu.jsp‎

Lines changed: 65 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -7,19 +7,73 @@
77
<%
88
final String menuContextPath = AppProps.getInstance().getContextPath();
99
Container menuContainer = getViewContext().getContainer();
10+
// activeTab is set by parent JSP via request attribute before including menu.jsp
11+
String activeTab = (String) request.getAttribute("activeTab");
12+
if (activeTab == null) activeTab = "";
1013
%>
1114

12-
<div id="menu">
15+
<style>
16+
#menu.testresults-nav {
17+
background: #4b2e83;
18+
padding: 0 !important;
19+
padding-top: 6px !important;
20+
margin: 0 0 12px 0 !important;
21+
height: 40px !important;
22+
overflow: visible;
23+
}
24+
#menu.testresults-nav ul {
25+
list-style: none !important;
26+
margin: 0 !important;
27+
padding: 0 0 0 10px !important;
28+
display: inline;
29+
}
30+
#menu.testresults-nav li {
31+
display: inline;
32+
margin: 0;
33+
padding: 0;
34+
}
35+
#menu.testresults-nav li:hover {
36+
background-color: transparent !important;
37+
}
38+
.testresults-nav .nav-tab {
39+
display: inline-block;
40+
padding: 4px 10px;
41+
color: #fff;
42+
text-decoration: none;
43+
border-radius: 4px;
44+
transition: background 0.2s;
45+
}
46+
.testresults-nav .nav-tab:hover {
47+
background: #B8A506;
48+
}
49+
.testresults-nav .nav-tab.active {
50+
background: rgba(255, 255, 255, 0.25);
51+
font-weight: bold;
52+
}
53+
#menu.testresults-nav #stats {
54+
float: right;
55+
margin-right: 40px;
56+
margin-top: 6px;
57+
color: #fff;
58+
font-weight: 600;
59+
}
60+
#menu.testresults-nav #uw {
61+
float: right;
62+
margin-top: 2px;
63+
}
64+
</style>
65+
66+
<div id="menu" class="testresults-nav">
67+
<img src="<%=getWebappURL("TestResults/img/uw.png")%>" id="uw" alt="UW">
68+
<span id="stats"></span>
1369
<ul>
14-
<li><a href="<%=h(new ActionURL(TestResultsController.BeginAction.class, menuContainer))%>" style="color:#fff;">-Overview</a></li>
15-
<li><a href="<%=h(new ActionURL(TestResultsController.ShowUserAction.class, menuContainer))%>" style="color:#fff;">-User</a></li>
16-
<li><a href="<%=h(new ActionURL(TestResultsController.ShowRunAction.class, menuContainer))%>" style="color:#fff;">-Run</a></li>
17-
<li><a href="<%=h(new ActionURL(TestResultsController.LongTermAction.class, menuContainer))%>" style="color:#fff;">-Long Term</a></li>
18-
<li><a href="<%=h(new ActionURL(TestResultsController.ShowFlaggedAction.class, menuContainer))%>" style="color:#fff;">-Flags</a></li>
19-
<li><a href="<%=h(new ActionURL(TestResultsController.TrainingDataViewAction.class, menuContainer))%>" style="color:#fff;">-Training Data</a></li>
20-
<li><a href="<%=h(new ActionURL(TestResultsController.ErrorFilesAction.class, menuContainer))%>" style="color:#fff;">-Posting Errors</a></li>
21-
<li><a href="/home/issues/project-begin.view" target="_blank" title="Report bugs/Request features. Use 'TestResults' as area when creating new issue" style="color:#fff;">-Issues</a></li>
22-
<img src="<%=getWebappURL("TestResults/img/uw.png")%>" id="uw">
23-
<span id="stats"></span>
70+
<li><a href="<%=h(new ActionURL(TestResultsController.BeginAction.class, menuContainer))%>" class="<%=h("nav-tab" + (activeTab.equals("overview") ? " active" : ""))%>">Overview</a></li>
71+
<li><a href="<%=h(new ActionURL(TestResultsController.ShowUserAction.class, menuContainer))%>" class="<%=h("nav-tab" + (activeTab.equals("user") ? " active" : ""))%>">User</a></li>
72+
<li><a href="<%=h(new ActionURL(TestResultsController.ShowRunAction.class, menuContainer))%>" class="<%=h("nav-tab" + (activeTab.equals("run") ? " active" : ""))%>">Run</a></li>
73+
<li><a href="<%=h(new ActionURL(TestResultsController.LongTermAction.class, menuContainer))%>" class="<%=h("nav-tab" + (activeTab.equals("longterm") ? " active" : ""))%>">Long Term</a></li>
74+
<li><a href="<%=h(new ActionURL(TestResultsController.ShowFlaggedAction.class, menuContainer))%>" class="<%=h("nav-tab" + (activeTab.equals("flags") ? " active" : ""))%>">Flags</a></li>
75+
<li><a href="<%=h(new ActionURL(TestResultsController.TrainingDataViewAction.class, menuContainer))%>" class="<%=h("nav-tab" + (activeTab.equals("trainingdata") ? " active" : ""))%>">Training Data</a></li>
76+
<li><a href="<%=h(new ActionURL(TestResultsController.ErrorFilesAction.class, menuContainer))%>" class="<%=h("nav-tab" + (activeTab.equals("errors") ? " active" : ""))%>">Posting Errors</a></li>
77+
<li><a href="/home/issues/project-begin.view" target="_blank" title="Report bugs/Request features. Use 'TestResults' as area when creating new issue" class="nav-tab">Issues</a></li>
2478
</ul>
2579
</div>

‎testresults/src/org/labkey/testresults/view/multiFailureDetail.jsp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,8 @@
6363
}
6464
%>
6565

66+
<% request.setAttribute("activeTab", ""); %>
6667
<%@include file="menu.jsp" %>
67-
<br />
6868
<form action="<%=h(new ActionURL(TestResultsController.ShowFailures.class, c))%>">
6969
View Type: <select name="viewType">
7070
<option disabled selected> -- select an option -- </option>

0 commit comments

Comments
 (0)