Skip to content

Commit e334791

Browse files
saibulususaibulusu
saibulusu
and
saibulusu
authored
Speccpu limitations on Windows & specifying benchmarks in SPECCpuExecutor (#619)
* speccpu limitations on windows, specifying benchmarks for SpeccpuExecutor * minor documentation fix --------- Co-authored-by: saibulusu <saibulusu@microsoft.com>
1 parent 47650ff commit e334791

8 files changed

Lines changed: 84 additions & 25 deletions

File tree

‎src/VirtualClient/VirtualClient.Actions.UnitTests/SPEC/SpecCpuExecutorTests.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,7 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithDifferentProfiles
164164
this.mockFixture.Parameters = new Dictionary<string, IConvertible>()
165165
{
166166
{ nameof(SpecCpuExecutor.SpecProfile), "fprate" },
167+
{ nameof(SpecCpuExecutor.Benchmarks), "fprate" },
167168
{ nameof(SpecCpuExecutor.PackageName), "speccpu" },
168169
{ nameof(SpecCpuExecutor.RunPeak), false },
169170
};
@@ -200,6 +201,7 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithDifferentProfiles
200201
this.mockFixture.Parameters = new Dictionary<string, IConvertible>()
201202
{
202203
{ nameof(SpecCpuExecutor.SpecProfile), "intspeed" },
204+
{ nameof(SpecCpuExecutor.Benchmarks), "intspeed" },
203205
{ nameof(SpecCpuExecutor.PackageName), "speccpu" },
204206
{ nameof(SpecCpuExecutor.RunPeak), true }
205207
};
@@ -238,6 +240,7 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithDifferentProfiles
238240
this.mockFixture.Parameters = new Dictionary<string, IConvertible>()
239241
{
240242
{ nameof(SpecCpuExecutor.SpecProfile), "intspeed" },
243+
{ nameof(SpecCpuExecutor.Benchmarks), "intspeed" },
241244
{ nameof(SpecCpuExecutor.PackageName), "speccpu" },
242245
{ nameof(SpecCpuExecutor.Iterations), 1 },
243246
{ nameof(SpecCpuExecutor.RunPeak), true }
@@ -280,6 +283,7 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithDifferentProfiles
280283
this.mockFixture.Parameters = new Dictionary<string, IConvertible>()
281284
{
282285
{ nameof(SpecCpuExecutor.SpecProfile), "fprate" },
286+
{ nameof(SpecCpuExecutor.Benchmarks), "fprate" },
283287
{ nameof(SpecCpuExecutor.PackageName), "speccpu" },
284288
{ nameof(SpecCpuExecutor.RunPeak), false },
285289
};
@@ -317,6 +321,7 @@ public async Task SpecCpuExecutorExecutesTheCorrectCommandsWithDifferentProfiles
317321
this.mockFixture.Parameters = new Dictionary<string, IConvertible>()
318322
{
319323
{ nameof(SpecCpuExecutor.SpecProfile), "intspeed" },
324+
{ nameof(SpecCpuExecutor.Benchmarks), "intspeed" },
320325
{ nameof(SpecCpuExecutor.PackageName), "speccpu" },
321326
{ nameof(SpecCpuExecutor.RunPeak), true }
322327
};
@@ -374,6 +379,7 @@ private void SetupLinux()
374379
this.mockFixture.Parameters = new Dictionary<string, IConvertible>()
375380
{
376381
{ nameof(SpecCpuExecutor.SpecProfile), "intrate" },
382+
{ nameof(SpecCpuExecutor.Benchmarks), "intrate" },
377383
{ nameof(SpecCpuExecutor.PackageName), "speccpu" },
378384
{ nameof(SpecCpuExecutor.RunPeak), true },
379385
{ nameof(SpecCpuExecutor.Threads), 8 },
@@ -405,6 +411,7 @@ private void SetupWindows()
405411
this.mockFixture.Parameters = new Dictionary<string, IConvertible>()
406412
{
407413
{ nameof(SpecCpuExecutor.SpecProfile), "intrate" },
414+
{ nameof(SpecCpuExecutor.Benchmarks), "intrate" },
408415
{ nameof(SpecCpuExecutor.PackageName), "speccpu" },
409416
{ nameof(SpecCpuExecutor.RunPeak), true },
410417
{ nameof(SpecCpuExecutor.Threads), 8 },

‎src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuExecutor.cs‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,18 @@ public string SpecProfile
6464
}
6565
}
6666

67+
/// <summary>
68+
/// List of benchmarks to run.
69+
/// </summary>
70+
public string Benchmarks
71+
{
72+
get
73+
{
74+
this.Parameters.TryGetValue(nameof(SpecCpuExecutor.Benchmarks), out IConvertible benchmarks);
75+
return benchmarks?.ToString();
76+
}
77+
}
78+
6779
/// <summary>
6880
/// The whether SPECcpu runs base tuning or base+peak tuning.
6981
/// </summary>
@@ -396,7 +408,7 @@ private string GetCommandLineArguments()
396408
// Iterations has to be either 2 or 3 for reportable runs. https://www.spec.org/cpu2017/Docs/config.html#reportable
397409
bool reportable = (this.Platform == PlatformID.Unix) && (this.Iterations == 2 || this.Iterations == 3);
398410
cmd = reportable ? $"{cmd} --reportable" : $"{cmd} --noreportable";
399-
cmd = $"{cmd} {this.SpecProfile}";
411+
cmd = $"{cmd} {this.Benchmarks}";
400412
return cmd;
401413
}
402414

‎src/VirtualClient/VirtualClient.Core.UnitTests/ProfileExpressionEvaluatorTests.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1440,6 +1440,22 @@ public async Task ProfileExpressionEvaluatorSupportsTernaryFunctionReferencesInP
14401440
Assert.AreEqual(true, parameters["IsTLSEnabled"]);
14411441
}
14421442

1443+
[Test]
1444+
public async Task ProfileExpressionEvaluatorSupportsTernaryFunctionReferencesInParameterSets_Scenario_7()
1445+
{
1446+
this.SetupDefaults(PlatformID.Win32NT);
1447+
1448+
Dictionary<string, IConvertible> parameters = new Dictionary<string, IConvertible>
1449+
{
1450+
{ "Benchmarks" , "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"intrate\" : \"505 525 541 548 557\")}" },
1451+
};
1452+
1453+
await ProfileExpressionEvaluator.Instance.EvaluateAsync(this.mockFixture.Dependencies, parameters);
1454+
1455+
Assert.AreEqual("505 525 541 548 557", parameters["Benchmarks"]);
1456+
}
1457+
1458+
14431459
[Test]
14441460

14451461
public async Task ProfileExpressionEvaluatorSupportsNestedTernaryFunctionReferencesInParameterSets_Scenario_1()

‎src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPRATE.json‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,12 @@
66
"RecommendedMinimumExecutionTime": "(4-cores)=02:00:00,(16-cores)=05:00:00,(64-cores)=10:00:00",
77
"SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64",
88
"SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows",
9-
"Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24"
9+
"Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24",
10+
"Limitations": "The following benchmarks don't work on Windows: 507, 511, 521, 526, 527, 538"
1011
},
1112
"Parameters": {
1213
"CompilerVersion": "",
1314
"Iterations": 2,
14-
"SpecProfile": "fprate",
1515
"RunPeak": false,
1616
"Threads": "{LogicalCoreCount}",
1717
"Copies": "{LogicalCoreCount}",
@@ -24,7 +24,8 @@
2424
"Parameters": {
2525
"Scenario": "ExecuteSPECBenchmark",
2626
"Iterations": "$.Parameters.Iterations",
27-
"SpecProfile": "$.Parameters.SpecProfile",
27+
"SpecProfile": "fprate",
28+
"Benchmarks": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"fprate\" : \"503 508 510 519 544 549 554\")}",
2829
"PackageName": "speccpu2017",
2930
"RunPeak": "$.Parameters.RunPeak",
3031
"Threads": "$.Parameters.Threads",
@@ -39,23 +40,25 @@
3940
"Type": "ChocolateyInstallation",
4041
"Parameters": {
4142
"Scenario": "InstallChocolatey",
42-
"PackageName": "chocolatey"
43+
"PackageName": "chocolatey",
44+
"SupportedPlatforms": "win-x64,win-arm64"
4345
}
4446
},
4547
{
4648
"Type": "ChocolateyPackageInstallation",
4749
"Parameters": {
4850
"Scenario": "InstallCompiler",
4951
"PackageName": "chocolatey",
50-
"Packages": "cygwin"
52+
"Packages": "cygwin",
53+
"SupportedPlatforms": "win-x64,win-arm64"
5154
}
5255
},
5356
{
5457
"Type": "CompilerInstallation",
5558
"Parameters": {
5659
"Scenario": "InstallCompiler",
5760
"CompilerVersion": "$.Parameters.CompilerVersion",
58-
"CygwinPackages": "gcc-g++,gcc-fortran,gcc,libiconv-devel"
61+
"CygwinPackages": "gcc-g++,gcc-fortran,gcc-core,libiconv-devel"
5962
}
6063
},
6164
{

‎src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-FPSPEED.json‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,12 @@
66
"RecommendedMinimumExecutionTime": "(4-cores)=02:00:00,(16-cores)=06:00:00,(64-cores)=12:00:00",
77
"SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64",
88
"SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows",
9-
"Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24"
9+
"Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24",
10+
"Limitations": "The following benchmarks don't work on Windows: 607, 621, 627, 628, 638, 649"
1011
},
1112
"Parameters": {
1213
"CompilerVersion": "",
1314
"Iterations": 2,
14-
"SpecProfile": "fpspeed",
1515
"RunPeak": false,
1616
"Threads": "{LogicalCoreCount}",
1717
"Copies": "{LogicalCoreCount}",
@@ -24,7 +24,8 @@
2424
"Parameters": {
2525
"Scenario": "ExecuteSPECBenchmark",
2626
"Iterations": "$.Parameters.Iterations",
27-
"SpecProfile": "$.Parameters.SpecProfile",
27+
"SpecProfile": "fpspeed",
28+
"Benchmarks": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"fpspeed\" : \"603 619 644 654\")}",
2829
"PackageName": "speccpu2017",
2930
"RunPeak": "$.Parameters.RunPeak",
3031
"Threads": "$.Parameters.Threads",
@@ -39,23 +40,25 @@
3940
"Type": "ChocolateyInstallation",
4041
"Parameters": {
4142
"Scenario": "InstallChocolatey",
42-
"PackageName": "chocolatey"
43+
"PackageName": "chocolatey",
44+
"SupportedPlatforms": "win-x64,win-arm64"
4345
}
4446
},
4547
{
4648
"Type": "ChocolateyPackageInstallation",
4749
"Parameters": {
4850
"Scenario": "InstallCompiler",
4951
"PackageName": "chocolatey",
50-
"Packages": "cygwin"
52+
"Packages": "cygwin",
53+
"SupportedPlatforms": "win-x64,win-arm64"
5154
}
5255
},
5356
{
5457
"Type": "CompilerInstallation",
5558
"Parameters": {
5659
"Scenario": "InstallCompiler",
5760
"CompilerVersion": "$.Parameters.CompilerVersion",
58-
"CygwinPackages": "gcc-g++,gcc-fortran,gcc,libiconv-devel"
61+
"CygwinPackages": "gcc-g++,gcc-fortran,gcc-core,libiconv-devel"
5962
}
6063
},
6164
{

‎src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTRATE.json‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,12 @@
66
"RecommendedMinimumExecutionTime": "(8-cores)=04:00:00,(16-cores)=08:00:00,(64-cores)=10:00:00",
77
"SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64",
88
"SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows",
9-
"Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24"
9+
"Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24",
10+
"Limitations": "The following benchmarks don't work on Windows: 500, 502, 520, 523, 531"
1011
},
1112
"Parameters": {
1213
"CompilerVersion": "",
1314
"Iterations": 2,
14-
"SpecProfile": "intrate",
1515
"RunPeak": false,
1616
"Threads": "{LogicalCoreCount}",
1717
"Copies": "{LogicalCoreCount}",
@@ -24,7 +24,8 @@
2424
"Parameters": {
2525
"Scenario": "ExecuteSPECBenchmark",
2626
"Iterations": "$.Parameters.Iterations",
27-
"SpecProfile": "$.Parameters.SpecProfile",
27+
"SpecProfile": "intrate",
28+
"Benchmarks": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"intrate\" : \"505 525 541 548 557\")}",
2829
"PackageName": "speccpu2017",
2930
"RunPeak": "$.Parameters.RunPeak",
3031
"Threads": "$.Parameters.Threads",
@@ -39,23 +40,25 @@
3940
"Type": "ChocolateyInstallation",
4041
"Parameters": {
4142
"Scenario": "InstallChocolatey",
42-
"PackageName": "chocolatey"
43+
"PackageName": "chocolatey",
44+
"SupportedPlatforms": "win-x64,win-arm64"
4345
}
4446
},
4547
{
4648
"Type": "ChocolateyPackageInstallation",
4749
"Parameters": {
4850
"Scenario": "InstallCompiler",
4951
"PackageName": "chocolatey",
50-
"Packages": "cygwin"
52+
"Packages": "cygwin",
53+
"SupportedPlatforms": "win-x64,win-arm64"
5154
}
5255
},
5356
{
5457
"Type": "CompilerInstallation",
5558
"Parameters": {
5659
"Scenario": "InstallCompiler",
5760
"CompilerVersion": "$.Parameters.CompilerVersion",
58-
"CygwinPackages": "gcc-g++,gcc-fortran,gcc,libiconv-devel"
61+
"CygwinPackages": "gcc-g++,gcc-fortran,gcc-core,libiconv-devel"
5962
}
6063
},
6164
{

‎src/VirtualClient/VirtualClient.Main/profiles/PERF-SPECCPU-INTSPEED.json‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,12 @@
66
"RecommendedMinimumExecutionTime": "(8-cores)=04:00:00,(16-cores)=06:00:00,(64-cores)=08:00:00",
77
"SupportedPlatforms": "linux-x64,linux-arm64,win-x64,win-arm64",
88
"SupportedOperatingSystems": "CBL-Mariner,CentOS,Debian,RedHat,Suse,Ubuntu,Windows",
9-
"Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24"
9+
"Notes": "Using a private package 'speccpu.2017.1.1.9-danielbowers.zip' to enable support for speccpu on Ubuntu 24",
10+
"Limitations": "The following benchmarks don't work on Windows: 600, 602, 605, 620, 623, 625, 631, 648, 657"
1011
},
1112
"Parameters": {
1213
"CompilerVersion": "",
1314
"Iterations": 2,
14-
"SpecProfile": "intspeed",
1515
"RunPeak": false,
1616
"Threads": "{LogicalCoreCount}",
1717
"Copies": "{LogicalCoreCount}",
@@ -24,7 +24,8 @@
2424
"Parameters": {
2525
"Scenario": "ExecuteSPECBenchmark",
2626
"Iterations": "$.Parameters.Iterations",
27-
"SpecProfile": "$.Parameters.SpecProfile",
27+
"SpecProfile": "intspeed",
28+
"Benchmarks": "{calculate(\"{Platform}\".StartsWith(\"linux\") ? \"intspeed\" : \"641\")}",
2829
"PackageName": "speccpu2017",
2930
"RunPeak": "$.Parameters.RunPeak",
3031
"Threads": "$.Parameters.Threads",
@@ -39,23 +40,25 @@
3940
"Type": "ChocolateyInstallation",
4041
"Parameters": {
4142
"Scenario": "InstallChocolatey",
42-
"PackageName": "chocolatey"
43+
"PackageName": "chocolatey",
44+
"SupportedPlatforms": "win-x64,win-arm64"
4345
}
4446
},
4547
{
4648
"Type": "ChocolateyPackageInstallation",
4749
"Parameters": {
4850
"Scenario": "InstallCompiler",
4951
"PackageName": "chocolatey",
50-
"Packages": "cygwin"
52+
"Packages": "cygwin",
53+
"SupportedPlatforms": "win-x64,win-arm64"
5154
}
5255
},
5356
{
5457
"Type": "CompilerInstallation",
5558
"Parameters": {
5659
"Scenario": "InstallCompiler",
5760
"CompilerVersion": "$.Parameters.CompilerVersion",
58-
"CygwinPackages": "gcc-g++,gcc-fortran,gcc,libiconv-devel"
61+
"CygwinPackages": "gcc-g++,gcc-fortran,gcc-core,libiconv-devel"
5962
}
6063
},
6164
{

‎website/docs/workloads/speccpu/speccpu-profiles.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ for evaluating the performance of the CPU for processing calculations.
5959
The following section provides a few basic examples of how to use the workload profile. Additional usage examples can be found in the
6060
'Usage Scenarios/Examples' link at the top.
6161

62+
* **Notes**
63+
On Windows, only the following benchmarks are supported: 503.bwaves_r, 508.namd_r, 510.parest_r, 519.lbm_r, 544.nab_r, 549.fotonik3d_r, 554.roms_r.
64+
6265
``` csharp
6366
# Execute the workload profile
6467
./VirtualClient --profile=PERF-SPECCPU-FPRATE.json --system=Azure --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}"
@@ -110,6 +113,9 @@ for evaluating the performance of the CPU for processing calculations.
110113
* **Usage Examples**
111114
The following section provides a few basic examples of how to use the workload profile.
112115

116+
* **Notes**
117+
On Windows, only the following benchmarks are supported: 603.bwaves_s, 619.lbm_s, 644.nab_s, 654.roms_s.
118+
113119
``` bash
114120
# Execute the workload profile
115121
./VirtualClient --profile=PERF-SPECCPU-FPSPEED.json --system=Azure --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}"
@@ -158,6 +164,9 @@ for evaluating the performance of the CPU for processing calculations.
158164
* **Usage Examples**
159165
The following section provides a few basic examples of how to use the workload profile.
160166

167+
* **Notes**
168+
On Windows, only the following benchmarks are supported: 505.mcf_r, 525.x264_r, 541.leela_r, 548.exchange2_r, 557.xz_r.
169+
161170
``` bash
162171
# Execute the workload profile
163172
./VirtualClient --profile=PERF-SPECCPU-INTRATE.json --system=Azure --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}"
@@ -206,6 +215,9 @@ for evaluating the performance of the CPU for processing calculations.
206215
* **Usage Examples**
207216
The following section provides a few basic examples of how to use the workload profile.
208217

218+
* **Notes**
219+
On Windows, only the following benchmarks are supported: 641.leela_s.
220+
209221
``` bash
210222
# Execute the workload profile
211223
./VirtualClient --profile=PERF-SPECCPU-INTSPEED.json --system=Azure --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}"

0 commit comments

Comments
 (0)