From c08687a5abec6c7d716ad1501998068c06fe3bdb Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Wed, 10 Jun 2026 15:53:53 +0200 Subject: [PATCH 01/13] Enable VS integration tests on CI --- azure-pipelines-PR.yml | 8 ++++---- eng/Build.ps1 | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index b627fab985c..9e5b5cfb4f7 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -448,10 +448,10 @@ stages: _testKind: testCoreclr TEST_TRANSPARENT_COMPILER: 1 # Pipeline variable will map to env var. transparentCompiler: TransparentCompiler - # inttests_release: - # _configuration: Release - # _testKind: testIntegration - # setupVsHive: true + inttests_release: + _configuration: Release + _testKind: testIntegration + setupVsHive: true steps: - checkout: self clean: true diff --git a/eng/Build.ps1 b/eng/Build.ps1 index 41a52df6395..508dda563a1 100644 --- a/eng/Build.ps1 +++ b/eng/Build.ps1 @@ -665,7 +665,7 @@ try { TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\UnitTests\VisualFSharp.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework } - if ($testIntegration) { + if ($testIntegration -and -not $noVisualStudio) { TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework } From 33c46fd10dd3ac55207c800ab3f7cfbf071bbf6a Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Wed, 10 Jun 2026 17:21:23 +0200 Subject: [PATCH 02/13] Use xunit.console.exe to run integration tests Can't use `dotnet` due to a conflict with mtp/xunit3 --- eng/Build.ps1 | 40 ++++++++++++++++++- eng/Versions.props | 2 + .../FSharp.Editor.IntegrationTests.csproj | 9 +++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/eng/Build.ps1 b/eng/Build.ps1 index 508dda563a1..8bfcce6c0ab 100644 --- a/eng/Build.ps1 +++ b/eng/Build.ps1 @@ -402,6 +402,44 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str Exec-Console $dotnetExe $test_args } +# Runs a test assembly via the xUnit v2 console runner, bypassing `dotnet test` (and therefore +# the repo-wide Microsoft.Testing.Platform gate declared in global.json). Used only for projects +# whose harness cannot run under MTP -- today that is FSharp.Editor.IntegrationTests, which +# depends on Microsoft.VisualStudio.Extensibility.Testing.Xunit (VS-hive launcher, xUnit-v2-locked). +# The runner is restored via a on the +# test project, so it is present in the NuGet global packages folder after a normal slnx restore. +function TestUsingXUnitConsole([string] $testProject, [string] $targetFramework) { + + $projectName = [System.IO.Path]::GetFileNameWithoutExtension($testProject) + $assemblyPath = "$ArtifactsDir\bin\$projectName\$configuration\$targetFramework\$projectName.dll" + if (-not (Test-Path $assemblyPath)) { + throw "Test assembly not found at $assemblyPath. Was $projectName built before -testIntegration was invoked?" + } + + $runnerVersion = (Select-Xml -Path "$RepoRoot\eng\Versions.props" -XPath "//XunitRunnerConsoleV2Version").Node.InnerText + if ([string]::IsNullOrWhiteSpace($runnerVersion)) { + throw "XunitRunnerConsoleV2Version is not defined in eng\Versions.props." + } + + $nugetRoot = GetNuGetPackageCachePath + $xunitConsole = Join-Path $nugetRoot "xunit.runner.console\$runnerVersion\tools\net472\xunit.console.exe" + if (-not (Test-Path $xunitConsole)) { + throw "xunit.console.exe not found at $xunitConsole. Ensure restore of $projectName ran first (PackageDownload of xunit.runner.console v$runnerVersion)." + } + + $testResultsDir = "$ArtifactsDir\TestResults\$configuration" + Create-Directory $testResultsDir + $jobName = if ($env:SYSTEM_JOBNAME) { $env:SYSTEM_JOBNAME } else { "local" } + $resultsXml = Join-Path $testResultsDir "$projectName.$targetFramework.$jobName.xml" + + # -parallel none / -noshadow mirror the project's xunit.runner.json (parallelizeTestCollections=false, shadowCopy=false). + $xunit_args = """$assemblyPath"" -xml ""$resultsXml"" -parallel none -noshadow -nologo" + + Write-Host("$xunitConsole $xunit_args") + + Exec-Console $xunitConsole $xunit_args +} + function Prepare-TempDir() { Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.props") $TempDir Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.targets") $TempDir @@ -666,7 +704,7 @@ try { } if ($testIntegration -and -not $noVisualStudio) { - TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework + TestUsingXUnitConsole -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework } if ($testAOT) { diff --git a/eng/Versions.props b/eng/Versions.props index 08cde86aa77..7c3283063e6 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -161,6 +161,8 @@ 13.0.4 3.2.2 3.2.2 + + 2.9.3 8.0.0 diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/FSharp.Editor.IntegrationTests.csproj b/vsintegration/tests/FSharp.Editor.IntegrationTests/FSharp.Editor.IntegrationTests.csproj index 374c8164a5b..af0a08656ae 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/FSharp.Editor.IntegrationTests.csproj +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/FSharp.Editor.IntegrationTests.csproj @@ -35,6 +35,15 @@ + + + + + From 1c638d26eb35bc0850b96b35f5081516ed9c1eac Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Wed, 10 Jun 2026 18:14:22 +0200 Subject: [PATCH 03/13] Refactor to use helper Get-PackageVersion --- eng/Build.ps1 | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/eng/Build.ps1 b/eng/Build.ps1 index 8bfcce6c0ab..960642bac95 100644 --- a/eng/Build.ps1 +++ b/eng/Build.ps1 @@ -416,13 +416,10 @@ function TestUsingXUnitConsole([string] $testProject, [string] $targetFramework) throw "Test assembly not found at $assemblyPath. Was $projectName built before -testIntegration was invoked?" } - $runnerVersion = (Select-Xml -Path "$RepoRoot\eng\Versions.props" -XPath "//XunitRunnerConsoleV2Version").Node.InnerText - if ([string]::IsNullOrWhiteSpace($runnerVersion)) { - throw "XunitRunnerConsoleV2Version is not defined in eng\Versions.props." - } - - $nugetRoot = GetNuGetPackageCachePath - $xunitConsole = Join-Path $nugetRoot "xunit.runner.console\$runnerVersion\tools\net472\xunit.console.exe" + # Get-PackageVersion (eng\build-utils.ps1) reads from eng\Versions.props, + # and Get-PackageDir resolves the NuGet cache path. Defensive Trim() covers accidental whitespace in the value. + $runnerVersion = (Get-PackageVersion "XunitRunnerConsoleV2").Trim() + $xunitConsole = Join-Path (Get-PackageDir "xunit.runner.console" $runnerVersion) "tools\net472\xunit.console.exe" if (-not (Test-Path $xunitConsole)) { throw "xunit.console.exe not found at $xunitConsole. Ensure restore of $projectName ran first (PackageDownload of xunit.runner.console v$runnerVersion)." } From 534e1c843fb684d2a95237443d03c4c25e467a73 Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Thu, 11 Jun 2026 10:02:04 +0200 Subject: [PATCH 04/13] Copy test dll config to build output --- .../FSharp.Editor.IntegrationTests.csproj | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/FSharp.Editor.IntegrationTests.csproj b/vsintegration/tests/FSharp.Editor.IntegrationTests/FSharp.Editor.IntegrationTests.csproj index af0a08656ae..9b04ed904fd 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/FSharp.Editor.IntegrationTests.csproj +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/FSharp.Editor.IntegrationTests.csproj @@ -46,6 +46,10 @@ + + From ee4a258fdc4596d90a960225a488ff12c77bd1f1 Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Thu, 11 Jun 2026 11:10:06 +0200 Subject: [PATCH 05/13] Bump vs-extensibility-testing --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 7c3283063e6..f93e881a823 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -130,7 +130,7 @@ $(VisualStudioEditorPackagesVersion) $(VisualStudioEditorPackagesVersion) 17.14.0 - 0.1.800-beta + 5.3.0-2.26055.8 $(MicrosoftVisualStudioExtensibilityTestingVersion) From 37d91f4555a7f739bdc121f9edfc19a53ff7c3fb Mon Sep 17 00:00:00 2001 From: Adam Boniecki <20281641+abonie@users.noreply.github.com> Date: Thu, 11 Jun 2026 13:11:24 +0200 Subject: [PATCH 06/13] Use dotnet new for test projects --- .../InProcess/SolutionExplorerInProcess.cs | 137 ++++++++++++++++-- .../WellKnownProjectTemplates.cs | 9 +- 2 files changed, 131 insertions(+), 15 deletions(-) diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/SolutionExplorerInProcess.cs b/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/SolutionExplorerInProcess.cs index 43743484e96..430a4940889 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/SolutionExplorerInProcess.cs +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/SolutionExplorerInProcess.cs @@ -4,9 +4,11 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; using System.Runtime.CompilerServices; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.VisualStudio.OperationProgress; @@ -50,17 +52,42 @@ public async Task CreateSingleProjectSolutionAsync(string name, string template, await AddProjectAsync(name, template, cancellationToken); } - // Repo root from compile-time source path — no runtime resolution needed. + // Repo root from compile-time source path -- no runtime resolution needed. private static readonly string RepoRoot = Path.GetFullPath(Path.Combine( Path.GetDirectoryName(GetSourceFilePath())!, "..", "..", "..", "..")); + // Configuration the test assembly itself was built with (Release on CI, typically Debug locally). + // Layout: /artifacts/bin/FSharp.Editor.IntegrationTests///FSharp.Editor.IntegrationTests.dll + // The synthesized standalone project (CreateStandaloneProjectFile) must reference the matching + // fsc.dll, otherwise the VS build fails with "Could not find file ...artifacts/bin/fsc///fsc.dll". + private static readonly string LocalCompilerConfiguration = new DirectoryInfo( + Path.GetDirectoryName(typeof(SolutionExplorerInProcess).Assembly.Location)!).Parent!.Name; + + // Repo-pinned dotnet host installed by Arcade. Falls back to PATH lookup if not present + // (developer scenarios that build the integration project outside the repo's eng infra). + private static readonly string DotnetExe = + File.Exists(Path.Combine(RepoRoot, ".dotnet", "dotnet.exe")) + ? Path.Combine(RepoRoot, ".dotnet", "dotnet.exe") + : "dotnet"; + private static string CreateStandaloneProjectFile() { var propsPath = Path.Combine(RepoRoot, "UseLocalCompiler.Directory.Build.props"); + // Sanity-check the inferred configuration: matching fsc must exist before VS tries to build. + // Validated here (not in the static initializer) so tests that don't use the standalone path + // can still load this type when fsc hasn't been built locally. + var fscRoot = Path.Combine(RepoRoot, "artifacts", "bin", "fsc", LocalCompilerConfiguration); + if (!Directory.Exists(fscRoot)) + { + throw new InvalidOperationException( + $"Inferred LocalCompilerConfiguration='{LocalCompilerConfiguration}' but no built fsc found at '{fscRoot}'. " + + $"The synthesized standalone fsproj would fail to load fsc.dll -- build the F# compiler in this configuration first."); + } + return $@" - Debug + {LocalCompilerConfiguration} {RepoRoot} @@ -115,22 +142,108 @@ private async Task GetDirectoryNameAsync(CancellationToken cancellationT public async Task AddProjectAsync(string projectName, string projectTemplate, CancellationToken cancellationToken) { - await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken); + // dotnet new must run off the UI thread (it's a synchronous shell-out). + var solutionDir = await GetDirectoryNameAsync(cancellationToken); + var projectDir = Path.Combine(solutionDir, projectName); - var projectPath = Path.Combine(await GetDirectoryNameAsync(cancellationToken), projectName); - var projectTemplatePath = await GetProjectTemplatePathAsync(projectTemplate, cancellationToken); - var solution = await GetRequiredGlobalServiceAsync(cancellationToken); - ErrorHandler.ThrowOnFailure(solution.AddNewProjectFromTemplate(projectTemplatePath, null, null, projectPath, projectName, null, out _)); - } + await TaskScheduler.Default; + await RunDotnetNewAsync(projectTemplate, projectName, projectDir, cancellationToken); - private async Task GetProjectTemplatePathAsync(string projectTemplate, CancellationToken cancellationToken) - { - await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken); + var projectFilePath = Path.Combine(projectDir, $"{projectName}.fsproj"); + if (!File.Exists(projectFilePath)) + { + throw new InvalidOperationException( + $"'dotnet new {projectTemplate}' completed but did not produce '{projectFilePath}'."); + } + await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken); var dte = await GetRequiredGlobalServiceAsync(cancellationToken); var solution = (EnvDTE80.Solution2)dte.Solution; + _ = solution.AddFromFile(projectFilePath, false); + + // Auto-open the project's main .fs file. The previous AddNewProjectFromTemplate path opened + // the file marked OpenInEditor="true" in the .vstemplate; tests rely on this implicit open + // (e.g. CodeActionTests calls Editor.SetTextAsync immediately afterwards). + // SDK templates currently produce exactly one .fs file per project (Library.fs / Program.fs / Tests.fs). + // If that ever changes (e.g. xunit template growing a Program.fs), fail loudly rather than + // silently opening the wrong file and producing confusing content-diff failures downstream. + var fsFiles = Directory.EnumerateFiles(projectDir, "*.fs", SearchOption.TopDirectoryOnly) + .OrderBy(p => p, StringComparer.OrdinalIgnoreCase) + .ToList(); + if (fsFiles.Count != 1) + { + throw new InvalidOperationException( + $"Expected exactly one *.fs file in '{projectDir}' produced by 'dotnet new {projectTemplate}', " + + $"found {fsFiles.Count}: [{string.Join(", ", fsFiles.Select(Path.GetFileName))}]. " + + $"Update AddProjectAsync's auto-open logic to pick the correct main file for this template."); + } + await OpenFileAsync(projectName, Path.GetFileName(fsFiles[0]), cancellationToken); + } + + // Shells out to `dotnet new