Enable VS integration tests on CI

azure-pipelines-PR.yml

@@ -448,10 +448,10 @@ stages:
                 _testKind: testCoreclr
                 TEST_TRANSPARENT_COMPILER: 1 # Pipeline variable will map to env var.
                 transparentCompiler: TransparentCompiler
-              # inttests_release:
-              #   _configuration: Release
-              #   _testKind: testIntegration
-              #   setupVsHive: true
+              inttests_release:
+                _configuration: Release
+                _testKind: testIntegration
+                setupVsHive: true
           steps:
           - checkout: self
             clean: true

eng/Build.ps1

@@ -402,6 +402,41 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str
     Exec-Console $dotnetExe $test_args
 }
 
+# Runs a test assembly via the xUnit v2 console runner, bypassing `dotnet test` (and therefore
+# the repo-wide Microsoft.Testing.Platform gate declared in global.json). Used only for projects
+# whose harness cannot run under MTP -- today that is FSharp.Editor.IntegrationTests, which
+# depends on Microsoft.VisualStudio.Extensibility.Testing.Xunit (VS-hive launcher, xUnit-v2-locked).
+# The runner is restored via a <PackageDownload Include="xunit.runner.console" .../> on the
+# test project, so it is present in the NuGet global packages folder after a normal slnx restore.
+function TestUsingXUnitConsole([string] $testProject, [string] $targetFramework) {
+
+    $projectName = [System.IO.Path]::GetFileNameWithoutExtension($testProject)
+    $assemblyPath = "$ArtifactsDir\bin\$projectName\$configuration\$targetFramework\$projectName.dll"
+    if (-not (Test-Path $assemblyPath)) {
+        throw "Test assembly not found at $assemblyPath. Was $projectName built before -testIntegration was invoked?"
+    }
+
+    # Get-PackageVersion (eng\build-utils.ps1) reads <XunitRunnerConsoleV2Version> from eng\Versions.props,
+    # and Get-PackageDir resolves the NuGet cache path. Defensive Trim() covers accidental whitespace in the value.
+    $runnerVersion = (Get-PackageVersion "XunitRunnerConsoleV2").Trim()
+    $xunitConsole = Join-Path (Get-PackageDir "xunit.runner.console" $runnerVersion) "tools\net472\xunit.console.exe"
+    if (-not (Test-Path $xunitConsole)) {
+        throw "xunit.console.exe not found at $xunitConsole. Ensure restore of $projectName ran first (PackageDownload of xunit.runner.console v$runnerVersion)."
+    }
+
+    $testResultsDir = "$ArtifactsDir\TestResults\$configuration"
+    Create-Directory $testResultsDir
+    $jobName = if ($env:SYSTEM_JOBNAME) { $env:SYSTEM_JOBNAME } else { "local" }
+    $resultsXml = Join-Path $testResultsDir "$projectName.$targetFramework.$jobName.xml"
+
+    # -parallel none / -noshadow mirror the project's xunit.runner.json (parallelizeTestCollections=false, shadowCopy=false).
+    $xunit_args = """$assemblyPath"" -xml ""$resultsXml"" -parallel none -noshadow -nologo"
+
+    Write-Host("$xunitConsole $xunit_args")
+
+    Exec-Console $xunitConsole $xunit_args
+}
+
 function Prepare-TempDir() {
     Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.props") $TempDir
     Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.targets") $TempDir
@@ -665,8 +700,8 @@ try {
         TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\UnitTests\VisualFSharp.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework
     }
 
-    if ($testIntegration) {
-        TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework
+    if ($testIntegration -and -not $noVisualStudio) {
+        TestUsingXUnitConsole -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework
     }
 
     if ($testAOT) {

eng/Versions.props

@@ -130,7 +130,7 @@
     <MicrosoftVisualStudioPlatformVSEditorVersion>$(VisualStudioEditorPackagesVersion)</MicrosoftVisualStudioPlatformVSEditorVersion>
     <MicrosoftVisualStudioTextUIWpfVersion>$(VisualStudioEditorPackagesVersion)</MicrosoftVisualStudioTextUIWpfVersion>
     <NuGetVisualStudioVersion>17.14.0</NuGetVisualStudioVersion>
-    <MicrosoftVisualStudioExtensibilityTestingVersion>0.1.800-beta</MicrosoftVisualStudioExtensibilityTestingVersion>
+    <MicrosoftVisualStudioExtensibilityTestingVersion>5.3.0-2.26055.8</MicrosoftVisualStudioExtensibilityTestingVersion>
     <MicrosoftVisualStudioExtensibilityTestingSourceGeneratorVersion>$(MicrosoftVisualStudioExtensibilityTestingVersion)</MicrosoftVisualStudioExtensibilityTestingSourceGeneratorVersion>
 
     <!-- Visual Studio Threading packages -->
@@ -161,6 +161,8 @@
     <NewtonsoftJsonVersion>13.0.4</NewtonsoftJsonVersion>
     <XunitVersion>3.2.2</XunitVersion>
     <XunitRunnerConsoleVersion>3.2.2</XunitRunnerConsoleVersion>
+    <!-- xUnit v2 runner, only consumed by FSharp.Editor.IntegrationTests (VS-hive harness is xUnit-v2-locked). -->
+    <XunitRunnerConsoleV2Version>2.9.3</XunitRunnerConsoleV2Version>
     <XunitXmlTestLoggerVersion>8.0.0</XunitXmlTestLoggerVersion>
     <!-- -->
   </PropertyGroup>

vsintegration/tests/FSharp.Editor.IntegrationTests/AbstractIntegrationTest.cs

@@ -9,7 +9,7 @@
 
 namespace Microsoft.CodeAnalysis.Testing
 {
-    [IdeSettings(MinVersion = VisualStudioVersion.VS2022)]
+    [IdeSettings(MinVersion = VisualStudioVersion.VS18, MaxVersion = VisualStudioVersion.VS18)]
     public abstract class AbstractIntegrationTest : AbstractIdeIntegrationTest
     {
         protected CancellationToken TestToken => HangMitigatingCancellationToken;

vsintegration/tests/FSharp.Editor.IntegrationTests/BuildProjectTests.cs

@@ -43,7 +43,7 @@ module Test
 let answer =
 """;
         var expectedBuildSummary = "========== Build: 0 succeeded, 1 failed, 0 up-to-date, 0 skipped ==========";
-        var expectedError = "(Compiler) Library.fs(3, 1): error FS0010: Incomplete structured construct at or before this point in binding";
+        var expectedError = "(Fsc) Library.fs(3, 1): error FS0010: Incomplete structured construct at or before this point in binding";
 
         await SolutionExplorer.CreateSingleProjectSolutionAsync("Library", template, TestToken);
         await SolutionExplorer.RestoreNuGetPackagesAsync(TestToken);

vsintegration/tests/FSharp.Editor.IntegrationTests/DebuggingSequencePointTests.cs

@@ -83,13 +83,23 @@ private async Task PrepareProjectAndBuildAsync()
         await SolutionExplorer.CreateSingleProjectSolutionAsync(ProjectName, SolutionExplorerInProcess.ExistingProjectTemplate, TestToken);
         await SolutionExplorer.SetStartupProjectAsync(ProjectName, TestToken);
 
+        // Write the fixture directly to disk *before* opening it in VS, instead of using
+        // SetTextAsync + "File.SaveAll" command. Reason: the debugger binds breakpoints by
+        // matching the PDB's recorded source-file hash against the hash of Program.fs on disk.
+        // If the edit-buffer is built first and SaveAll lands a moment later, the PDB hash
+        // points at one snapshot while disk holds another -- breakpoints stay unbound
+        // (children=0) and never fire. Writing to disk first keeps the two in lockstep.
+        await SolutionExplorer.WriteFileAsync(ProjectName, "Program.fs", File.ReadAllText(GetFixturePath()), TestToken);
         await SolutionExplorer.OpenFileAsync(ProjectName, "Program.fs", TestToken);
-        await Editor.SetTextAsync(File.ReadAllText(GetFixturePath()), TestToken);
-        await Shell.ExecuteCommandAsync("File.SaveAll", TestToken);
 
         var buildSummary = await SolutionExplorer.BuildSolutionAsync(TestToken);
         Assert.NotNull(buildSummary);
         Assert.Contains("Build: 1 succeeded, 0 failed", string.Join(Environment.NewLine, buildSummary));
+
+        // BuildSolutionAsync leaves the Build Output pane as the active text view; re-open Program.fs
+        // so the subsequent PlaceCaretAsync / ToggleBreakpointAtMarkerAsync operate on the F# source
+        // (rather than searching the build log for "BP_..." markers).
+        await SolutionExplorer.OpenFileAsync(ProjectName, "Program.fs", TestToken);
     }
 
     private static string GetFixturePath()

vsintegration/tests/FSharp.Editor.IntegrationTests/FSharp.Editor.IntegrationTests.csproj

@@ -35,8 +35,21 @@
     <PackageReference Include="Nullable" Version="1.3.0" />
   </ItemGroup>
 
+  <ItemGroup>
+    <!-- xUnit v2 console runner: this project is excluded from the repo-wide Microsoft.Testing.Platform
+         gate (ExcludeFromTestPackageReferences above) because Microsoft.VisualStudio.Extensibility.Testing.Xunit
+         is locked to xUnit v2. PackageDownload restores the runner to the NuGet global packages folder
+         so eng\Build.ps1's TestUsingXUnitConsole can locate it, without adding it to this project's
+         compile/runtime closure. PackageDownload requires the bracketed exact-pin syntax. -->
+    <PackageDownload Include="xunit.runner.console" Version="[$(XunitRunnerConsoleV2Version)]" />
+  </ItemGroup>
+
   <Target Name="SyncXunitDesktopDll" AfterTargets="Build" Condition="'$(OutputType)' == 'Exe'">
     <Copy SourceFiles="$(TargetPath)" DestinationFiles="$(TargetDir)$(AssemblyName).dll" SkipUnchangedFiles="true" />
+    <!-- xunit.console.exe loads <dll>.config (not <exe>.config) for the test-AppDomain's binding redirects.
+         Without this copy, the IdeTestFramework's heavy VS-interop dependency graph fails to bind and the
+         custom xUnit test framework silently reports 0 discovered tests. -->
+    <Copy SourceFiles="$(TargetPath).config" DestinationFiles="$(TargetDir)$(AssemblyName).dll.config" SkipUnchangedFiles="true" Condition="Exists('$(TargetPath).config')" />
   </Target>
 
 </Project>

vsintegration/tests/FSharp.Editor.IntegrationTests/GoToDefinitionTests.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using Microsoft.CodeAnalysis.Testing;
+using System;
 using System.Threading.Tasks;
 using Xunit;
 using static Microsoft.VisualStudio.VSConstants;
@@ -11,6 +12,14 @@ namespace FSharp.Editor.IntegrationTests;
 
 public class GoToDefinitionTests : AbstractIntegrationTest
 {
+    // Roslyn's workspace propagates buffer edits to its solution snapshot asynchronously, and the
+    // F# checker schedules its own typecheck on top. WaitForProjectSystemAsync only covers VS
+    // project-system loading -- not these two async hops -- so GoToDefn invoked too quickly after
+    // SetTextAsync / OpenFileAsync sees a stale Document and no-ops. A short delay is the cheapest
+    // pragmatic bridge until a proper Roslyn-style WaitForAsyncOperationsAsync(FeatureAttribute.Workspace)
+    // partial is wired up here.
+    private static readonly TimeSpan FSharpCheckerSettleDelay = TimeSpan.FromSeconds(2);
+
     [IdeFact]
     public async Task GoesToDefinition()
     {
@@ -28,7 +37,8 @@ module Test
         await SolutionExplorer.CreateSingleProjectSolutionAsync("Library", template, TestToken);
         await SolutionExplorer.RestoreNuGetPackagesAsync(TestToken);
         await Editor.SetTextAsync(code, TestToken);
-
+        await Task.Delay(FSharpCheckerSettleDelay, TestToken);
+
         await Editor.PlaceCaretAsync("add 1", TestToken);
         await Shell.ExecuteCommandAsync(VSStd97CmdID.GotoDefn, TestToken);
         var actualText = await Editor.GetCurrentLineTextAsync(TestToken);
@@ -72,6 +82,7 @@ let id (t: SomeType) = t
         await SolutionExplorer.BuildSolutionAsync(TestToken);
 
         await SolutionExplorer.OpenFileAsync("Library", "Module.fsi", TestToken);
+        await Task.Delay(FSharpCheckerSettleDelay, TestToken);
         await Editor.PlaceCaretAsync("SomeType ->", TestToken);
         await Shell.ExecuteCommandAsync(VSStd97CmdID.GotoDefn, TestToken);
         var expectedText = "type SomeType =";
@@ -82,6 +93,7 @@ let id (t: SomeType) = t
         Assert.Equal(expectedWindow, actualWindow);
 
         await SolutionExplorer.OpenFileAsync("Library", "Module.fs", TestToken);
+        await Task.Delay(FSharpCheckerSettleDelay, TestToken);
         await Editor.PlaceCaretAsync("SomeType)", TestToken);
         await Shell.ExecuteCommandAsync(VSStd97CmdID.GotoDefn, TestToken);
         expectedText = "type SomeType =";

vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/EditorInProcess.cs

@@ -79,16 +79,35 @@ public async Task<IEnumerable<SuggestedActionSet>> InvokeCodeActionListAsync(Can
 
         var shell = await GetRequiredGlobalServiceAsync<SVsUIShell, IVsUIShell>(cancellationToken);
         var cmdGroup = typeof(VSConstants.VSStd14CmdID).GUID;
-        var cmdExecOpt = OLECMDEXECOPT.OLECMDEXECOPT_DONTPROMPTUSER;
-
         var cmdID = VSConstants.VSStd14CmdID.ShowQuickFixes;
+
+        // Post ShowQuickFixes once. PostExecCommand is fire-and-forget; the broker spins up the
+        // session asynchronously after F# diagnostics surface, so we have to wait for the session
+        // to materialize before LightBulbHelper.WaitForItemsAsync can subscribe to its events.
+        // Repeated PostExecCommand calls would dismiss any in-flight session and reset the race.
         object? obj = null;
-        shell.PostExecCommand(cmdGroup, (uint)cmdID, (uint)cmdExecOpt, ref obj);
+        shell.PostExecCommand(cmdGroup, (uint)cmdID, (uint)OLECMDEXECOPT.OLECMDEXECOPT_DONTPROMPTUSER, ref obj);
 
         var view = await GetActiveTextViewAsync(cancellationToken);
         var broker = await GetComponentModelServiceAsync<ILightBulbBroker>(cancellationToken);
 
-        var lightbulbs = await LightBulbHelper.WaitForItemsAsync(broker, view, cancellationToken);
-        return lightbulbs;
+        // Poll for the session to appear (up to ~5 s). Without this wait, the F#-analyzer-driven
+        // code fixes (UnusedOpenDeclarations, AddMissingFunKeyword) NRE because broker.GetSession
+        // returns null before the lightbulb session is ready -- LightBulbHelper.WaitForItemsAsync
+        // then casts null to IAsyncLightBulbSession and subscribes to its events.
+        var sessionTimeout = TimeSpan.FromSeconds(5);
+        var sw = System.Diagnostics.Stopwatch.StartNew();
+        while (broker.GetSession(view) is null)
+        {
+            if (sw.Elapsed > sessionTimeout)
+            {
+                // Final attempt -- let LightBulbHelper's existing NRE propagate so the test failure
+                // points at the actual root cause, not a manufactured timeout exception.
+                break;
+            }
+            await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken);
+        }
+
+        return await LightBulbHelper.WaitForItemsAsync(broker, view, cancellationToken);
     }
 }

vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/ShellInProcess_Debugging.cs

@@ -84,10 +84,12 @@ public async Task WaitForBreakpointHitAsync(TimeSpan timeout, bool continueOnSte
                 dbg.Go(true);
 
             if (seenRun && mode == EnvDTE.dbgDebugMode.dbgDesignMode)
-                throw new InvalidOperationException("Debug session ended before breakpoint hit.");
+                throw new InvalidOperationException(
+                    $"Debug session ended before breakpoint hit. Breakpoints={BreakpointSummary(dbg)}.");
 
             if (sw.Elapsed > timeout)
-                throw new TimeoutException($"No breakpoint hit after {timeout}. Mode={mode}; Breakpoints={BreakpointSummary(dbg)}.");
+                throw new TimeoutException(
+                    $"No breakpoint hit after {timeout}. Mode={mode}; Breakpoints={BreakpointSummary(dbg)}.");
 
             await Task.Delay(100, cancellationToken);
         }
@@ -122,7 +124,11 @@ private static string BreakpointSummary(EnvDTE.Debugger dbg)
         {
             var children = 0;
             foreach (EnvDTE.Breakpoint _ in bp.Children) children++;
-            items.Add($"{Path.GetFileName(bp.File)}:{bp.FileLine}(children={children})");
+            // CurrentHits + Children counts together let us distinguish unbound (children=0, hits=0)
+            // from "bound but never executed" (children>0, hits=0) from "bound and hit" (hits>0).
+            // Unbound means the PDB has no IL location for this source line; the disk-source hash
+            // didn't match the PDB's recorded hash; or the loaded module doesn't have this code.
+            items.Add($"{Path.GetFileName(bp.File)}:{bp.FileLine}(children={children},hits={bp.CurrentHits})");
         }
         return items.Count == 0 ? "none" : string.Join(",", items);
     }