diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index b627fab985c..9e5b5cfb4f7 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -448,10 +448,10 @@ stages: _testKind: testCoreclr TEST_TRANSPARENT_COMPILER: 1 # Pipeline variable will map to env var. transparentCompiler: TransparentCompiler - # inttests_release: - # _configuration: Release - # _testKind: testIntegration - # setupVsHive: true + inttests_release: + _configuration: Release + _testKind: testIntegration + setupVsHive: true steps: - checkout: self clean: true diff --git a/eng/Build.ps1 b/eng/Build.ps1 index 41a52df6395..960642bac95 100644 --- a/eng/Build.ps1 +++ b/eng/Build.ps1 @@ -402,6 +402,41 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str Exec-Console $dotnetExe $test_args } +# Runs a test assembly via the xUnit v2 console runner, bypassing `dotnet test` (and therefore +# the repo-wide Microsoft.Testing.Platform gate declared in global.json). Used only for projects +# whose harness cannot run under MTP -- today that is FSharp.Editor.IntegrationTests, which +# depends on Microsoft.VisualStudio.Extensibility.Testing.Xunit (VS-hive launcher, xUnit-v2-locked). +# The runner is restored via a on the +# test project, so it is present in the NuGet global packages folder after a normal slnx restore. +function TestUsingXUnitConsole([string] $testProject, [string] $targetFramework) { + + $projectName = [System.IO.Path]::GetFileNameWithoutExtension($testProject) + $assemblyPath = "$ArtifactsDir\bin\$projectName\$configuration\$targetFramework\$projectName.dll" + if (-not (Test-Path $assemblyPath)) { + throw "Test assembly not found at $assemblyPath. Was $projectName built before -testIntegration was invoked?" + } + + # Get-PackageVersion (eng\build-utils.ps1) reads from eng\Versions.props, + # and Get-PackageDir resolves the NuGet cache path. Defensive Trim() covers accidental whitespace in the value. + $runnerVersion = (Get-PackageVersion "XunitRunnerConsoleV2").Trim() + $xunitConsole = Join-Path (Get-PackageDir "xunit.runner.console" $runnerVersion) "tools\net472\xunit.console.exe" + if (-not (Test-Path $xunitConsole)) { + throw "xunit.console.exe not found at $xunitConsole. Ensure restore of $projectName ran first (PackageDownload of xunit.runner.console v$runnerVersion)." + } + + $testResultsDir = "$ArtifactsDir\TestResults\$configuration" + Create-Directory $testResultsDir + $jobName = if ($env:SYSTEM_JOBNAME) { $env:SYSTEM_JOBNAME } else { "local" } + $resultsXml = Join-Path $testResultsDir "$projectName.$targetFramework.$jobName.xml" + + # -parallel none / -noshadow mirror the project's xunit.runner.json (parallelizeTestCollections=false, shadowCopy=false). + $xunit_args = """$assemblyPath"" -xml ""$resultsXml"" -parallel none -noshadow -nologo" + + Write-Host("$xunitConsole $xunit_args") + + Exec-Console $xunitConsole $xunit_args +} + function Prepare-TempDir() { Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.props") $TempDir Copy-Item (Join-Path $RepoRoot "tests\Resources\Directory.Build.targets") $TempDir @@ -665,8 +700,8 @@ try { TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\UnitTests\VisualFSharp.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework } - if ($testIntegration) { - TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework + if ($testIntegration -and -not $noVisualStudio) { + TestUsingXUnitConsole -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework } if ($testAOT) { diff --git a/eng/Versions.props b/eng/Versions.props index 08cde86aa77..f93e881a823 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -130,7 +130,7 @@ $(VisualStudioEditorPackagesVersion) $(VisualStudioEditorPackagesVersion) 17.14.0 - 0.1.800-beta + 5.3.0-2.26055.8 $(MicrosoftVisualStudioExtensibilityTestingVersion) @@ -161,6 +161,8 @@ 13.0.4 3.2.2 3.2.2 + + 2.9.3 8.0.0 diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/AbstractIntegrationTest.cs b/vsintegration/tests/FSharp.Editor.IntegrationTests/AbstractIntegrationTest.cs index 8a42bed252b..10ba847c11e 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/AbstractIntegrationTest.cs +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/AbstractIntegrationTest.cs @@ -9,7 +9,7 @@ namespace Microsoft.CodeAnalysis.Testing { - [IdeSettings(MinVersion = VisualStudioVersion.VS2022)] + [IdeSettings(MinVersion = VisualStudioVersion.VS18, MaxVersion = VisualStudioVersion.VS18)] public abstract class AbstractIntegrationTest : AbstractIdeIntegrationTest { protected CancellationToken TestToken => HangMitigatingCancellationToken; diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/BuildProjectTests.cs b/vsintegration/tests/FSharp.Editor.IntegrationTests/BuildProjectTests.cs index ca89983189e..6605eda0199 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/BuildProjectTests.cs +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/BuildProjectTests.cs @@ -43,7 +43,7 @@ module Test let answer = """; var expectedBuildSummary = "========== Build: 0 succeeded, 1 failed, 0 up-to-date, 0 skipped =========="; - var expectedError = "(Compiler) Library.fs(3, 1): error FS0010: Incomplete structured construct at or before this point in binding"; + var expectedError = "(Fsc) Library.fs(3, 1): error FS0010: Incomplete structured construct at or before this point in binding"; await SolutionExplorer.CreateSingleProjectSolutionAsync("Library", template, TestToken); await SolutionExplorer.RestoreNuGetPackagesAsync(TestToken); diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/DebuggingSequencePointTests.cs b/vsintegration/tests/FSharp.Editor.IntegrationTests/DebuggingSequencePointTests.cs index 7d902dbab6d..5aa5f121c01 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/DebuggingSequencePointTests.cs +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/DebuggingSequencePointTests.cs @@ -83,13 +83,23 @@ private async Task PrepareProjectAndBuildAsync() await SolutionExplorer.CreateSingleProjectSolutionAsync(ProjectName, SolutionExplorerInProcess.ExistingProjectTemplate, TestToken); await SolutionExplorer.SetStartupProjectAsync(ProjectName, TestToken); + // Write the fixture directly to disk *before* opening it in VS, instead of using + // SetTextAsync + "File.SaveAll" command. Reason: the debugger binds breakpoints by + // matching the PDB's recorded source-file hash against the hash of Program.fs on disk. + // If the edit-buffer is built first and SaveAll lands a moment later, the PDB hash + // points at one snapshot while disk holds another -- breakpoints stay unbound + // (children=0) and never fire. Writing to disk first keeps the two in lockstep. + await SolutionExplorer.WriteFileAsync(ProjectName, "Program.fs", File.ReadAllText(GetFixturePath()), TestToken); await SolutionExplorer.OpenFileAsync(ProjectName, "Program.fs", TestToken); - await Editor.SetTextAsync(File.ReadAllText(GetFixturePath()), TestToken); - await Shell.ExecuteCommandAsync("File.SaveAll", TestToken); var buildSummary = await SolutionExplorer.BuildSolutionAsync(TestToken); Assert.NotNull(buildSummary); Assert.Contains("Build: 1 succeeded, 0 failed", string.Join(Environment.NewLine, buildSummary)); + + // BuildSolutionAsync leaves the Build Output pane as the active text view; re-open Program.fs + // so the subsequent PlaceCaretAsync / ToggleBreakpointAtMarkerAsync operate on the F# source + // (rather than searching the build log for "BP_..." markers). + await SolutionExplorer.OpenFileAsync(ProjectName, "Program.fs", TestToken); } private static string GetFixturePath() diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/FSharp.Editor.IntegrationTests.csproj b/vsintegration/tests/FSharp.Editor.IntegrationTests/FSharp.Editor.IntegrationTests.csproj index 374c8164a5b..9b04ed904fd 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/FSharp.Editor.IntegrationTests.csproj +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/FSharp.Editor.IntegrationTests.csproj @@ -35,8 +35,21 @@ + + + + + + + diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/GoToDefinitionTests.cs b/vsintegration/tests/FSharp.Editor.IntegrationTests/GoToDefinitionTests.cs index 66bb33eccd9..a5a141de5bd 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/GoToDefinitionTests.cs +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/GoToDefinitionTests.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using Microsoft.CodeAnalysis.Testing; +using System; using System.Threading.Tasks; using Xunit; using static Microsoft.VisualStudio.VSConstants; @@ -11,6 +12,14 @@ namespace FSharp.Editor.IntegrationTests; public class GoToDefinitionTests : AbstractIntegrationTest { + // Roslyn's workspace propagates buffer edits to its solution snapshot asynchronously, and the + // F# checker schedules its own typecheck on top. WaitForProjectSystemAsync only covers VS + // project-system loading -- not these two async hops -- so GoToDefn invoked too quickly after + // SetTextAsync / OpenFileAsync sees a stale Document and no-ops. A short delay is the cheapest + // pragmatic bridge until a proper Roslyn-style WaitForAsyncOperationsAsync(FeatureAttribute.Workspace) + // partial is wired up here. + private static readonly TimeSpan FSharpCheckerSettleDelay = TimeSpan.FromSeconds(2); + [IdeFact] public async Task GoesToDefinition() { @@ -28,7 +37,8 @@ module Test await SolutionExplorer.CreateSingleProjectSolutionAsync("Library", template, TestToken); await SolutionExplorer.RestoreNuGetPackagesAsync(TestToken); await Editor.SetTextAsync(code, TestToken); - + await Task.Delay(FSharpCheckerSettleDelay, TestToken); + await Editor.PlaceCaretAsync("add 1", TestToken); await Shell.ExecuteCommandAsync(VSStd97CmdID.GotoDefn, TestToken); var actualText = await Editor.GetCurrentLineTextAsync(TestToken); @@ -72,6 +82,7 @@ let id (t: SomeType) = t await SolutionExplorer.BuildSolutionAsync(TestToken); await SolutionExplorer.OpenFileAsync("Library", "Module.fsi", TestToken); + await Task.Delay(FSharpCheckerSettleDelay, TestToken); await Editor.PlaceCaretAsync("SomeType ->", TestToken); await Shell.ExecuteCommandAsync(VSStd97CmdID.GotoDefn, TestToken); var expectedText = "type SomeType ="; @@ -82,6 +93,7 @@ let id (t: SomeType) = t Assert.Equal(expectedWindow, actualWindow); await SolutionExplorer.OpenFileAsync("Library", "Module.fs", TestToken); + await Task.Delay(FSharpCheckerSettleDelay, TestToken); await Editor.PlaceCaretAsync("SomeType)", TestToken); await Shell.ExecuteCommandAsync(VSStd97CmdID.GotoDefn, TestToken); expectedText = "type SomeType ="; diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/EditorInProcess.cs b/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/EditorInProcess.cs index 2e9009ba9c6..2dbde4de1d2 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/EditorInProcess.cs +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/EditorInProcess.cs @@ -79,16 +79,35 @@ public async Task> InvokeCodeActionListAsync(Can var shell = await GetRequiredGlobalServiceAsync(cancellationToken); var cmdGroup = typeof(VSConstants.VSStd14CmdID).GUID; - var cmdExecOpt = OLECMDEXECOPT.OLECMDEXECOPT_DONTPROMPTUSER; - var cmdID = VSConstants.VSStd14CmdID.ShowQuickFixes; + + // Post ShowQuickFixes once. PostExecCommand is fire-and-forget; the broker spins up the + // session asynchronously after F# diagnostics surface, so we have to wait for the session + // to materialize before LightBulbHelper.WaitForItemsAsync can subscribe to its events. + // Repeated PostExecCommand calls would dismiss any in-flight session and reset the race. object? obj = null; - shell.PostExecCommand(cmdGroup, (uint)cmdID, (uint)cmdExecOpt, ref obj); + shell.PostExecCommand(cmdGroup, (uint)cmdID, (uint)OLECMDEXECOPT.OLECMDEXECOPT_DONTPROMPTUSER, ref obj); var view = await GetActiveTextViewAsync(cancellationToken); var broker = await GetComponentModelServiceAsync(cancellationToken); - var lightbulbs = await LightBulbHelper.WaitForItemsAsync(broker, view, cancellationToken); - return lightbulbs; + // Poll for the session to appear (up to ~5 s). Without this wait, the F#-analyzer-driven + // code fixes (UnusedOpenDeclarations, AddMissingFunKeyword) NRE because broker.GetSession + // returns null before the lightbulb session is ready -- LightBulbHelper.WaitForItemsAsync + // then casts null to IAsyncLightBulbSession and subscribes to its events. + var sessionTimeout = TimeSpan.FromSeconds(5); + var sw = System.Diagnostics.Stopwatch.StartNew(); + while (broker.GetSession(view) is null) + { + if (sw.Elapsed > sessionTimeout) + { + // Final attempt -- let LightBulbHelper's existing NRE propagate so the test failure + // points at the actual root cause, not a manufactured timeout exception. + break; + } + await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken); + } + + return await LightBulbHelper.WaitForItemsAsync(broker, view, cancellationToken); } } diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/ShellInProcess_Debugging.cs b/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/ShellInProcess_Debugging.cs index 2dac220006f..d123851d366 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/ShellInProcess_Debugging.cs +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/ShellInProcess_Debugging.cs @@ -84,10 +84,12 @@ public async Task WaitForBreakpointHitAsync(TimeSpan timeout, bool continueOnSte dbg.Go(true); if (seenRun && mode == EnvDTE.dbgDebugMode.dbgDesignMode) - throw new InvalidOperationException("Debug session ended before breakpoint hit."); + throw new InvalidOperationException( + $"Debug session ended before breakpoint hit. Breakpoints={BreakpointSummary(dbg)}."); if (sw.Elapsed > timeout) - throw new TimeoutException($"No breakpoint hit after {timeout}. Mode={mode}; Breakpoints={BreakpointSummary(dbg)}."); + throw new TimeoutException( + $"No breakpoint hit after {timeout}. Mode={mode}; Breakpoints={BreakpointSummary(dbg)}."); await Task.Delay(100, cancellationToken); } @@ -122,7 +124,11 @@ private static string BreakpointSummary(EnvDTE.Debugger dbg) { var children = 0; foreach (EnvDTE.Breakpoint _ in bp.Children) children++; - items.Add($"{Path.GetFileName(bp.File)}:{bp.FileLine}(children={children})"); + // CurrentHits + Children counts together let us distinguish unbound (children=0, hits=0) + // from "bound but never executed" (children>0, hits=0) from "bound and hit" (hits>0). + // Unbound means the PDB has no IL location for this source line; the disk-source hash + // didn't match the PDB's recorded hash; or the loaded module doesn't have this code. + items.Add($"{Path.GetFileName(bp.File)}:{bp.FileLine}(children={children},hits={bp.CurrentHits})"); } return items.Count == 0 ? "none" : string.Join(",", items); } diff --git a/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/SolutionExplorerInProcess.cs b/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/SolutionExplorerInProcess.cs index 43743484e96..a0a08c4d9fa 100644 --- a/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/SolutionExplorerInProcess.cs +++ b/vsintegration/tests/FSharp.Editor.IntegrationTests/InProcess/SolutionExplorerInProcess.cs @@ -4,9 +4,10 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.IO; using System.Linq; -using System.Runtime.CompilerServices; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.VisualStudio.OperationProgress; @@ -50,17 +51,47 @@ public async Task CreateSingleProjectSolutionAsync(string name, string template, await AddProjectAsync(name, template, cancellationToken); } - // Repo root from compile-time source path — no runtime resolution needed. - private static readonly string RepoRoot = Path.GetFullPath(Path.Combine( - Path.GetDirectoryName(GetSourceFilePath())!, "..", "..", "..", "..")); + // RepoRoot, LocalCompilerConfiguration: derived at runtime from the test assembly's location, + // NOT from [CallerFilePath]. Arcade builds with deterministic source-root mapping that rewrites + // CallerFilePath to "/_/..." (for symbol-server reproducibility); using it at runtime gives + // "D:\_" on Windows CI agents, which then breaks Process.Start(WorkingDirectory) and fsc.dll + // path resolution. Assembly.Location IS the real post-build path on disk in all environments. + // + // Layout: /artifacts/bin/FSharp.Editor.IntegrationTests///FSharp.Editor.IntegrationTests.dll + private static readonly string AssemblyDir = + Path.GetDirectoryName(typeof(SolutionExplorerInProcess).Assembly.Location)!; + + private static readonly string LocalCompilerConfiguration = + new DirectoryInfo(AssemblyDir).Parent!.Name; + + private static readonly string RepoRoot = Path.GetFullPath( + Path.Combine(AssemblyDir, "..", "..", "..", "..", "..")); + + // Repo-pinned dotnet host installed by Arcade. Falls back to PATH lookup if not present + // (developer scenarios that build the integration project outside the repo's eng infra). + private static readonly string DotnetExe = + File.Exists(Path.Combine(RepoRoot, ".dotnet", "dotnet.exe")) + ? Path.Combine(RepoRoot, ".dotnet", "dotnet.exe") + : "dotnet"; private static string CreateStandaloneProjectFile() { var propsPath = Path.Combine(RepoRoot, "UseLocalCompiler.Directory.Build.props"); + // Sanity-check the inferred configuration: matching fsc must exist before VS tries to build. + // Validated here (not in the static initializer) so tests that don't use the standalone path + // can still load this type when fsc hasn't been built locally. + var fscRoot = Path.Combine(RepoRoot, "artifacts", "bin", "fsc", LocalCompilerConfiguration); + if (!Directory.Exists(fscRoot)) + { + throw new InvalidOperationException( + $"Inferred LocalCompilerConfiguration='{LocalCompilerConfiguration}' but no built fsc found at '{fscRoot}'. " + + $"The synthesized standalone fsproj would fail to load fsc.dll -- build the F# compiler in this configuration first."); + } + return $@" - Debug + {LocalCompilerConfiguration} {RepoRoot} @@ -74,9 +105,6 @@ private static string CreateStandaloneProjectFile() "; } - private static string GetSourceFilePath([CallerFilePath] string path = "") - => path; - public async Task CreateSolutionAsync(string solutionName, CancellationToken cancellationToken) { await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken); @@ -115,22 +143,108 @@ private async Task GetDirectoryNameAsync(CancellationToken cancellationT public async Task AddProjectAsync(string projectName, string projectTemplate, CancellationToken cancellationToken) { - await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken); + // dotnet new must run off the UI thread (it's a synchronous shell-out). + var solutionDir = await GetDirectoryNameAsync(cancellationToken); + var projectDir = Path.Combine(solutionDir, projectName); - var projectPath = Path.Combine(await GetDirectoryNameAsync(cancellationToken), projectName); - var projectTemplatePath = await GetProjectTemplatePathAsync(projectTemplate, cancellationToken); - var solution = await GetRequiredGlobalServiceAsync(cancellationToken); - ErrorHandler.ThrowOnFailure(solution.AddNewProjectFromTemplate(projectTemplatePath, null, null, projectPath, projectName, null, out _)); - } + await TaskScheduler.Default; + await RunDotnetNewAsync(projectTemplate, projectName, projectDir, cancellationToken); - private async Task GetProjectTemplatePathAsync(string projectTemplate, CancellationToken cancellationToken) - { - await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken); + var projectFilePath = Path.Combine(projectDir, $"{projectName}.fsproj"); + if (!File.Exists(projectFilePath)) + { + throw new InvalidOperationException( + $"'dotnet new {projectTemplate}' completed but did not produce '{projectFilePath}'."); + } + await JoinableTaskFactory.SwitchToMainThreadAsync(cancellationToken); var dte = await GetRequiredGlobalServiceAsync(cancellationToken); var solution = (EnvDTE80.Solution2)dte.Solution; + _ = solution.AddFromFile(projectFilePath, false); + + // Auto-open the project's main .fs file. The previous AddNewProjectFromTemplate path opened + // the file marked OpenInEditor="true" in the .vstemplate; tests rely on this implicit open + // (e.g. CodeActionTests calls Editor.SetTextAsync immediately afterwards). + // SDK templates currently produce exactly one .fs file per project (Library.fs / Program.fs / Tests.fs). + // If that ever changes (e.g. xunit template growing a Program.fs), fail loudly rather than + // silently opening the wrong file and producing confusing content-diff failures downstream. + var fsFiles = Directory.EnumerateFiles(projectDir, "*.fs", SearchOption.TopDirectoryOnly) + .OrderBy(p => p, StringComparer.OrdinalIgnoreCase) + .ToList(); + if (fsFiles.Count != 1) + { + throw new InvalidOperationException( + $"Expected exactly one *.fs file in '{projectDir}' produced by 'dotnet new {projectTemplate}', " + + $"found {fsFiles.Count}: [{string.Join(", ", fsFiles.Select(Path.GetFileName))}]. " + + $"Update AddProjectAsync's auto-open logic to pick the correct main file for this template."); + } + await OpenFileAsync(projectName, Path.GetFileName(fsFiles[0]), cancellationToken); + } - return solution.GetProjectTemplate(projectTemplate, "FSharp"); + // Shells out to `dotnet new