Merge remote-tracking branch 'upstream/main' into jit-refcount-call-type

Author: tomasr8

.gitattributes

@@ -83,8 +83,10 @@ Include/opcode.h                                    generated
 Include/opcode_ids.h                                generated
 Include/token.h                                     generated
 Lib/_opcode_metadata.py                             generated
-Lib/keyword.py                                      generated
 Lib/idlelib/help.html                               generated
+Lib/keyword.py                                      generated
+Lib/pydoc_data/topics.py                            generated
+Lib/pydoc_data/module_docs.py                       generated
 Lib/test/certdata/*.pem                             generated
 Lib/test/certdata/*.0                               generated
 Lib/test/levenshtein_examples.json                  generated

.github/workflows/tail-call.yml

@@ -79,19 +79,17 @@ jobs:
         with:
           python-version: '3.11'
 
-      - name: Native Windows (debug)
+      - name: Native Windows MSVC (release)
         if: runner.os == 'Windows' && matrix.architecture != 'ARM64'
         shell: cmd
         run: |
-          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0
-          set PlatformToolset=clangcl
-          set LLVMToolsVersion=${{ matrix.llvm }}.1.0
-          set LLVMInstallDir=C:\Program Files\LLVM
-          call ./PCbuild/build.bat --tail-call-interp -d -p ${{ matrix.architecture }}
-          call ./PCbuild/rt.bat -d -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+          choco install visualstudio2026buildtools --no-progress -y --force --params "--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64  --locale en-US --passive"
+          $env:PATH = "C:\Program Files (x86)\Microsoft Visual Studio\18\BuildTools\MSBuild\Current\bin;$env:PATH"
+          ./PCbuild/build.bat --tail-call-interp -c Release -p ${{ matrix.architecture }} "/p:PlatformToolset=v145"
+          ./PCbuild/rt.bat -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
       # No tests (yet):
-      - name: Emulated Windows (release)
+      - name: Emulated Windows Clang (release)
         if: runner.os == 'Windows' && matrix.architecture == 'ARM64'
         shell: cmd
         run: |

.pre-commit-config.yaml

@@ -40,15 +40,15 @@ repos:
         files: ^Apple
       - id: ruff-format
         name: Run Ruff (format) on Doc/
-        args: [--check]
+        args: [--exit-non-zero-on-fix]
         files: ^Doc/
       - id: ruff-format
         name: Run Ruff (format) on Tools/build/check_warnings.py
-        args: [--check, --config=Tools/build/.ruff.toml]
+        args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml]
         files: ^Tools/build/check_warnings.py
       - id: ruff-format
         name: Run Ruff (format) on Tools/wasm/
-        args: [--check, --config=Tools/wasm/.ruff.toml]
+        args: [--exit-non-zero-on-fix, --config=Tools/wasm/.ruff.toml]
         files: ^Tools/wasm/
 
   - repo: https://github.com/psf/black-pre-commit-mirror

Doc/Makefile

@@ -140,7 +140,8 @@ doctest:
 pydoc-topics: BUILDER = pydoc-topics
 pydoc-topics: build
 	@echo "Building finished; now run this:" \
-	      "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py"
+	      "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py" \
+	      "&& cp build/pydoc-topics/module_docs.py ../Lib/pydoc_data/module_docs.py"
 
 .PHONY: gettext
 gettext: BUILDER = gettext

Doc/library/profiling.sampling.rst

@@ -200,6 +200,36 @@ On most systems, attaching to another process requires appropriate permissions.
 See :ref:`profiling-permissions` for platform-specific requirements.
 
 
+.. _replay-command:
+
+The ``replay`` command
+----------------------
+
+The ``replay`` command converts binary profile files to other output formats::
+
+   python -m profiling.sampling replay profile.bin
+   python -m profiling.sampling replay --flamegraph -o profile.html profile.bin
+
+This command is useful when you have captured profiling data in binary format
+and want to analyze it later or convert it to a visualization format. Binary
+profiles can be replayed multiple times to different formats without
+re-profiling.
+
+::
+
+   # Convert binary to pstats (default, prints to stdout)
+   python -m profiling.sampling replay profile.bin
+
+   # Convert binary to flame graph
+   python -m profiling.sampling replay --flamegraph -o output.html profile.bin
+
+   # Convert binary to gecko format for Firefox Profiler
+   python -m profiling.sampling replay --gecko -o profile.json profile.bin
+
+   # Convert binary to heatmap
+   python -m profiling.sampling replay --heatmap -o my_heatmap profile.bin
+
+
 Profiling in production
 -----------------------
 
@@ -1041,6 +1071,59 @@ intuitive view that shows exactly where time is spent without requiring
 interpretation of hierarchical visualizations.
 
 
+Binary format
+-------------
+
+Binary format (:option:`--binary`) produces a compact binary file for efficient
+storage of profiling data::
+
+   python -m profiling.sampling run --binary -o profile.bin script.py
+   python -m profiling.sampling attach --binary -o profile.bin 12345
+
+The :option:`--compression` option controls data compression:
+
+- ``auto`` (default): Use zstd compression if available, otherwise no
+  compression
+- ``zstd``: Force zstd compression (requires :mod:`compression.zstd` support)
+- ``none``: Disable compression
+
+::
+
+   python -m profiling.sampling run --binary --compression=zstd -o profile.bin script.py
+
+To analyze binary profiles, use the :ref:`replay-command` to convert them to
+other formats like flame graphs or pstats output.
+
+
+Record and replay workflow
+==========================
+
+The binary format combined with the replay command enables a record-and-replay
+workflow that separates data capture from analysis. Rather than generating
+visualizations during profiling, you capture raw data to a compact binary file
+and convert it to different formats later.
+
+This approach has three main benefits:
+
+- Sampling runs faster because the work of building data structures for
+  visualization is deferred until replay.
+- A single binary capture can be converted to multiple output formats
+  without re-profiling: pstats for a quick overview, flame graph for visual
+  exploration, heatmap for line-level detail.
+- Binary files are compact and easy to share with colleagues who can convert
+  them to their preferred format.
+
+A typical workflow::
+
+   # Capture profile in production or during tests
+   python -m profiling.sampling attach --binary -o profile.bin 12345
+
+   # Later, analyze with different formats
+   python -m profiling.sampling replay profile.bin
+   python -m profiling.sampling replay --flamegraph -o profile.html profile.bin
+   python -m profiling.sampling replay --heatmap -o heatmap profile.bin
+
+
 Live mode
 =========
 
@@ -1252,6 +1335,10 @@ Global options
 
    Attach to and profile a running process by PID.
 
+.. option:: replay
+
+   Convert a binary profile file to another output format.
+
 
 Sampling options
 ----------------
@@ -1335,12 +1422,22 @@ Output options
 
    Generate HTML heatmap with line-level sample counts.
 
+.. option:: --binary
+
+   Generate high-performance binary format for later conversion with the
+   ``replay`` command.
+
+.. option:: --compression <type>
+
+   Compression for binary format: ``auto`` (use zstd if available, default),
+   ``zstd``, or ``none``.
+
 .. option:: -o <path>, --output <path>
 
    Output file or directory path. Default behavior varies by format:
-   ``--pstats`` writes to stdout, ``--flamegraph`` and ``--gecko`` generate
-   files like ``flamegraph.PID.html``, and ``--heatmap`` creates a directory
-   named ``heatmap_PID``.
+   :option:`--pstats` writes to stdout, while other formats generate a file
+   named ``<format>_<PID>.<ext>`` (for example, ``flamegraph_12345.html``).
+   :option:`--heatmap` creates a directory named ``heatmap_<PID>``.
 
 
 pstats display options

Doc/tools/extensions/pydoc_topics.py

@@ -109,6 +109,7 @@ class PydocTopicsBuilder(TextBuilder):
     def init(self) -> None:
         super().init()
         self.topics: dict[str, str] = {}
+        self.module_docs: dict[str, str] = {}
 
     def get_outdated_docs(self) -> str:
         # Return a string describing what an update build will build.
@@ -130,6 +131,15 @@ def write_documents(self, _docnames: Set[str]) -> None:
                 continue
             doc_labels.setdefault(docname, []).append((topic_label, label_id))
 
+        py_domain = env.domains['py']
+        for module_name, module_info in py_domain.data['modules'].items():
+            docname = module_info[0]
+            if docname.startswith('library/'):
+                doc_file = docname.replace('library/', '')
+                self.module_docs[module_name] = (
+                    f"{doc_file}#module-{module_name}"
+                )
+
         for docname, label_ids in status_iterator(
             doc_labels.items(),
             "building topics... ",
@@ -161,6 +171,22 @@ def finish(self) -> None:
 """
         self.outdir.joinpath("topics.py").write_text(topics, encoding="utf-8")
 
+        module_docs_repr = "\n".join(
+            f"    '{module}': '{doc_file}',"
+            for module, doc_file in sorted(self.module_docs.items())
+        )
+        module_docs = f"""\
+# Autogenerated by Sphinx on {asctime()}
+# as part of the release process.
+
+module_docs = {{
+{module_docs_repr}
+}}
+"""
+        self.outdir.joinpath("module_docs.py").write_text(
+            module_docs, encoding="utf-8"
+        )
+
 
 def _display_labels(item: tuple[str, Sequence[tuple[str, str]]]) -> str:
     _docname, label_ids = item

Doc/whatsnew/3.15.rst

@@ -849,6 +849,16 @@ zlib
 Optimizations
 =============
 
+* Builds using Visual Studio 2026 (MSVC 18) may now use the new
+  :ref:`tail-calling interpreter <whatsnew314-tail-call-interpreter>`.
+  Results on an early experimental MSVC compiler reported roughly 15% speedup
+  on the geometric mean of pyperformance on Windows x86-64 over
+  the switch-case interpreter. We have
+  observed speedups ranging from 15% for large pure-Python libraries
+  to 40% for long-running small pure-Python scripts on Windows.
+  (Contributed by Chris Eibl, Ken Jin, and Brandt Bucher in :gh:`143068`.
+  Special thanks to the MSVC team including Hulon Jenkins.)
+
 csv
 ---
 

Include/internal/pycore_ceval.h

@@ -415,6 +415,17 @@ _Py_VectorCall_StackRefSteal(
     int total_args,
     _PyStackRef kwnames);
 
+PyAPI_FUNC(PyObject*)
+_Py_VectorCallInstrumentation_StackRefSteal(
+    _PyStackRef callable,
+    _PyStackRef* arguments,
+    int total_args,
+    _PyStackRef kwnames,
+    bool call_instrumentation,
+    _PyInterpreterFrame* frame,
+    _Py_CODEUNIT* this_instr,
+    PyThreadState* tstate);
+
 PyAPI_FUNC(PyObject *)
 _Py_BuiltinCallFast_StackRefSteal(
     _PyStackRef callable,
@@ -464,6 +475,11 @@ _Py_assert_within_stack_bounds(
     _PyInterpreterFrame *frame, _PyStackRef *stack_pointer,
     const char *filename, int lineno);
 
+// Like PyMapping_GetOptionalItem, but returns the PyObject* instead of taking
+// it as an out parameter. This helps MSVC's escape analysis when used with
+// tail calling.
+PyAPI_FUNC(PyObject*) _PyMapping_GetOptionalItem2(PyObject* obj, PyObject* key, int* err);
+
 #ifdef __cplusplus
 }
 #endif