Merge branch 'main' into subprocess-fast-wait

Author: giampaolo

.github/CODEOWNERS

@@ -176,6 +176,7 @@ Tools/wasm/config.site-wasm32-emscripten  @freakboy3742 @emmatyping
 Tools/wasm/emscripten                     @freakboy3742 @emmatyping
 
 # WebAssembly (WASI)
+Platforms/WASI                @brettcannon @emmatyping @savannahostrowski
 Tools/wasm/wasi-env           @brettcannon @emmatyping @savannahostrowski
 Tools/wasm/wasi.py            @brettcannon @emmatyping @savannahostrowski
 Tools/wasm/wasi               @brettcannon @emmatyping @savannahostrowski

.github/workflows/reusable-wasi.yml

@@ -47,14 +47,14 @@ jobs:
     - name: "Runner image version"
       run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
     - name: "Configure build Python"
-      run: python3 Tools/wasm/wasi configure-build-python -- --config-cache --with-pydebug
+      run: python3 Platforms/WASI configure-build-python -- --config-cache --with-pydebug
     - name: "Make build Python"
-      run: python3 Tools/wasm/wasi make-build-python
+      run: python3 Platforms/WASI make-build-python
     - name: "Configure host"
       # `--with-pydebug` inferred from configure-build-python
-      run: python3 Tools/wasm/wasi configure-host -- --config-cache
+      run: python3 Platforms/WASI configure-host -- --config-cache
     - name: "Make host"
-      run: python3 Tools/wasm/wasi make-host
+      run: python3 Platforms/WASI make-host
     - name: "Display build info"
       run: make --directory "${CROSS_BUILD_WASI}" pythoninfo
     - name: "Test"

Doc/c-api/conversion.rst

@@ -130,6 +130,8 @@ The following functions provide locale-independent string to number conversions.
 
    *flags* can be zero or more of the following values or-ed together:
 
+   .. c:namespace:: NULL
+
    .. c:macro:: Py_DTSF_SIGN
 
       Always precede the returned string with a sign
@@ -151,9 +153,21 @@ The following functions provide locale-independent string to number conversions.
 
       .. versionadded:: 3.11
 
-   If *ptype* is non-``NULL``, then the value it points to will be set to one of
-   ``Py_DTST_FINITE``, ``Py_DTST_INFINITE``, or ``Py_DTST_NAN``, signifying that
-   *val* is a finite number, an infinite number, or not a number, respectively.
+   If *ptype* is non-``NULL``, then the value it points to will be set to one
+   of the following constants depending on the type of *val*:
+
+   .. list-table::
+      :header-rows: 1
+      :align: left
+
+      * - *\*ptype*
+        - type of *val*
+      * - .. c:macro:: Py_DTST_FINITE
+        - finite number
+      * - .. c:macro:: Py_DTST_INFINITE
+        - infinite number
+      * - .. c:macro:: Py_DTST_NAN
+        - not a number
 
    The return value is a pointer to *buffer* with the converted string or
    ``NULL`` if the conversion failed. The caller is responsible for freeing the

Doc/howto/remote_debugging.rst

@@ -8,6 +8,16 @@ execute Python code remotely.
 
 Most platforms require elevated privileges to attach to another Python process.
 
+Disabling remote debugging
+--------------------------
+
+To disable remote debugging support, use any of the following:
+
+* Set the :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` environment variable to ``1`` before
+  starting the interpreter.
+* Use the :option:`-X disable_remote_debug` command-line option.
+* Compile Python with the :option:`--without-remote-debug` build flag.
+
 .. _permission-requirements:
 
 Permission requirements
@@ -614,4 +624,3 @@ To inject and execute a Python script in a remote process:
 6. Set ``_PY_EVAL_PLEASE_STOP_BIT`` in the ``eval_breaker`` field.
 7. Resume the process (if suspended). The script will execute at the next safe
    evaluation point.
-

Doc/library/sys.rst

@@ -1997,6 +1997,9 @@ always available. Unless explicitly noted otherwise, all variables are read-only
    interpreter is pre-release (alpha, beta, or release candidate) then the
    local and remote interpreters must be the same exact version.
 
+   See :ref:`remote-debugging` for more information about the remote debugging
+   mechanism.
+
    .. audit-event:: sys.remote_exec pid script_path
 
       When the code is executed in the remote process, an
@@ -2015,6 +2018,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only
 
    .. availability:: Unix, Windows.
    .. versionadded:: 3.14
+      See :pep:`768` for more details.
 
 
 .. function:: _enablelegacywindowsfsencoding()

Include/internal/pycore_optimizer.h

@@ -16,12 +16,102 @@ extern "C" {
 #include <stdbool.h>
 
 
+typedef struct _PyJitUopBuffer {
+    _PyUOpInstruction *start;
+    _PyUOpInstruction *next;
+    _PyUOpInstruction *end;
+} _PyJitUopBuffer;
+
+
+typedef struct _JitOptContext {
+    char done;
+    char out_of_space;
+    bool contradiction;
+    // Has the builtins dict been watched?
+    bool builtins_watched;
+    // The current "executing" frame.
+    _Py_UOpsAbstractFrame *frame;
+    _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH];
+    int curr_frame_depth;
+
+    // Arena for the symbolic types.
+    ty_arena t_arena;
+
+    JitOptRef *n_consumed;
+    JitOptRef *limit;
+    JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE];
+    _PyJitUopBuffer out_buffer;
+} JitOptContext;
+
+
+static inline void
+uop_buffer_init(_PyJitUopBuffer *trace, _PyUOpInstruction *start, uint32_t size)
+{
+    trace->next = trace->start = start;
+    trace->end = start + size;
+}
+
+static inline _PyUOpInstruction *
+uop_buffer_last(_PyJitUopBuffer *trace)
+{
+    assert(trace->next > trace->start);
+    return trace->next-1;
+}
+
+static inline int
+uop_buffer_length(_PyJitUopBuffer *trace)
+{
+    return (int)(trace->next - trace->start);
+}
+
+static inline int
+uop_buffer_remaining_space(_PyJitUopBuffer *trace)
+{
+    return (int)(trace->end - trace->next);
+}
+
+typedef struct _PyJitTracerInitialState {
+    int stack_depth;
+    int chain_depth;
+    struct _PyExitData *exit;
+    PyCodeObject *code; // Strong
+    PyFunctionObject *func; // Strong
+    struct _PyExecutorObject *executor; // Strong
+    _Py_CODEUNIT *start_instr;
+    _Py_CODEUNIT *close_loop_instr;
+    _Py_CODEUNIT *jump_backward_instr;
+} _PyJitTracerInitialState;
+
+typedef struct _PyJitTracerPreviousState {
+    bool dependencies_still_valid;
+    int instr_oparg;
+    int instr_stacklevel;
+    _Py_CODEUNIT *instr;
+    PyCodeObject *instr_code; // Strong
+    struct _PyInterpreterFrame *instr_frame;
+    _PyBloomFilter dependencies;
+} _PyJitTracerPreviousState;
+
+typedef struct _PyJitTracerTranslatorState {
+    int jump_backward_seen;
+} _PyJitTracerTranslatorState;
+
+typedef struct _PyJitTracerState {
+    bool is_tracing;
+    _PyJitTracerInitialState initial_state;
+    _PyJitTracerPreviousState prev_state;
+    _PyJitTracerTranslatorState translator_state;
+    JitOptContext opt_context;
+    _PyJitUopBuffer code_buffer;
+    _PyJitUopBuffer out_buffer;
+    _PyUOpInstruction uop_array[2 * UOP_MAX_TRACE_LENGTH];
+} _PyJitTracerState;
+
 typedef struct _PyExecutorLinkListNode {
     struct _PyExecutorObject *next;
     struct _PyExecutorObject *previous;
 } _PyExecutorLinkListNode;
 
-
 typedef struct {
     uint8_t opcode;
     uint8_t oparg;
@@ -86,8 +176,8 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp);
 
 int _Py_uop_analyze_and_optimize(
     _PyThreadStateImpl *tstate,
-    _PyUOpInstruction *trace, int trace_len, int curr_stackentries,
-    _PyBloomFilter *dependencies);
+    _PyUOpInstruction *input, int trace_len, int curr_stackentries,
+    _PyUOpInstruction *output, _PyBloomFilter *dependencies);
 
 extern PyTypeObject _PyUOpExecutor_Type;
 
@@ -205,6 +295,8 @@ extern JitOptRef _Py_uop_sym_new_truthiness(JitOptContext *ctx, JitOptRef value,
 extern bool _Py_uop_sym_is_compact_int(JitOptRef sym);
 extern JitOptRef _Py_uop_sym_new_compact_int(JitOptContext *ctx);
 extern void _Py_uop_sym_set_compact_int(JitOptContext *ctx,  JitOptRef sym);
+extern JitOptRef _Py_uop_sym_new_predicate(JitOptContext *ctx, JitOptRef lhs_ref, JitOptRef rhs_ref, JitOptPredicateKind kind);
+extern void _Py_uop_sym_apply_predicate_narrowing(JitOptContext *ctx, JitOptRef sym, bool branch_is_true);
 
 extern void _Py_uop_abstractcontext_init(JitOptContext *ctx);
 extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx);

Include/internal/pycore_optimizer_types.h

@@ -40,6 +40,7 @@ typedef enum _JitSymType {
     JIT_SYM_TUPLE_TAG = 8,
     JIT_SYM_TRUTHINESS_TAG = 9,
     JIT_SYM_COMPACT_INT = 10,
+    JIT_SYM_PREDICATE_TAG = 11,
 } JitSymType;
 
 typedef struct _jit_opt_known_class {
@@ -72,6 +73,18 @@ typedef struct {
     uint16_t value;
 } JitOptTruthiness;
 
+typedef enum {
+    JIT_PRED_IS,
+    JIT_PRED_IS_NOT,
+} JitOptPredicateKind;
+
+typedef struct {
+    uint8_t tag;
+    uint8_t kind;
+    uint16_t lhs;
+    uint16_t rhs;
+} JitOptPredicate;
+
 typedef struct {
     uint8_t tag;
 } JitOptCompactInt;
@@ -84,6 +97,7 @@ typedef union _jit_opt_symbol {
     JitOptTuple tuple;
     JitOptTruthiness truthiness;
     JitOptCompactInt compact;
+    JitOptPredicate predicate;
 } JitOptSymbol;
 
 // This mimics the _PyStackRef API
@@ -112,27 +126,6 @@ typedef struct ty_arena {
     JitOptSymbol arena[TY_ARENA_SIZE];
 } ty_arena;
 
-typedef struct _JitOptContext {
-    char done;
-    char out_of_space;
-    bool contradiction;
-    // Has the builtins dict been watched?
-    bool builtins_watched;
-    // The current "executing" frame.
-    _Py_UOpsAbstractFrame *frame;
-    _Py_UOpsAbstractFrame frames[MAX_ABSTRACT_FRAME_DEPTH];
-    int curr_frame_depth;
-
-    // Arena for the symbolic types.
-    ty_arena t_arena;
-
-    JitOptRef *n_consumed;
-    JitOptRef *limit;
-    JitOptRef locals_and_stack[MAX_ABSTRACT_INTERP_SIZE];
-    _PyUOpInstruction *out_buffer;
-    int out_len;
-} JitOptContext;
-
 
 #ifdef __cplusplus
 }

Include/internal/pycore_tstate.h

@@ -12,7 +12,6 @@ extern "C" {
 #include "pycore_freelist_state.h"  // struct _Py_freelists
 #include "pycore_interpframe_structs.h"  // _PyInterpreterFrame
 #include "pycore_mimalloc.h"        // struct _mimalloc_thread_state
-#include "pycore_optimizer_types.h" // JitOptContext
 #include "pycore_qsbr.h"            // struct qsbr
 #include "pycore_uop.h"             // struct _PyUOpInstruction
 #include "pycore_structs.h"
@@ -24,46 +23,6 @@ struct _gc_thread_state {
 };
 #endif
 
-#if _Py_TIER2
-typedef struct _PyJitTracerInitialState {
-    int stack_depth;
-    int chain_depth;
-    struct _PyExitData *exit;
-    PyCodeObject *code; // Strong
-    PyFunctionObject *func; // Strong
-    struct _PyExecutorObject *executor; // Strong
-    _Py_CODEUNIT *start_instr;
-    _Py_CODEUNIT *close_loop_instr;
-    _Py_CODEUNIT *jump_backward_instr;
-} _PyJitTracerInitialState;
-
-typedef struct _PyJitTracerPreviousState {
-    bool dependencies_still_valid;
-    int code_max_size;
-    int code_curr_size;
-    int instr_oparg;
-    int instr_stacklevel;
-    _Py_CODEUNIT *instr;
-    PyCodeObject *instr_code; // Strong
-    struct _PyInterpreterFrame *instr_frame;
-    _PyBloomFilter dependencies;
-} _PyJitTracerPreviousState;
-
-typedef struct _PyJitTracerTranslatorState {
-    int jump_backward_seen;
-} _PyJitTracerTranslatorState;
-
-typedef struct _PyJitTracerState {
-    bool is_tracing;
-    _PyJitTracerInitialState initial_state;
-    _PyJitTracerPreviousState prev_state;
-    _PyJitTracerTranslatorState translator_state;
-    JitOptContext opt_context;
-    _PyUOpInstruction code_buffer[UOP_MAX_TRACE_LENGTH];
-    _PyUOpInstruction out_buffer[UOP_MAX_TRACE_LENGTH];
-} _PyJitTracerState;
-
-#endif
 
 // Every PyThreadState is actually allocated as a _PyThreadStateImpl. The
 // PyThreadState fields are exposed as part of the C API, although most fields
@@ -141,7 +100,7 @@ typedef struct _PyThreadStateImpl {
     Py_ssize_t reftotal;  // this thread's total refcount operations
 #endif
 #if _Py_TIER2
-    _PyJitTracerState *jit_tracer_state;
+    struct _PyJitTracerState *jit_tracer_state;
 #endif
 } _PyThreadStateImpl;
 

Include/internal/pycore_uop.h

@@ -38,11 +38,10 @@ typedef struct _PyUOpInstruction{
 // This is the length of the trace we translate initially.
 #ifdef Py_DEBUG
     // With asserts, the stencils are a lot larger
-#define UOP_MAX_TRACE_LENGTH 2000
+#define UOP_MAX_TRACE_LENGTH 1000
 #else
-#define UOP_MAX_TRACE_LENGTH 5000
+#define UOP_MAX_TRACE_LENGTH 2500
 #endif
-#define UOP_BUFFER_SIZE (UOP_MAX_TRACE_LENGTH * sizeof(_PyUOpInstruction))
 
 /* Bloom filter with m = 256
  * https://en.wikipedia.org/wiki/Bloom_filter */