gh-131253: free-threaded build support for pystats

Allow the --enable-pystats build option to be used with
free-threading.  For the free-threaded builds, the stats structure is
allocated per-thread and then periodically merged into a global stats
structure (on thread exit or when the reporting function is called).

Summary of changes:

* introduce _Py_tss_stats thread-local variable.  This is set
  when stats are on, replacing the _Py_stats global that's used
  in the non-free-threaded build.

* replace _Py_stats references with _PyStats_GET()

* move pystats logic from Python/specialize.c into Python/pystats.c

* add some free-threaded specific stat counters

Author: nascheme

Include/cpython/pystats.h

@@ -4,7 +4,7 @@
 //
 // - _Py_INCREF_STAT_INC() and _Py_DECREF_STAT_INC() used by Py_INCREF()
 //   and Py_DECREF().
-// - _Py_stats variable
+// - _PyStats_GET()
 //
 // Functions of the sys module:
 //
@@ -14,7 +14,7 @@
 // - sys._stats_dump()
 //
 // Python must be built with ./configure --enable-pystats to define the
-// Py_STATS macro.
+// _PyStats_GET() function.
 //
 // Define _PY_INTERPRETER macro to increment interpreter_increfs and
 // interpreter_decrefs. Otherwise, increment increfs and decrefs.
@@ -109,6 +109,15 @@ typedef struct _gc_stats {
     uint64_t objects_not_transitively_reachable;
 } GCStats;
 
+#ifdef Py_GIL_DISABLED
+// stats specific to free-threaded build
+typedef struct _ft_stats {
+    uint64_t mutex_sleeps;
+    uint64_t qsbr_polls;
+    uint64_t world_stops;
+} FTStats;
+#endif
+
 typedef struct _uop_stats {
     uint64_t execution_count;
     uint64_t miss;
@@ -173,22 +182,74 @@ typedef struct _stats {
     CallStats call_stats;
     ObjectStats object_stats;
     OptimizationStats optimization_stats;
+#ifdef Py_GIL_DISABLED
+    FTStats ft_stats;
+#endif
     RareEventStats rare_event_stats;
     GCStats *gc_stats;
 } PyStats;
 
 
+#ifdef Py_GIL_DISABLED
+
+#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE)
+extern _Py_thread_local PyStats *_Py_tss_stats;
+#endif
+
+// Export for most shared extensions, used via _PyStats_GET() static
+// inline function.
+PyAPI_FUNC(PyStats *) _PyStats_GetLocal(void);
+
+#else // !Py_GIL_DISABLED
+
 // Export for shared extensions like 'math'
 PyAPI_DATA(PyStats*) _Py_stats;
 
+#endif
+
+// Return pointer to the PyStats structure, NULL if recording is off.
+static inline PyStats*
+_PyStats_GET(void)
+{
+#ifdef Py_GIL_DISABLED
+
+#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE)
+    return _Py_tss_stats;
+#else
+    return _PyStats_GetLocal();
+#endif
+
+#else // !Py_GIL_DISABLED
+
+    return _Py_stats;
+
+#endif
+}
+
+#define _Py_STATS_EXPR(expr) \
+    do { \
+        PyStats *s = _PyStats_GET(); \
+        if (s != NULL) { \
+            s->expr; \
+        } \
+    } while (0)
+
+#define _Py_STATS_COND_EXPR(cond, expr) \
+    do { \
+        PyStats *s = _PyStats_GET(); \
+        if (s != NULL && cond) { \
+            s->expr; \
+        } \
+    } while (0)
+
 #ifdef _PY_INTERPRETER
-#  define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_increfs++; } while (0)
-#  define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_decrefs++; } while (0)
-#  define _Py_INCREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_immortal_increfs++; } while (0)
-#  define _Py_DECREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_immortal_decrefs++; } while (0)
+#  define _Py_INCREF_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_increfs++)
+#  define _Py_DECREF_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_decrefs++)
+#  define _Py_INCREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_immortal_increfs++)
+#  define _Py_DECREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.interpreter_immortal_decrefs++)
 #else
-#  define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.increfs++; } while (0)
-#  define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.decrefs++; } while (0)
-#  define _Py_INCREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.immortal_increfs++; } while (0)
-#  define _Py_DECREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.immortal_decrefs++; } while (0)
+#  define _Py_INCREF_STAT_INC() _Py_STATS_EXPR(object_stats.increfs++)
+#  define _Py_DECREF_STAT_INC() _Py_STATS_EXPR(object_stats.decrefs++)
+#  define _Py_INCREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.immortal_increfs++)
+#  define _Py_DECREF_IMMORTAL_STAT_INC() _Py_STATS_EXPR(object_stats.immortal_decrefs++)
 #endif

Include/internal/pycore_stats.h

@@ -15,39 +15,56 @@ extern "C" {
 
 #include "pycore_bitutils.h"  // _Py_bit_length
 
-#define STAT_INC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name++; } while (0)
-#define STAT_DEC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name--; } while (0)
-#define OPCODE_EXE_INC(opname) do { if (_Py_stats) _Py_stats->opcode_stats[opname].execution_count++; } while (0)
-#define CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.name++; } while (0)
-#define OBJECT_STAT_INC(name) do { if (_Py_stats) _Py_stats->object_stats.name++; } while (0)
-#define OBJECT_STAT_INC_COND(name, cond) \
-    do { if (_Py_stats && cond) _Py_stats->object_stats.name++; } while (0)
-#define EVAL_CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.eval_calls[name]++; } while (0)
-#define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \
-    do { if (_Py_stats && PyFunction_Check(callable)) _Py_stats->call_stats.eval_calls[name]++; } while (0)
-#define GC_STAT_ADD(gen, name, n) do { if (_Py_stats) _Py_stats->gc_stats[(gen)].name += (n); } while (0)
-#define OPT_STAT_INC(name) do { if (_Py_stats) _Py_stats->optimization_stats.name++; } while (0)
-#define OPT_STAT_ADD(name, n) do { if (_Py_stats) _Py_stats->optimization_stats.name += (n); } while (0)
-#define UOP_STAT_INC(opname, name) do { if (_Py_stats) { assert(opname < 512); _Py_stats->optimization_stats.opcode[opname].name++; } } while (0)
-#define UOP_PAIR_INC(uopcode, lastuop)                                              \
-    do {                                                                            \
-        if (lastuop && _Py_stats) {                                                 \
-            _Py_stats->optimization_stats.opcode[lastuop].pair_count[uopcode]++;    \
-        }                                                                           \
-        lastuop = uopcode;                                                          \
+#define STAT_INC(opname, name) _Py_STATS_EXPR(opcode_stats[opname].specialization.name++)
+#define STAT_DEC(opname, name) _Py_STATS_EXPR(opcode_stats[opname].specialization.name--)
+#define OPCODE_EXE_INC(opname) _Py_STATS_EXPR(opcode_stats[opname].execution_count++)
+#define CALL_STAT_INC(name) _Py_STATS_EXPR(call_stats.name++)
+#define OBJECT_STAT_INC(name) _Py_STATS_EXPR(object_stats.name++)
+#define OBJECT_STAT_INC_COND(name, cond) _Py_STATS_COND_EXPR(cond, object_stats.name++)
+#define EVAL_CALL_STAT_INC(name) _Py_STATS_EXPR(call_stats.eval_calls[name]++)
+#define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) _Py_STATS_COND_EXPR(PyFunction_Check(callable), call_stats.eval_calls[name]++)
+#define GC_STAT_ADD(gen, name, n) _Py_STATS_EXPR(gc_stats[(gen)].name += (n))
+#define OPT_STAT_INC(name) _Py_STATS_EXPR(optimization_stats.name++)
+#define OPT_STAT_ADD(name, n) _Py_STATS_EXPR(optimization_stats.name += (n))
+#define UOP_STAT_INC(opname, name) \
+    do { \
+        PyStats *s = _PyStats_GET(); \
+        if (s) { \
+            assert(opname < 512); \
+            s->optimization_stats.opcode[opname].name++; \
+        } \
+    } while (0)
+#define UOP_PAIR_INC(uopcode, lastuop) \
+    do { \
+        PyStats *s = _PyStats_GET(); \
+        if (lastuop && s) { \
+            s->optimization_stats.opcode[lastuop].pair_count[uopcode]++; \
+        } \
+        lastuop = uopcode; \
     } while (0)
-#define OPT_UNSUPPORTED_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.unsupported_opcode[opname]++; } while (0)
-#define OPT_ERROR_IN_OPCODE(opname) do { if (_Py_stats) _Py_stats->optimization_stats.error_in_opcode[opname]++; } while (0)
+#define OPT_UNSUPPORTED_OPCODE(opname) _Py_STATS_EXPR(optimization_stats.unsupported_opcode[opname]++)
+#define OPT_ERROR_IN_OPCODE(opname) _Py_STATS_EXPR(optimization_stats.error_in_opcode[opname]++)
 #define OPT_HIST(length, name) \
     do { \
-        if (_Py_stats) { \
+        PyStats *s = _PyStats_GET(); \
+        if (s) { \
             int bucket = _Py_bit_length(length >= 1 ? length - 1 : 0); \
             bucket = (bucket >= _Py_UOP_HIST_SIZE) ? _Py_UOP_HIST_SIZE - 1 : bucket; \
-            _Py_stats->optimization_stats.name[bucket]++; \
+            s->optimization_stats.name[bucket]++; \
         } \
     } while (0)
-#define RARE_EVENT_STAT_INC(name) do { if (_Py_stats) _Py_stats->rare_event_stats.name++; } while (0)
-#define OPCODE_DEFERRED_INC(opname) do { if (_Py_stats && opcode == opname) _Py_stats->opcode_stats[opname].specialization.deferred++; } while (0)
+#define RARE_EVENT_STAT_INC(name) _Py_STATS_EXPR(rare_event_stats.name++)
+#define OPCODE_DEFERRED_INC(opname) _Py_STATS_COND_EXPR(opcode==opname, opcode_stats[opname].specialization.deferred++)
+
+#ifdef Py_GIL_DISABLED
+#define FT_STAT_MUTEX_SLEEP_INC() _Py_STATS_EXPR(ft_stats.mutex_sleeps++)
+#define FT_STAT_QSBR_POLL_INC() _Py_STATS_EXPR(ft_stats.qsbr_polls++)
+#define FT_STAT_WORLD_STOP_INC() _Py_STATS_EXPR(ft_stats.world_stops++)
+#else
+#define FT_STAT_MUTEX_SLEEP_INC()
+#define FT_STAT_QSBR_POLL_INC()
+#define FT_STAT_WORLD_STOP_INC()
+#endif
 
 // Export for '_opcode' shared extension
 PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);
@@ -71,6 +88,9 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);
 #define OPT_HIST(length, name) ((void)0)
 #define RARE_EVENT_STAT_INC(name) ((void)0)
 #define OPCODE_DEFERRED_INC(opname) ((void)0)
+#define FT_STAT_MUTEX_SLEEP_INC()
+#define FT_STAT_QSBR_POLL_INC()
+#define FT_STAT_WORLD_STOP_INC()
 #endif  // !Py_STATS
 
 
@@ -90,6 +110,10 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);
         RARE_EVENT_INTERP_INC(interp, name); \
     } while (0); \
 
+bool _PyStats_ThreadInit(_PyThreadStateImpl *);
+void _PyStats_ThreadFini(_PyThreadStateImpl *);
+void _PyStats_Attach(_PyThreadStateImpl *);
+void _PyStats_Detach(_PyThreadStateImpl *);
 
 #ifdef __cplusplus
 }

Include/internal/pycore_tstate.h

@@ -70,8 +70,15 @@ typedef struct _PyThreadStateImpl {
 
     // When >1, code objects do not immortalize their non-string constants.
     int suppress_co_const_immortalization;
+
+#ifdef Py_STATS
+    // per-thread stats, will be merged into the _Py_stats_struct global
+    PyStats *pystats_struct; // allocated by _PyStats_ThreadInit()
+    PyStats **pystats_tss;  // pointer to tss variable
 #endif
 
+#endif // Py_GIL_DISABLED
+
 #if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED)
     Py_ssize_t reftotal;  // this thread's total refcount operations
 #endif

Makefile.pre.in

@@ -484,6 +484,7 @@ PYTHON_OBJS=	\
 		Python/pylifecycle.o \
 		Python/pymath.o \
 		Python/pystate.o \
+		Python/pystats.o \
 		Python/pythonrun.o \
 		Python/pytime.o \
 		Python/qsbr.o \

Modules/_xxtestfuzz/fuzzer.c

@@ -10,8 +10,8 @@
 
   See the source code for LLVMFuzzerTestOneInput for details. */
 
-#ifndef Py_BUILD_CORE
-#  define Py_BUILD_CORE 1
+#ifndef Py_BUILD_CORE_MODULE
+#  define Py_BUILD_CORE_MODULE 1
 #endif
 
 #include <Python.h>

PCbuild/_freeze_module.vcxproj

@@ -255,6 +255,7 @@
     <ClCompile Include="..\Python\pylifecycle.c" />
     <ClCompile Include="..\Python\pymath.c" />
     <ClCompile Include="..\Python\pystate.c" />
+    <ClCompile Include="..\Python\pystats.c" />
     <ClCompile Include="..\Python\pystrcmp.c" />
     <ClCompile Include="..\Python\pystrhex.c" />
     <ClCompile Include="..\Python\pystrtod.c" />

PCbuild/_freeze_module.vcxproj.filters

@@ -370,6 +370,9 @@
     <ClCompile Include="..\Python\pystate.c">
       <Filter>Source Files</Filter>
     </ClCompile>
+    <ClCompile Include="..\Python\pystats.c">
+      <Filter>Source Files</Filter>
+    </ClCompile>
     <ClCompile Include="..\Python\pystrcmp.c">
       <Filter>Source Files</Filter>
     </ClCompile>

PCbuild/pythoncore.vcxproj

@@ -655,6 +655,7 @@
     <ClCompile Include="..\Python\pymath.c" />
     <ClCompile Include="..\Python\pytime.c" />
     <ClCompile Include="..\Python\pystate.c" />
+    <ClCompile Include="..\Python\pystats.c" />
     <ClCompile Include="..\Python\pystrcmp.c" />
     <ClCompile Include="..\Python\pystrhex.c" />
     <ClCompile Include="..\Python\pystrtod.c" />

Python/ceval_macros.h

@@ -62,8 +62,9 @@
 #ifdef Py_STATS
 #define INSTRUCTION_STATS(op) \
     do { \
+        PyStats *s = _PyStats_GET(); \
         OPCODE_EXE_INC(op); \
-        if (_Py_stats) _Py_stats->opcode_stats[lastopcode].pair_count[op]++; \
+        if (s) s->opcode_stats[lastopcode].pair_count[op]++; \
         lastopcode = op; \
     } while (0)
 #else

Python/gc.c

@@ -2062,10 +2062,11 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason)
     _PyErr_SetRaisedException(tstate, exc);
     GC_STAT_ADD(generation, objects_collected, stats.collected);
 #ifdef Py_STATS
-    if (_Py_stats) {
+    PyStats *s = _PyStats_GET();
+    if (s) {
         GC_STAT_ADD(generation, object_visits,
-            _Py_stats->object_stats.object_visits);
-        _Py_stats->object_stats.object_visits = 0;
+            s->object_stats.object_visits);
+        s->object_stats.object_visits = 0;
     }
 #endif
     validate_spaces(gcstate);