Fix big endian

Author: pablogsal

InternalDocs/profiling_binary_format.md

@@ -5,7 +5,8 @@ profiler data. This document describes the format's structure and the
 design decisions behind it.
 
 The implementation is in
-[`Modules/_remote_debugging/binary_io.c`](../Modules/_remote_debugging/binary_io.c)
+[`Modules/_remote_debugging/binary_io_writer.c`](../Modules/_remote_debugging/binary_io_writer.c)
+and [`Modules/_remote_debugging/binary_io_reader.c`](../Modules/_remote_debugging/binary_io_reader.c),
 with declarations in
 [`Modules/_remote_debugging/binary_io.h`](../Modules/_remote_debugging/binary_io.h).
 
@@ -412,6 +413,36 @@ index-to-value mapping, not value-to-index.
 
 ## Platform Considerations
 
+### Byte Ordering and Cross-Platform Portability
+
+The binary format uses **native byte order** for all multi-byte integer
+fields when writing. However, the reader supports **cross-endian reading**:
+files written on a little-endian system (x86, ARM) can be read on a
+big-endian system (s390x, PowerPC), and vice versa.
+
+**Endianness Detection**: The magic number serves as an endianness marker.
+When read on a system with different byte order, it appears byte-swapped
+(`0x48434154` instead of `0x54414348`). The reader detects this and
+automatically byte-swaps all fixed-width integer fields during parsing.
+
+**Writer Requirements**: Fixed-width integer fields must be written using
+`memcpy()` from properly-sized integer types. When the source variable's
+type differs from the field width (e.g., `size_t` being written as 4 bytes),
+explicit casting to the correct type (e.g., `uint32_t`) is required before
+`memcpy()`. On big-endian systems, copying from an oversized type would
+copy the wrong bytes (high-order zeros instead of the actual value).
+
+**Reader Implementation**: The reader tracks whether byte-swapping is needed
+via a `needs_swap` flag set during header parsing. All fixed-width fields
+in the header, footer, and sample data are conditionally byte-swapped using
+inline swap functions (`bswap32`, `bswap64`).
+
+Variable-length integers (varints) are byte-order independent since they
+encode values one byte at a time using the LEB128 scheme, so they require
+no special handling for cross-endian reading.
+
+### Memory-Mapped I/O
+
 On Unix systems (Linux, macOS), the reader uses `mmap()` to map the file
 into the process address space. The kernel handles paging data in and out
 as needed, no explicit read() calls or buffer management are required,

Modules/_remote_debugging/binary_io.h

@@ -21,9 +21,16 @@ extern "C" {
  * BINARY FORMAT CONSTANTS
  * ============================================================================ */
 
-#define BINARY_FORMAT_MAGIC     0x54414348  /* "TACH" (Tachyon) */
+#define BINARY_FORMAT_MAGIC     0x54414348  /* "TACH" (Tachyon) in native byte order */
+#define BINARY_FORMAT_MAGIC_SWAPPED 0x48434154  /* Byte-swapped magic for endianness detection */
 #define BINARY_FORMAT_VERSION   1
 
+/* Conditional byte-swap macros for cross-endian file reading.
+ * Uses Python's optimized byte-swap functions from pycore_bitutils.h */
+#define SWAP16_IF(swap, x) ((swap) ? _Py_bswap16(x) : (x))
+#define SWAP32_IF(swap, x) ((swap) ? _Py_bswap32(x) : (x))
+#define SWAP64_IF(swap, x) ((swap) ? _Py_bswap64(x) : (x))
+
 /* Header field offsets and sizes */
 #define HDR_OFF_MAGIC        0
 #define HDR_SIZE_MAGIC       4
@@ -286,6 +293,7 @@ typedef struct {
     uint8_t py_major;
     uint8_t py_minor;
     uint8_t py_micro;
+    int needs_swap;  /* Non-zero if file was written on different-endian system */
     uint64_t start_time_us;
     uint64_t sample_interval_us;
     uint32_t sample_count;

Modules/_remote_debugging/binary_io_reader.c

@@ -11,6 +11,7 @@
 
 #include "binary_io.h"
 #include "_remote_debugging.h"
+#include "pycore_bitutils.h"   /* _Py_bswap32, _Py_bswap64 for cross-endian reading */
 #include <string.h>
 
 #ifdef HAVE_ZSTD
@@ -49,7 +50,13 @@ reader_parse_header(BinaryReader *reader, const uint8_t *data, size_t file_size)
     memcpy(&magic, &data[0], sizeof(magic));
     memcpy(&version, &data[4], sizeof(version));
 
-    if (magic != BINARY_FORMAT_MAGIC) {
+    /* Detect endianness from magic number */
+    if (magic == BINARY_FORMAT_MAGIC) {
+        reader->needs_swap = 0;
+    } else if (magic == BINARY_FORMAT_MAGIC_SWAPPED) {
+        reader->needs_swap = 1;
+        version = _Py_bswap32(version);
+    } else {
         PyErr_Format(PyExc_ValueError, "Invalid magic number: 0x%08x", magic);
         return -1;
     }
@@ -77,13 +84,26 @@ reader_parse_header(BinaryReader *reader, const uint8_t *data, size_t file_size)
     reader->py_major = data[HDR_OFF_PY_MAJOR];
     reader->py_minor = data[HDR_OFF_PY_MINOR];
     reader->py_micro = data[HDR_OFF_PY_MICRO];
-    memcpy(&reader->start_time_us, &data[HDR_OFF_START_TIME], HDR_SIZE_START_TIME);
-    memcpy(&reader->sample_interval_us, &data[HDR_OFF_INTERVAL], HDR_SIZE_INTERVAL);
-    memcpy(&reader->sample_count, &data[HDR_OFF_SAMPLES], HDR_SIZE_SAMPLES);
-    memcpy(&reader->thread_count, &data[HDR_OFF_THREADS], HDR_SIZE_THREADS);
-    memcpy(&reader->string_table_offset, &data[HDR_OFF_STR_TABLE], HDR_SIZE_STR_TABLE);
-    memcpy(&reader->frame_table_offset, &data[HDR_OFF_FRAME_TABLE], HDR_SIZE_FRAME_TABLE);
-    memcpy(&reader->compression_type, &data[HDR_OFF_COMPRESSION], HDR_SIZE_COMPRESSION);
+
+    /* Read header fields with byte-swapping if needed */
+    uint64_t start_time_us, sample_interval_us, string_table_offset, frame_table_offset;
+    uint32_t sample_count, thread_count, compression_type;
+
+    memcpy(&start_time_us, &data[HDR_OFF_START_TIME], HDR_SIZE_START_TIME);
+    memcpy(&sample_interval_us, &data[HDR_OFF_INTERVAL], HDR_SIZE_INTERVAL);
+    memcpy(&sample_count, &data[HDR_OFF_SAMPLES], HDR_SIZE_SAMPLES);
+    memcpy(&thread_count, &data[HDR_OFF_THREADS], HDR_SIZE_THREADS);
+    memcpy(&string_table_offset, &data[HDR_OFF_STR_TABLE], HDR_SIZE_STR_TABLE);
+    memcpy(&frame_table_offset, &data[HDR_OFF_FRAME_TABLE], HDR_SIZE_FRAME_TABLE);
+    memcpy(&compression_type, &data[HDR_OFF_COMPRESSION], HDR_SIZE_COMPRESSION);
+
+    reader->start_time_us = SWAP64_IF(reader->needs_swap, start_time_us);
+    reader->sample_interval_us = SWAP64_IF(reader->needs_swap, sample_interval_us);
+    reader->sample_count = SWAP32_IF(reader->needs_swap, sample_count);
+    reader->thread_count = SWAP32_IF(reader->needs_swap, thread_count);
+    reader->string_table_offset = SWAP64_IF(reader->needs_swap, string_table_offset);
+    reader->frame_table_offset = SWAP64_IF(reader->needs_swap, frame_table_offset);
+    reader->compression_type = (int)SWAP32_IF(reader->needs_swap, compression_type);
 
     return 0;
 }
@@ -98,8 +118,12 @@ reader_parse_footer(BinaryReader *reader, const uint8_t *data, size_t file_size)
 
     const uint8_t *footer = data + file_size - FILE_FOOTER_SIZE;
     /* Use memcpy to avoid strict aliasing violations */
-    memcpy(&reader->strings_count, &footer[0], sizeof(reader->strings_count));
-    memcpy(&reader->frames_count, &footer[4], sizeof(reader->frames_count));
+    uint32_t strings_count, frames_count;
+    memcpy(&strings_count, &footer[0], sizeof(strings_count));
+    memcpy(&frames_count, &footer[4], sizeof(frames_count));
+
+    reader->strings_count = SWAP32_IF(reader->needs_swap, strings_count);
+    reader->frames_count = SWAP32_IF(reader->needs_swap, frames_count);
 
     return 0;
 }
@@ -857,15 +881,18 @@ binary_reader_replay(BinaryReader *reader, PyObject *collector, PyObject *progre
             break;  /* End of data */
         }
 
-        /* Use memcpy to avoid strict aliasing violations */
-        uint64_t thread_id;
-        uint32_t interpreter_id;
-        memcpy(&thread_id, &reader->sample_data[offset], sizeof(thread_id));
+        /* Use memcpy to avoid strict aliasing violations, then byte-swap if needed */
+        uint64_t thread_id_raw;
+        uint32_t interpreter_id_raw;
+        memcpy(&thread_id_raw, &reader->sample_data[offset], sizeof(thread_id_raw));
         offset += 8;
 
-        memcpy(&interpreter_id, &reader->sample_data[offset], sizeof(interpreter_id));
+        memcpy(&interpreter_id_raw, &reader->sample_data[offset], sizeof(interpreter_id_raw));
         offset += 4;
 
+        uint64_t thread_id = SWAP64_IF(reader->needs_swap, thread_id_raw);
+        uint32_t interpreter_id = SWAP32_IF(reader->needs_swap, interpreter_id_raw);
+
         /* Get or create thread state for reconstruction */
         ReaderThreadState *ts = reader_get_or_create_thread_state(reader, thread_id, interpreter_id);
         if (!ts) {

Modules/_remote_debugging/binary_io_writer.c

@@ -1055,8 +1055,12 @@ binary_writer_finalize(BinaryWriter *writer)
     }
     uint64_t file_size = (uint64_t)footer_offset + 32;
     uint8_t footer[32] = {0};
-    memcpy(footer + 0, &writer->string_count, 4);
-    memcpy(footer + 4, &writer->frame_count, 4);
+    /* Cast size_t to uint32_t before memcpy to ensure correct bytes are copied
+     * on both little-endian and big-endian systems (size_t is 8 bytes on 64-bit) */
+    uint32_t string_count_u32 = (uint32_t)writer->string_count;
+    uint32_t frame_count_u32 = (uint32_t)writer->frame_count;
+    memcpy(footer + 0, &string_count_u32, 4);
+    memcpy(footer + 4, &frame_count_u32, 4);
     memcpy(footer + 8, &file_size, 8);
     /* bytes 16-31: checksum placeholder (zeros) */
     if (fwrite_checked_allow_threads(footer, 32, writer->fp) < 0) {
@@ -1068,9 +1072,12 @@ binary_writer_finalize(BinaryWriter *writer)
         return -1;
     }
 
-    /* Convert file offsets to uint64_t for portable header format */
+    /* Convert file offsets and counts to fixed-width types for portable header format.
+     * This ensures correct behavior on both little-endian and big-endian systems. */
     uint64_t string_table_offset_u64 = (uint64_t)string_table_offset;
     uint64_t frame_table_offset_u64 = (uint64_t)frame_table_offset;
+    uint32_t thread_count_u32 = (uint32_t)writer->thread_count;
+    uint32_t compression_type_u32 = (uint32_t)writer->compression_type;
 
     uint8_t header[FILE_HEADER_SIZE] = {0};
     uint32_t magic = BINARY_FORMAT_MAGIC;
@@ -1083,10 +1090,10 @@ binary_writer_finalize(BinaryWriter *writer)
     memcpy(header + HDR_OFF_START_TIME, &writer->start_time_us, HDR_SIZE_START_TIME);
     memcpy(header + HDR_OFF_INTERVAL, &writer->sample_interval_us, HDR_SIZE_INTERVAL);
     memcpy(header + HDR_OFF_SAMPLES, &writer->total_samples, HDR_SIZE_SAMPLES);
-    memcpy(header + HDR_OFF_THREADS, &writer->thread_count, HDR_SIZE_THREADS);
+    memcpy(header + HDR_OFF_THREADS, &thread_count_u32, HDR_SIZE_THREADS);
     memcpy(header + HDR_OFF_STR_TABLE, &string_table_offset_u64, HDR_SIZE_STR_TABLE);
     memcpy(header + HDR_OFF_FRAME_TABLE, &frame_table_offset_u64, HDR_SIZE_FRAME_TABLE);
-    memcpy(header + HDR_OFF_COMPRESSION, &writer->compression_type, HDR_SIZE_COMPRESSION);
+    memcpy(header + HDR_OFF_COMPRESSION, &compression_type_u32, HDR_SIZE_COMPRESSION);
     if (fwrite_checked_allow_threads(header, FILE_HEADER_SIZE, writer->fp) < 0) {
         return -1;
     }