Merge branch 'remove-__package__' of github.com:brettcannon/cpython into remove-__package__

Author: brettcannon

Lib/encodings/punycode.py

@@ -17,7 +17,7 @@ def segregate(str):
         else:
             extended.add(c)
     extended = sorted(extended)
-    return bytes(base), extended
+    return base.take_bytes(), extended
 
 def selective_len(str, max):
     """Return the length of str, considering only characters below max."""
@@ -83,7 +83,7 @@ def generate_generalized_integer(N, bias):
         t = T(j, bias)
         if N < t:
             result.append(digits[N])
-            return bytes(result)
+            return result.take_bytes()
         result.append(digits[t + ((N - t) % (36 - t))])
         N = (N - t) // (36 - t)
         j += 1
@@ -112,7 +112,7 @@ def generate_integers(baselen, deltas):
         s = generate_generalized_integer(delta, bias)
         result.extend(s)
         bias = adapt(delta, points==0, baselen+points+1)
-    return bytes(result)
+    return result.take_bytes()
 
 def punycode_encode(text):
     base, extended = segregate(text)

Lib/importlib/_bootstrap_external.py

@@ -208,12 +208,8 @@ def _write_atomic(path, data, mode=0o666):
     try:
         # We first write data to a temporary file, and then use os.replace() to
         # perform an atomic rename.
-        with _io.FileIO(fd, 'wb') as file:
-            bytes_written = file.write(data)
-        if bytes_written != len(data):
-            # Raise an OSError so the 'except' below cleans up the partially
-            # written file.
-            raise OSError("os.write() didn't write the full pyc file")
+        with _io.open(fd, 'wb') as file:
+            file.write(data)
         _os.replace(path_tmp, path)
     except OSError:
         try:

Lib/re/_compiler.py

@@ -375,7 +375,7 @@ def _optimize_charset(charset, iscased=None, fixup=None, fixes=None):
     # less significant byte is a bit index in the chunk (just like the
     # CHARSET matching).
 
-    charmap = bytes(charmap) # should be hashable
+    charmap = charmap.take_bytes() # should be hashable
     comps = {}
     mapping = bytearray(256)
     block = 0

Lib/test/test_importlib/test_util.py

@@ -783,31 +783,70 @@ def test_complete_multi_phase_init_module(self):
             self.run_with_own_gil(script)
 
 
-class MiscTests(unittest.TestCase):
-    def test_atomic_write_should_notice_incomplete_writes(self):
+class PatchAtomicWrites:
+    def __init__(self, truncate_at_length, never_complete=False):
+        self.truncate_at_length = truncate_at_length
+        self.never_complete = never_complete
+        self.seen_write = False
+        self._children = []
+
+    def __enter__(self):
         import _pyio
 
         oldwrite = os.write
-        seen_write = False
-
-        truncate_at_length = 100
 
         # Emulate an os.write that only writes partial data.
         def write(fd, data):
-            nonlocal seen_write
-            seen_write = True
-            return oldwrite(fd, data[:truncate_at_length])
+            if self.seen_write and self.never_complete:
+                return None
+            self.seen_write = True
+            return oldwrite(fd, data[:self.truncate_at_length])
 
         # Need to patch _io to be _pyio, so that io.FileIO is affected by the
         # os.write patch.
-        with (support.swap_attr(_bootstrap_external, '_io', _pyio),
-              support.swap_attr(os, 'write', write)):
-            with self.assertRaises(OSError):
-                # Make sure we write something longer than the point where we
-                # truncate.
-                content = b'x' * (truncate_at_length * 2)
-                _bootstrap_external._write_atomic(os_helper.TESTFN, content)
-        assert seen_write
+        self.children = [
+            support.swap_attr(_bootstrap_external, '_io', _pyio),
+            support.swap_attr(os, 'write', write)
+        ]
+        for child in self.children:
+            child.__enter__()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        for child in self.children:
+            child.__exit__(exc_type, exc_val, exc_tb)
+
+
+class MiscTests(unittest.TestCase):
+
+    def test_atomic_write_retries_incomplete_writes(self):
+        truncate_at_length = 100
+        length = truncate_at_length * 2
+
+        with PatchAtomicWrites(truncate_at_length=truncate_at_length) as cm:
+            # Make sure we write something longer than the point where we
+            # truncate.
+            content = b'x' * length
+            _bootstrap_external._write_atomic(os_helper.TESTFN, content)
+        self.assertTrue(cm.seen_write)
+
+        self.assertEqual(os.stat(support.os_helper.TESTFN).st_size, length)
+        os.unlink(support.os_helper.TESTFN)
+
+    def test_atomic_write_errors_if_unable_to_complete(self):
+        truncate_at_length = 100
+
+        with (
+            PatchAtomicWrites(
+                truncate_at_length=truncate_at_length, never_complete=True,
+            ) as cm,
+            self.assertRaises(OSError)
+        ):
+            # Make sure we write something longer than the point where we
+            # truncate.
+            content = b'x' * (truncate_at_length * 2)
+            _bootstrap_external._write_atomic(os_helper.TESTFN, content)
+        self.assertTrue(cm.seen_write)
 
         with self.assertRaises(OSError):
             os.stat(support.os_helper.TESTFN) # Check that the file did not get written.

Misc/NEWS.d/next/Core_and_Builtins/2025-11-24-21-09-30.gh-issue-141930.hIIzSd.rst

@@ -0,0 +1,2 @@
+When importing a module, use Python's regular file object to ensure that
+writes to ``.pyc`` files are complete or an appropriate error is raised.

Misc/NEWS.d/next/Library/2025-11-25-23-29-08.gh-issue-141968.0JnjXf.rst

@@ -0,0 +1,2 @@
+Remove data copy from :mod:`codecs` ``punycode`` encoding by using
+:meth:`bytearray.take_bytes`.

Misc/NEWS.d/next/Library/2025-11-26-14-20-10.gh-issue-141968.W139Pv.rst

@@ -0,0 +1,2 @@
+Remove data copy from :mod:`re` compilation of regexes with large charsets
+by using :meth:`bytearray.take_bytes`.

Tools/c-analyzer/c_parser/preprocessor/__init__.py

@@ -16,6 +16,7 @@
 from . import (
     pure as _pure,
     gcc as _gcc,
+    clang as _clang,
 )
 
 
@@ -234,7 +235,7 @@ def handling_errors(ignore_exc=None, *, log_err=None):
     'bcpp': None,
     # aliases/extras:
     'gcc': _gcc.preprocess,
-    'clang': None,
+    'clang': _clang.preprocess,
 }
 
 

Tools/c-analyzer/c_parser/preprocessor/clang.py

@@ -0,0 +1,104 @@
+import os.path
+import re, sys
+
+from . import common as _common
+from . import gcc as _gcc
+
+_normpath = _gcc._normpath
+
+TOOL = 'clang'
+
+META_FILES = {
+    '<built-in>',
+    '<command line>',
+}
+
+
+def preprocess(filename,
+               incldirs=None,
+               includes=None,
+               macros=None,
+               samefiles=None,
+               cwd=None,
+               ):
+    if not cwd or not os.path.isabs(cwd):
+        cwd = os.path.abspath(cwd or '.')
+    filename = _normpath(filename, cwd)
+
+    postargs = _gcc.POST_ARGS
+    basename = os.path.basename(filename)
+    dirname = os.path.basename(os.path.dirname(filename))
+    if (basename not in _gcc.FILES_WITHOUT_INTERNAL_CAPI
+       and dirname not in _gcc.DIRS_WITHOUT_INTERNAL_CAPI):
+        postargs += ('-DPy_BUILD_CORE=1',)
+
+    text = _common.preprocess(
+        TOOL,
+        filename,
+        incldirs=incldirs,
+        includes=includes,
+        macros=macros,
+        #preargs=PRE_ARGS,
+        postargs=postargs,
+        executable=['clang'],
+        compiler='unix',
+        cwd=cwd,
+    )
+    return _iter_lines(text, filename, samefiles, cwd)
+
+
+EXIT_MARKERS = {'# 2 "<built-in>" 2', '# 3 "<built-in>" 2', '# 4 "<built-in>" 2'}
+
+
+def _iter_lines(text, reqfile, samefiles, cwd, raw=False):
+    lines = iter(text.splitlines())
+
+    # The first line is special.
+    # The subsequent lines are consistent.
+    firstlines = [
+        f'# 1 "{reqfile}"',
+        '# 1 "<built-in>" 1',
+        '# 1 "<built-in>" 3',
+        '# 370 "<built-in>" 3',
+        '# 1 "<command line>" 1',
+        '# 1 "<built-in>" 2',
+    ]
+    for expected in firstlines:
+        line = next(lines)
+        if line != expected:
+            raise NotImplementedError((line, expected))
+
+    # Do all the CLI-provided includes.
+    filter_reqfile = (lambda f: _gcc._filter_reqfile(f, reqfile, samefiles))
+    make_info = (lambda lno: _common.FileInfo(reqfile, lno))
+    last = None
+    for line in lines:
+        assert last != reqfile, (last,)
+        # NOTE:condition is clang specific
+        if not line:
+            continue
+        lno, included, flags = _gcc._parse_marker_line(line, reqfile)
+        if not included:
+            raise NotImplementedError((line,))
+        if included == reqfile:
+            # This will be the last one.
+            assert 2 in flags, (line, flags)
+        else:
+            # NOTE:first condition is specific to clang
+            if _normpath(included, cwd) == reqfile:
+                assert 1 in flags or 2 in flags, (line, flags, included, reqfile)
+            else:
+                assert 1 in flags, (line, flags, included, reqfile)
+        yield from _gcc._iter_top_include_lines(
+            lines,
+            _normpath(included, cwd),
+            cwd,
+            filter_reqfile,
+            make_info,
+            raw,
+            EXIT_MARKERS
+        )
+        last = included
+    # The last one is always the requested file.
+    # NOTE:_normpath is clang specific
+    assert _normpath(included, cwd) == reqfile, (line,)

Tools/c-analyzer/c_parser/preprocessor/gcc.py

@@ -65,6 +65,8 @@
     '-E',
 )
 
+EXIT_MARKERS = {'# 0 "<command-line>" 2', '# 1 "<command-line>" 2'}
+
 
 def preprocess(filename,
                incldirs=None,
@@ -138,6 +140,7 @@ def _iter_lines(text, reqfile, samefiles, cwd, raw=False):
             filter_reqfile,
             make_info,
             raw,
+            EXIT_MARKERS
         )
         last = included
     # The last one is always the requested file.
@@ -146,20 +149,28 @@ def _iter_lines(text, reqfile, samefiles, cwd, raw=False):
 
 def _iter_top_include_lines(lines, topfile, cwd,
                             filter_reqfile, make_info,
-                            raw):
+                            raw, exit_markers):
     partial = 0  # depth
     files = [topfile]
     # We start at 1 in case there are source lines (including blank ones)
     # before the first marker line.  Also, we already verified in
     # _parse_marker_line() that the preprocessor reported lno as 1.
     lno = 1
     for line in lines:
-        if line == '# 0 "<command-line>" 2' or line == '# 1 "<command-line>" 2':
+        if line in exit_markers:
             # We're done with this top-level include.
             return
 
         _lno, included, flags = _parse_marker_line(line)
         if included:
+            # HACK:
+            # Mixes curses.h and ncurses.h marker lines
+            # gcc silently passes this, while clang fails
+            # See: /Include/py_curses.h #if-elif directives
+            # And compare with preprocessor output
+            if os.path.basename(included) == 'curses.h':
+                included = os.path.join(os.path.dirname(included), 'ncurses.h')
+
             lno = _lno
             included = _normpath(included, cwd)
             # We hit a marker line.