From d0bfc2eb52c955d3a7a2f00d5aa40051e61ecb27 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 27 Mar 2025 15:40:52 +0100 Subject: [PATCH 01/15] PEP 782: Add PyBytesWriter C API --- peps/pep-0782.rst | 371 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 371 insertions(+) create mode 100644 peps/pep-0782.rst diff --git a/peps/pep-0782.rst b/peps/pep-0782.rst new file mode 100644 index 00000000000..6e88480948e --- /dev/null +++ b/peps/pep-0782.rst @@ -0,0 +1,371 @@ +PEP: 782 +Title: Add PyBytesWriter C API +Author: Victor Stinner +Discussions-To: https://discuss.python.org/t/81182 +Status: Draft +Type: Standards Track +Created: 27-Mar-2025 +Python-Version: 3.14 + + +.. highlight:: c + + +Abstract +======== + +Add a new ``PyBytesWriter`` C API to create ``bytes`` object. + +It replaces the ``PyBytes_FromStringAndSize(NULL, size)`` and +``_PyBytes_Resize()`` APIs which treat an immutable ``bytes`` object as +a mutable object. + + +Rationale +========= + +Disallow creation of incomplete/inconsistent objects +---------------------------------------------------- + +Creating a Python :class:`bytes` object using +``PyBytes_FromStringAndSize(NULL, size)`` and ``_PyBytes_Resize()`` +treats an immutable :class:`bytes` object as mutable. It goes against +the principle that :class:`bytes` objects are immutable. It also creates +an incomplete or "invalid" object since bytes are not initialized. In +Python, a :class:`bytes` object should always have its bytes fully +initialized. + +* `Avoid creating incomplete/invalid objects api-evolution#36 + `_ +* `Disallow mutating immutable objects api-evolution#20 + `_ +* `Disallow creation of incomplete/inconsistent objects problems#56 + `_ + +Overallocation +-------------- + +When the output size is unknown, there are two allocation strategies: + +* Overallocate by the worst case, and then shrink at the end. +* Extend the buffer when a larger write is needed, multiple times if + needed. + +Both strategies are inefficient. Overallocating by the worst case +consumes too much memory. Extending the buffer multiple times is +inefficient. + +A better strategy is to overallocate the buffer when extending the +buffer to reduce the number of expensive ``realloc()`` operations which +can imply a memory copy. + + +Specification +============= + +API +--- + +.. c:type:: PyBytesWriter + + A Python :class:`bytes` writer instance created by + :c:func:`PyBytesWriter_Create`. + + The instance must be destroyed by :c:func:`PyBytesWriter_Finish` or + :c:func:`PyBytesWriter_Discard`. + +Create, Finish, Discard +^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:function:: PyBytesWriter* PyBytesWriter_Create(Py_ssize_t size) + + Create a :c:type:`PyBytesWriter` to write *size* bytes. + + If *size* is greater than zero, allocate *size* bytes for the + returned buffer. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + *size* must be positive or zero. + +.. c:function:: PyObject* PyBytesWriter_Finish(PyBytesWriter *writer) + + Finish a :c:type:`PyBytesWriter` created by + :c:func:`PyBytesWriter_Create`. + + On success, return a Python :class:`bytes` object. + On error, set an exception and return ``NULL``. + + The writer instance is invalid after the call in any case. + +.. c:function:: PyObject* PyBytesWriter_FinishWithSize(PyBytesWriter *writer, Py_ssize_t size) + + Similar to :c:func:`PyBytesWriter_Finish`, but resize the writer + to *size* bytes before creating the :class:`bytes` object. + +.. c:function:: PyObject* PyBytesWriter_FinishWithPointer(PyBytesWriter *writer, void *buf) + + Similar to :c:func:`PyBytesWriter_Finish`, but resize the writer + using *buf* pointer before creating the :class:`bytes` object. + +.. c:function:: void PyBytesWriter_Discard(PyBytesWriter *writer) + + Discard a :c:type:`PyBytesWriter` created by :c:func:`PyBytesWriter_Create`. + + The writer instance is invalid after the call. + +High-level API +^^^^^^^^^^^^^^ + +.. c:function:: int PyBytesWriter_WriteBytes(PyBytesWriter *writer, const void *bytes, Py_ssize_t size) + + Write *size* bytes of *bytes* into the writer. + + If *size* is equal to ``-1``, call ``strlen(bytes)`` to get the + string length. + + On success, return ``0``. + On error, set an exception and return ``-1``. + +.. c:function:: int PyBytesWriter_Format(PyBytesWriter *writer, const char *format, ...) + + Similar to ``PyBytes_FromFormat()``, but write the output directly + into the writer. + + On success, return ``0``. + On error, set an exception and return ``-1``. + +Getters +^^^^^^^ + +.. c:function:: void* PyBytesWriter_GetData(PyBytesWriter *writer) + + Get the writer data. + +.. c:function:: Py_ssize_t PyBytesWriter_GetSize(PyBytesWriter *writer) + + Get the writer size. + +Low-level API +^^^^^^^^^^^^^ + +.. c:function:: int PyBytesWriter_Resize(PyBytesWriter *writer, Py_ssize_t size) + + Resize the writer to *size* bytes. It can be used to enlarge or to + shrink the writer. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + *size* must be positive or zero. + +.. c:function:: int PyBytesWriter_Grow(PyBytesWriter *writer, Py_ssize_t grow) + + Resize the writer by adding *grow* bytes to the current writer size. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + *size* must be positive or zero. + +.. c:function:: void* PyBytesWriter_GrowAndUpdatePointer(PyBytesWriter *writer, Py_ssize_t size, void *buf) + + Similar to :c:func:`PyBytesWriter_Grow`, but update also the *buf* + pointer. + + +Overallocation +-------------- + +:c:func:`PyBytesWriter_Resize` overallocates the internal buffer to +reduce the number of ``realloc()`` calls and so reduce memory copies. + + +Thread safety +------------- + +The API is not thread safe: a writer should only be used by a single +thread at the same time. + + +Examples +-------- + +High-level API +^^^^^^^^^^^^^^ + +Create the string ``"Hello World!"``:: + + PyObject* hello_world(void) + { + PyBytesWriter *writer = PyBytesWriter_Create(0); + if (writer == NULL) { + goto error; + } + if (PyBytesWriter_WriteBytes(writer, "Hello", -1) < 0) { + goto error; + } + if (PyBytesWriter_Format(writer, " %s!", "World") < 0) { + goto error; + } + return PyBytesWriter_Finish(writer); + + error: + PyBytesWriter_Discard(writer); + return NULL; + } + + +Create the bytes string "abc" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example creating the string ``"abc"``, with a fixed size of 3 bytes:: + + PyObject* create_abc(void) + { + PyBytesWriter *writer = PyBytesWriter_Create(3); + if (writer == NULL) { + return NULL; + } + + char *str = PyBytesWriter_GetData(writer); + memcpy(str, "abc", 3); + return PyBytesWriter_Finish(writer); + } + +GrowAndUpdatePointer() example +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example using a pointer to write bytes and to track the written size. + +Create the string ``"Hello World"``:: + + PyObject* grow_example(void) + { + // Allocate 10 bytes + PyBytesWriter *writer = PyBytesWriter_Create(10); + if (writer == NULL) { + return NULL; + } + + // Write some bytes + char *buf = PyBytesWriter_GetData(writer); + memcpy(buf, "Hello ", strlen("Hello ")); + buf += strlen("Hello "); + + // Allocate 10 more bytes + buf = PyBytesWriter_GrowAndUpdatePointer(writer, 10, buf); + if (buf == NULL) { + PyBytesWriter_Discard(writer); + return NULL; + } + + // Write more bytes + memcpy(buf, "World", strlen("World")); + buf += strlen("World"); + + // Truncate the string at 'buf' position + // and create a bytes object + return PyBytesWriter_FinishWithPointer(writer, buf); + } + + +Implementation +============== + +* `Pull request gh-131681 `__. + + +Backwards Compatibility +======================= + +There is no impact on the backward compatibility, only new APIs are +added. + + +Projects using _PyBytes_Resize() +================================ + +A code search on PyPI top 8,000 projects finds 41 projects using +``_PyBytes_Resize``: + +* Nuitka (2.6) +* PyBluez (0.23) +* PyICU (2.14) +* PyICU-binary (2.7.4) +* SimpleParse (2.2.4) +* apsw (3.48.0.0) +* asyncio (3.4.3) +* billiard (4.2.1) +* bitarray (3.0.0) +* blosc (1.11.2) +* casadi (3.6.7) +* catboost (1.2.7) +* cython (3.0.11) +* ddtrace (2.20.0) +* deflate (0.7.0) +* isal (1.7.1) +* m2crypto (0.43.0) +* msgspec (0.19.0) +* multiprocess (0.70.17) +* mysql-connector (2.2.9) +* mysql-connector-python-rf (2.2.2) +* mysqlclient (2.2.7) +* orjson (3.10.15) +* ormsgpack (1.7.0) +* pickle5 (0.0.12) +* pillow (11.1.0) +* psycopg2 (2.9.10) +* psycopg2-binary (2.9.10) +* pyarrow (19.0.0) +* pybase64 (1.4.0) +* pygobject (3.50.0) +* pygresql (6.1.0) +* pyobjc_core (11.0) +* pysam (0.22.1) +* pyzstd (0.16.2) +* rcssmin (1.2.0) +* rjsmin (1.2.3) +* zipfile-deflate64 (0.2.0) +* zlib_ng (0.5.1) +* zodbpickle (4.1.1) +* zstandard (0.23.0) + + +Discussions +=========== + +* March 2025: Third public API attempt, using size rather than pointers: + + * `Discussion `_ + * `Pull request gh-131681 `__ + +* February 2025: Second public API attempt: + + * `Issue gh-129813 `_ + and + `pull request gh-129814 + `_ + +* July 2024: First public API attempt: + + * C API Working Group decision: + `Add PyBytes_Writer() API + `_ + (August 2024) + * `Pull request gh-121726 + `_: + first public API attempt (July 2024) + +* March 2016: + `Fast _PyAccu, _PyUnicodeWriter and _PyBytesWriter APIs to produce + strings in CPython `_: + Article on the original private ``_PyBytesWriter`` C API. + + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. From 1a25a38152afe9dfaf8c7675ee9c4dbafec0eef0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 27 Mar 2025 16:33:27 +0100 Subject: [PATCH 02/15] Add myself to CODEOWNERS --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index de4e365de56..3eb3aef6031 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -659,6 +659,7 @@ peps/pep-0777.rst @warsaw # ... peps/pep-0779.rst @Yhg1s @colesbury @mpage peps/pep-0780.rst @lysnikolaou +peps/pep-0782.rst @vstinner # ... peps/pep-0789.rst @njsmith # ... From 79bdc300b09ab9e5b4ba8d60678d966d9d04b512 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 28 Mar 2025 08:53:46 +0100 Subject: [PATCH 03/15] Apply suggestions from code review Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- peps/pep-0782.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/peps/pep-0782.rst b/peps/pep-0782.rst index 6e88480948e..ecea0cb87dc 100644 --- a/peps/pep-0782.rst +++ b/peps/pep-0782.rst @@ -6,6 +6,8 @@ Status: Draft Type: Standards Track Created: 27-Mar-2025 Python-Version: 3.14 +Post-History: + `18-Feb-2025 __, .. highlight:: c @@ -271,8 +273,8 @@ Create the string ``"Hello World"``:: } -Implementation -============== +Reference Implementation +======================== * `Pull request gh-131681 `__. @@ -333,8 +335,8 @@ A code search on PyPI top 8,000 projects finds 41 projects using * zstandard (0.23.0) -Discussions -=========== +Prior Discussions +================= * March 2025: Third public API attempt, using size rather than pointers: From 496a1fa663d1d0606cb6780881348f9688961069 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 28 Mar 2025 08:55:35 +0100 Subject: [PATCH 04/15] Remove PyPI study Examples becomes a top-level section. --- peps/pep-0782.rst | 60 ++++------------------------------------------- 1 file changed, 5 insertions(+), 55 deletions(-) diff --git a/peps/pep-0782.rst b/peps/pep-0782.rst index ecea0cb87dc..13baa2cb9ab 100644 --- a/peps/pep-0782.rst +++ b/peps/pep-0782.rst @@ -1,13 +1,12 @@ PEP: 782 Title: Add PyBytesWriter C API Author: Victor Stinner -Discussions-To: https://discuss.python.org/t/81182 Status: Draft Type: Standards Track Created: 27-Mar-2025 Python-Version: 3.14 Post-History: - `18-Feb-2025 __, + `18-Feb-2025 `__ .. highlight:: c @@ -192,10 +191,10 @@ thread at the same time. Examples --------- +======== High-level API -^^^^^^^^^^^^^^ +-------------- Create the string ``"Hello World!"``:: @@ -220,7 +219,7 @@ Create the string ``"Hello World!"``:: Create the bytes string "abc" -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +----------------------------- Example creating the string ``"abc"``, with a fixed size of 3 bytes:: @@ -237,7 +236,7 @@ Example creating the string ``"abc"``, with a fixed size of 3 bytes:: } GrowAndUpdatePointer() example -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------------------------ Example using a pointer to write bytes and to track the written size. @@ -286,55 +285,6 @@ There is no impact on the backward compatibility, only new APIs are added. -Projects using _PyBytes_Resize() -================================ - -A code search on PyPI top 8,000 projects finds 41 projects using -``_PyBytes_Resize``: - -* Nuitka (2.6) -* PyBluez (0.23) -* PyICU (2.14) -* PyICU-binary (2.7.4) -* SimpleParse (2.2.4) -* apsw (3.48.0.0) -* asyncio (3.4.3) -* billiard (4.2.1) -* bitarray (3.0.0) -* blosc (1.11.2) -* casadi (3.6.7) -* catboost (1.2.7) -* cython (3.0.11) -* ddtrace (2.20.0) -* deflate (0.7.0) -* isal (1.7.1) -* m2crypto (0.43.0) -* msgspec (0.19.0) -* multiprocess (0.70.17) -* mysql-connector (2.2.9) -* mysql-connector-python-rf (2.2.2) -* mysqlclient (2.2.7) -* orjson (3.10.15) -* ormsgpack (1.7.0) -* pickle5 (0.0.12) -* pillow (11.1.0) -* psycopg2 (2.9.10) -* psycopg2-binary (2.9.10) -* pyarrow (19.0.0) -* pybase64 (1.4.0) -* pygobject (3.50.0) -* pygresql (6.1.0) -* pyobjc_core (11.0) -* pysam (0.22.1) -* pyzstd (0.16.2) -* rcssmin (1.2.0) -* rjsmin (1.2.3) -* zipfile-deflate64 (0.2.0) -* zlib_ng (0.5.1) -* zodbpickle (4.1.1) -* zstandard (0.23.0) - - Prior Discussions ================= From 54ebc677e8eac2546e12a98c904b44e4546c90eb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 28 Mar 2025 11:37:37 +0100 Subject: [PATCH 05/15] Update --- peps/pep-0782.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/peps/pep-0782.rst b/peps/pep-0782.rst index 13baa2cb9ab..09b728b4958 100644 --- a/peps/pep-0782.rst +++ b/peps/pep-0782.rst @@ -140,14 +140,14 @@ High-level API Getters ^^^^^^^ -.. c:function:: void* PyBytesWriter_GetData(PyBytesWriter *writer) - - Get the writer data. - .. c:function:: Py_ssize_t PyBytesWriter_GetSize(PyBytesWriter *writer) Get the writer size. +.. c:function:: void* PyBytesWriter_GetData(PyBytesWriter *writer) + + Get the writer data. + Low-level API ^^^^^^^^^^^^^ @@ -179,8 +179,9 @@ Low-level API Overallocation -------------- -:c:func:`PyBytesWriter_Resize` overallocates the internal buffer to -reduce the number of ``realloc()`` calls and so reduce memory copies. +:c:func:`PyBytesWriter_Resize` and :c:func:`PyBytesWriter_Grow` +overallocate the internal buffer to reduce the number of ``realloc()`` +calls and so reduce memory copies. Thread safety From a4aa0f22fa53a4d6069127fcadb3d309e4398d11 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 29 Mar 2025 00:43:59 +0100 Subject: [PATCH 06/15] Update peps/pep-0782.rst Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- peps/pep-0782.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/peps/pep-0782.rst b/peps/pep-0782.rst index 09b728b4958..ec88357816c 100644 --- a/peps/pep-0782.rst +++ b/peps/pep-0782.rst @@ -15,7 +15,7 @@ Post-History: Abstract ======== -Add a new ``PyBytesWriter`` C API to create ``bytes`` object. +Add a new ``PyBytesWriter`` C API to create ``bytes`` objects. It replaces the ``PyBytes_FromStringAndSize(NULL, size)`` and ``_PyBytes_Resize()`` APIs which treat an immutable ``bytes`` object as From 7380b813c57cad6710a6fbc661aadd105746463c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 29 Mar 2025 11:15:33 +0100 Subject: [PATCH 07/15] Update --- peps/pep-0782.rst | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/peps/pep-0782.rst b/peps/pep-0782.rst index ec88357816c..f20c1372cbf 100644 --- a/peps/pep-0782.rst +++ b/peps/pep-0782.rst @@ -43,22 +43,17 @@ initialized. * `Disallow creation of incomplete/inconsistent objects problems#56 `_ -Overallocation --------------- - -When the output size is unknown, there are two allocation strategies: - -* Overallocate by the worst case, and then shrink at the end. -* Extend the buffer when a larger write is needed, multiple times if - needed. +Inefficient allocation strategy +------------------------------- -Both strategies are inefficient. Overallocating by the worst case -consumes too much memory. Extending the buffer multiple times is -inefficient. +When creating a bytes string and the output size is unknown, one +strategy is to allocate a short buffer and extend it (to the exact size) +each time a larger write is needed. -A better strategy is to overallocate the buffer when extending the -buffer to reduce the number of expensive ``realloc()`` operations which -can imply a memory copy. +This strategy is inefficient because it requires to enlarge the buffer +multiple timess. It's more efficient to overallocate the buffer the +first time that a larger write is needed. It reduces the number of +expensive ``realloc()`` operations which can imply a memory copy. Specification From f536c9e442dc382e8857f611bdca5dee649faaf0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 29 Mar 2025 11:27:24 +0100 Subject: [PATCH 08/15] Update --- peps/pep-0782.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/peps/pep-0782.rst b/peps/pep-0782.rst index f20c1372cbf..e5d4e0f76b2 100644 --- a/peps/pep-0782.rst +++ b/peps/pep-0782.rst @@ -271,7 +271,11 @@ Create the string ``"Hello World"``:: Reference Implementation ======================== -* `Pull request gh-131681 `__. +`Pull request gh-131681 `__. + +The implementation allocates internally a :class:`bytes` object, so +:c:func:`PyBytesWriter_Finish` just returns the object without having +to copy memory. Backwards Compatibility From c26246e44bc84b6519d602167ce61006fcb7580b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 29 Mar 2025 11:30:48 +0100 Subject: [PATCH 09/15] Implementation --- peps/pep-0782.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/peps/pep-0782.rst b/peps/pep-0782.rst index e5d4e0f76b2..f57a409b5f8 100644 --- a/peps/pep-0782.rst +++ b/peps/pep-0782.rst @@ -277,6 +277,11 @@ The implementation allocates internally a :class:`bytes` object, so :c:func:`PyBytesWriter_Finish` just returns the object without having to copy memory. +For strings up to 256 bytes, a small internal raw buffer of bytes is +used. It avoids having to resize a :class:`bytes` object which is +inefficient. At the end, :c:func:`PyBytesWriter_Finish` creates the +:class:`bytes` object from this small buffer. + Backwards Compatibility ======================= From dee2be5d9591104754fa3638c6700dde5466e448 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 29 Mar 2025 11:39:21 +0100 Subject: [PATCH 10/15] Update --- peps/pep-0782.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/peps/pep-0782.rst b/peps/pep-0782.rst index f57a409b5f8..5c4fe2f70f7 100644 --- a/peps/pep-0782.rst +++ b/peps/pep-0782.rst @@ -143,6 +143,9 @@ Getters Get the writer data. + The pointer is valid until :c:func:`PyBytesWriter_Finish` or + :c:func:`PyBytesWriter_Discard` is called on *writer*. + Low-level API ^^^^^^^^^^^^^ @@ -282,6 +285,9 @@ used. It avoids having to resize a :class:`bytes` object which is inefficient. At the end, :c:func:`PyBytesWriter_Finish` creates the :class:`bytes` object from this small buffer. +A free list is used to reduce the cost of allocating a +:c:type:`PyBytesWriter` on the heap memory. + Backwards Compatibility ======================= From d6a63ae057b1a6d78c929f521a51030aa7dd4f68 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 29 Mar 2025 12:00:39 +0100 Subject: [PATCH 11/15] Soft deprecations --- peps/pep-0782.rst | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/peps/pep-0782.rst b/peps/pep-0782.rst index 5c4fe2f70f7..40540123036 100644 --- a/peps/pep-0782.rst +++ b/peps/pep-0782.rst @@ -17,9 +17,11 @@ Abstract Add a new ``PyBytesWriter`` C API to create ``bytes`` objects. -It replaces the ``PyBytes_FromStringAndSize(NULL, size)`` and -``_PyBytes_Resize()`` APIs which treat an immutable ``bytes`` object as -a mutable object. +Soft deprecate ``PyBytes_FromStringAndSize(NULL, size)`` and +``_PyBytes_Resize()`` APIs. These APIs treat an immutable ``bytes`` +object as a mutable object. They remain available and maintained, don't +emit deprecation warning, but are no longer recommended when writing new +code. Rationale @@ -188,6 +190,18 @@ Thread safety The API is not thread safe: a writer should only be used by a single thread at the same time. +Soft deprecations +----------------- + +Soft deprecate ``PyBytes_FromStringAndSize(NULL, size)`` and +``_PyBytes_Resize()`` APIs. These APIs treat an immutable ``bytes`` +object as a mutable object. They remain available and maintained, don't +emit deprecation warning, but are no longer recommended when writing new +code. + +``PyBytes_FromStringAndSize(str, size)`` is not soft deprecated. Only +calls with ``NULL`` *str* are soft deprecated. + Examples ======== From 8a2fcb6f23307000cbd058fd4195970e2f5b9ec6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 31 Mar 2025 13:47:49 +0200 Subject: [PATCH 12/15] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- peps/pep-0782.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/peps/pep-0782.rst b/peps/pep-0782.rst index 40540123036..8a22e7d3dd4 100644 --- a/peps/pep-0782.rst +++ b/peps/pep-0782.rst @@ -82,8 +82,7 @@ Create, Finish, Discard If *size* is greater than zero, allocate *size* bytes for the returned buffer. - On success, return ``0``. - On error, set an exception and return ``-1``. + On error, set an exception and return NULL. *size* must be positive or zero. @@ -118,7 +117,7 @@ High-level API .. c:function:: int PyBytesWriter_WriteBytes(PyBytesWriter *writer, const void *bytes, Py_ssize_t size) - Write *size* bytes of *bytes* into the writer. + Write *size* bytes of *bytes* into the *writer*. If *size* is equal to ``-1``, call ``strlen(bytes)`` to get the string length. @@ -190,6 +189,7 @@ Thread safety The API is not thread safe: a writer should only be used by a single thread at the same time. + Soft deprecations ----------------- @@ -209,7 +209,7 @@ Examples High-level API -------------- -Create the string ``"Hello World!"``:: +Create the bytes ``b"Hello World!"``:: PyObject* hello_world(void) { @@ -234,7 +234,7 @@ Create the string ``"Hello World!"``:: Create the bytes string "abc" ----------------------------- -Example creating the string ``"abc"``, with a fixed size of 3 bytes:: +Example creating the bytes ``b"abc"``, with a fixed size of 3 bytes:: PyObject* create_abc(void) { From 07e69a075ed89508e4271eac876cec1a9b09994e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 31 Mar 2025 13:59:13 +0200 Subject: [PATCH 13/15] Address Benedikt's review --- peps/pep-0782.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/peps/pep-0782.rst b/peps/pep-0782.rst index 8a22e7d3dd4..6c82009e95d 100644 --- a/peps/pep-0782.rst +++ b/peps/pep-0782.rst @@ -106,10 +106,20 @@ Create, Finish, Discard Similar to :c:func:`PyBytesWriter_Finish`, but resize the writer using *buf* pointer before creating the :class:`bytes` object. + Pseudo-code:: + + Py_ssize_t size = (char*)buf - (char*)PyBytesWriter_GetData(writer); + return PyBytesWriter_FinishWithSize(writer, size); + + Set an exception and return ``NULL`` if *buf* pointer is invalid: + outside the internal buffer bounds. + .. c:function:: void PyBytesWriter_Discard(PyBytesWriter *writer) Discard a :c:type:`PyBytesWriter` created by :c:func:`PyBytesWriter_Create`. + Do nothing if *writer* is ``NULL``. + The writer instance is invalid after the call. High-level API @@ -155,6 +165,8 @@ Low-level API Resize the writer to *size* bytes. It can be used to enlarge or to shrink the writer. + Newly allocated bytes are left uninitialized. + On success, return ``0``. On error, set an exception and return ``-1``. @@ -164,6 +176,8 @@ Low-level API Resize the writer by adding *grow* bytes to the current writer size. + Newly allocated bytes are left uninitialized. + On success, return ``0``. On error, set an exception and return ``-1``. @@ -174,6 +188,16 @@ Low-level API Similar to :c:func:`PyBytesWriter_Grow`, but update also the *buf* pointer. + On error, set an exception and return ``NULL``. + + Pseudo-code:: + + Py_ssize_t pos = (char*)buf - (char*)PyBytesWriter_GetData(writer); + if (PyBytesWriter_Grow(writer, size) < 0) { + return NULL; + } + return (char*)PyBytesWriter_GetData(writer) + pos; + Overallocation -------------- From 79403f254ed1c2f3b3596fd016789511b8fa0fea Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 31 Mar 2025 14:09:35 +0200 Subject: [PATCH 14/15] Update --- peps/pep-0782.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/peps/pep-0782.rst b/peps/pep-0782.rst index 6c82009e95d..bc975c13c09 100644 --- a/peps/pep-0782.rst +++ b/peps/pep-0782.rst @@ -111,8 +111,8 @@ Create, Finish, Discard Py_ssize_t size = (char*)buf - (char*)PyBytesWriter_GetData(writer); return PyBytesWriter_FinishWithSize(writer, size); - Set an exception and return ``NULL`` if *buf* pointer is invalid: - outside the internal buffer bounds. + Set an exception and return ``NULL`` if *buf* pointer is outside the + internal buffer bounds. .. c:function:: void PyBytesWriter_Discard(PyBytesWriter *writer) @@ -233,7 +233,7 @@ Examples High-level API -------------- -Create the bytes ``b"Hello World!"``:: +Create the bytes string ``b"Hello World!"``:: PyObject* hello_world(void) { @@ -277,7 +277,7 @@ GrowAndUpdatePointer() example Example using a pointer to write bytes and to track the written size. -Create the string ``"Hello World"``:: +Create the bytes string ``b"Hello World"``:: PyObject* grow_example(void) { From 141f531033fe23adf0e1510e360294b4446644f7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 31 Mar 2025 14:18:19 +0200 Subject: [PATCH 15/15] Update --- peps/pep-0782.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/peps/pep-0782.rst b/peps/pep-0782.rst index bc975c13c09..b785882a1ed 100644 --- a/peps/pep-0782.rst +++ b/peps/pep-0782.rst @@ -258,7 +258,7 @@ Create the bytes string ``b"Hello World!"``:: Create the bytes string "abc" ----------------------------- -Example creating the bytes ``b"abc"``, with a fixed size of 3 bytes:: +Example creating the bytes string ``b"abc"``, with a fixed size of 3 bytes:: PyObject* create_abc(void) {