diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b53c3cc1f465ff..dcebcbb434643f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -424,6 +424,7 @@ Lib/test/test_dataclasses/ @ericvsmith # Dates and times Doc/**/*time.rst @pganssle @abalkin +Doc/library/datetime-* @pganssle Doc/library/zoneinfo.rst @pganssle Include/datetime.h @pganssle @abalkin Include/internal/pycore_time.h @pganssle @abalkin diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 138a5ca7a7516f..7a6f88d90a9ea5 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1852,6 +1852,8 @@ to the object: 13891296 +.. _faq-identity-with-is: + When can I rely on identity tests with the *is* operator? --------------------------------------------------------- diff --git a/Doc/library/datetime-inheritance.dot b/Doc/library/datetime-inheritance.dot new file mode 100644 index 00000000000000..3c6b9b4beb7ab1 --- /dev/null +++ b/Doc/library/datetime-inheritance.dot @@ -0,0 +1,31 @@ +// Used to generate datetime-inheritance.svg with Graphviz +// (https://graphviz.org/) for the datetime documentation. + +digraph { + comment="Generated with datetime-inheritance.dot" + graph [ + bgcolor="transparent" + fontnames="svg" + layout="dot" + ranksep=0.5 + nodesep=0.5 + splines=line + ] + node [ + fontname="Courier" + fontsize=14.0 + shape=box + style=rounded + margin="0.15,0.07" + ] + edge [ + arrowhead=none + ] + + object -> tzinfo + object -> timedelta + object -> time + object -> date + tzinfo -> timezone + date -> datetime +} diff --git a/Doc/library/datetime-inheritance.svg b/Doc/library/datetime-inheritance.svg new file mode 100644 index 00000000000000..e6b1cf877a574f --- /dev/null +++ b/Doc/library/datetime-inheritance.svg @@ -0,0 +1,84 @@ + + + + + + +datetime class hierarchy + + +object + +object + + + +tzinfo + +tzinfo + + + +object->tzinfo + + + + +timedelta + +timedelta + + + +object->timedelta + + + + +time + +time + + + +object->time + + + + +date + +date + + + +object->date + + + + +timezone + +timezone + + + +tzinfo->timezone + + + + +datetime + +datetime + + + +date->datetime + + + + diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 39a7a1530a95cc..b806a49e1be903 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -12,8 +12,6 @@ -------------- -.. XXX what order should the types be discussed in? - The :mod:`!datetime` module supplies classes for manipulating dates and times. While date and time arithmetic is supported, the focus of the implementation is @@ -38,13 +36,14 @@ on efficient attribute extraction for output formatting and manipulation. Third-party library with expanded time zone and parsing support. Package :pypi:`DateType` - Third-party library that introduces distinct static types to e.g. allow - :term:`static type checkers ` + Third-party library that introduces distinct static types to for example, + allow :term:`static type checkers ` to differentiate between naive and aware datetimes. + .. _datetime-naive-aware: -Aware and Naive Objects +Aware and naive objects ----------------------- Date and time objects may be categorized as "aware" or "naive" depending on @@ -77,6 +76,7 @@ detail is up to the application. The rules for time adjustment across the world are more political than rational, change frequently, and there is no standard suitable for every application aside from UTC. + Constants --------- @@ -93,13 +93,15 @@ The :mod:`!datetime` module exports the following constants: The largest year number allowed in a :class:`date` or :class:`.datetime` object. :const:`MAXYEAR` is 9999. + .. data:: UTC Alias for the UTC time zone singleton :attr:`datetime.timezone.utc`. .. versionadded:: 3.11 -Available Types + +Available types --------------- .. class:: date @@ -142,6 +144,7 @@ Available Types time adjustment (for example, to account for time zone and/or daylight saving time). + .. class:: timezone :noindex: @@ -150,19 +153,19 @@ Available Types .. versionadded:: 3.2 + Objects of these types are immutable. -Subclass relationships:: +Subclass relationships: + +.. figure:: datetime-inheritance.svg + :class: invert-in-dark-mode + :align: center + :alt: timedelta, tzinfo, time, and date inherit from object; timezone inherits + from tzinfo; and datetime inherits from date. - object - timedelta - tzinfo - timezone - time - date - datetime -Common Properties +Common properties ^^^^^^^^^^^^^^^^^ The :class:`date`, :class:`.datetime`, :class:`.time`, and :class:`timezone` types @@ -173,7 +176,8 @@ share these common features: dictionary keys. - Objects of these types support efficient pickling via the :mod:`pickle` module. -Determining if an Object is Aware or Naive + +Determining if an object is aware or naive ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Objects of the :class:`date` type are always naive. @@ -197,10 +201,11 @@ Otherwise, ``t`` is naive. The distinction between aware and naive doesn't apply to :class:`timedelta` objects. + .. _datetime-timedelta: -:class:`timedelta` Objects --------------------------- +:class:`!timedelta` objects +--------------------------- A :class:`timedelta` object represents a duration, the difference between two :class:`.datetime` or :class:`date` instances. @@ -296,6 +301,7 @@ Class attributes: The smallest possible difference between non-equal :class:`timedelta` objects, ``timedelta(microseconds=1)``. + Note that, because of normalization, ``timedelta.max`` is greater than ``-timedelta.min``. ``-timedelta.max`` is not representable as a :class:`timedelta` object. @@ -326,6 +332,7 @@ Instance attributes (read-only): >>> duration.total_seconds() 11235813.0 + .. attribute:: timedelta.microseconds Between 0 and 999,999 inclusive. @@ -333,8 +340,6 @@ Instance attributes (read-only): Supported operations: -.. XXX this table is too wide! - +--------------------------------+-----------------------------------------------+ | Operation | Result | +================================+===============================================+ @@ -396,7 +401,6 @@ Supported operations: | | call with canonical attribute values. | +--------------------------------+-----------------------------------------------+ - Notes: (1) @@ -447,15 +451,16 @@ Instance methods: Return the total number of seconds contained in the duration. Equivalent to ``td / timedelta(seconds=1)``. For interval units other than seconds, use the - division form directly (e.g. ``td / timedelta(microseconds=1)``). + division form directly (for example, ``td / timedelta(microseconds=1)``). Note that for very large time intervals (greater than 270 years on most platforms) this method will lose microsecond accuracy. .. versionadded:: 3.2 -Examples of usage: :class:`timedelta` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Examples of usage: :class:`!timedelta` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ An additional example of normalization:: @@ -485,10 +490,11 @@ Examples of :class:`timedelta` arithmetic:: >>> three_years, three_years.days // 365 (datetime.timedelta(days=1095), 3) + .. _datetime-date: -:class:`date` Objects ---------------------- +:class:`!date` objects +---------------------- A :class:`date` object represents a date (year, month and day) in an idealized calendar, the current Gregorian calendar indefinitely extended in both @@ -517,9 +523,10 @@ Other constructors, all class methods: This is equivalent to ``date.fromtimestamp(time.time())``. + .. classmethod:: date.fromtimestamp(timestamp) - Return the local date corresponding to the POSIX timestamp, such as is + Return the local date corresponding to the POSIX *timestamp*, such as is returned by :func:`time.time`. This may raise :exc:`OverflowError`, if the timestamp is out @@ -541,7 +548,7 @@ Other constructors, all class methods: .. classmethod:: date.fromordinal(ordinal) - Return the date corresponding to the proleptic Gregorian ordinal, where + Return the date corresponding to the proleptic Gregorian *ordinal*, where January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1 <= ordinal <= @@ -574,13 +581,15 @@ Other constructors, all class methods: .. versionchanged:: 3.11 Previously, this method only supported the format ``YYYY-MM-DD``. + .. classmethod:: date.fromisocalendar(year, week, day) Return a :class:`date` corresponding to the ISO calendar date specified by - year, week and day. This is the inverse of the function :meth:`date.isocalendar`. + *year*, *week* and *day*. This is the inverse of the function :meth:`date.isocalendar`. .. versionadded:: 3.8 + .. classmethod:: date.strptime(date_string, format) Return a :class:`.date` corresponding to *date_string*, parsed according to @@ -791,6 +800,7 @@ Instance methods: .. versionchanged:: 3.9 Result changed from a tuple to a :term:`named tuple`. + .. method:: date.isoformat() Return a string representing the date in ISO 8601 format, ``YYYY-MM-DD``:: @@ -799,6 +809,7 @@ Instance methods: >>> date(2002, 12, 4).isoformat() '2002-12-04' + .. method:: date.__str__() For a date ``d``, ``str(d)`` is equivalent to ``d.isoformat()``. @@ -835,8 +846,9 @@ Instance methods: literals ` and when using :meth:`str.format`. See also :ref:`strftime-strptime-behavior` and :meth:`date.isoformat`. -Examples of Usage: :class:`date` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Examples of usage: :class:`!date` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Example of counting days to an event:: @@ -878,7 +890,7 @@ More examples of working with :class:`date`: >>> 'The {1} is {0:%d}, the {2} is {0:%B}.'.format(d, "day", "month") 'The day is 11, the month is March.' - >>> # Methods for to extracting 'components' under different calendars + >>> # Methods for extracting 'components' under different calendars >>> t = d.timetuple() >>> for i in t: # doctest: +SKIP ... print(i) @@ -905,7 +917,7 @@ More examples of working with :class:`date`: .. _datetime-datetime: -:class:`.datetime` Objects +:class:`!datetime` objects -------------------------- A :class:`.datetime` object is a single object containing all the information @@ -937,6 +949,7 @@ Constructor: .. versionchanged:: 3.6 Added the *fold* parameter. + Other constructors, all class methods: .. classmethod:: datetime.today() @@ -952,6 +965,7 @@ Other constructors, all class methods: This method is functionally equivalent to :meth:`now`, but without a ``tz`` parameter. + .. classmethod:: datetime.now(tz=None) Return the current local date and time. @@ -972,6 +986,7 @@ Other constructors, all class methods: Subsequent calls to :meth:`!datetime.now` may return the same instant depending on the precision of the underlying clock. + .. classmethod:: datetime.utcnow() Return the current UTC date and time, with :attr:`.tzinfo` ``None``. @@ -1063,13 +1078,13 @@ Other constructors, all class methods: :c:func:`gmtime` function. Raise :exc:`OSError` instead of :exc:`ValueError` on :c:func:`gmtime` failure. + .. versionchanged:: 3.15 + Accepts any real number as *timestamp*, not only integer or float. + .. deprecated:: 3.12 Use :meth:`datetime.fromtimestamp` with :const:`UTC` instead. - .. versionchanged:: 3.15 - Accepts any real number as *timestamp*, not only integer or float. - .. classmethod:: datetime.fromordinal(ordinal) @@ -1142,12 +1157,13 @@ Other constructors, all class methods: .. classmethod:: datetime.fromisocalendar(year, week, day) Return a :class:`.datetime` corresponding to the ISO calendar date specified - by year, week and day. The non-date components of the datetime are populated + by *year*, *week* and *day*. The non-date components of the datetime are populated with their normal default values. This is the inverse of the function :meth:`datetime.isocalendar`. .. versionadded:: 3.8 + .. classmethod:: datetime.strptime(date_string, format) Return a :class:`.datetime` corresponding to *date_string*, parsed according to @@ -1255,6 +1271,7 @@ Instance attributes (read-only): .. versionadded:: 3.6 + Supported operations: +---------------------------------------+--------------------------------+ @@ -1345,6 +1362,7 @@ Supported operations: The default behavior can be changed by overriding the special comparison methods in subclasses. + Instance methods: .. method:: datetime.date() @@ -1500,11 +1518,13 @@ Instance methods: ``datetime.replace(tzinfo=timezone.utc)`` to make it aware, at which point you can use :meth:`.datetime.timetuple`. + .. method:: datetime.toordinal() Return the proleptic Gregorian ordinal of the date. The same as ``self.date().toordinal()``. + .. method:: datetime.timestamp() Return POSIX timestamp corresponding to the :class:`.datetime` @@ -1523,16 +1543,6 @@ Instance methods: (dt - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() - .. versionadded:: 3.3 - - .. versionchanged:: 3.6 - The :meth:`timestamp` method uses the :attr:`.fold` attribute to - disambiguate the times during a repeated interval. - - .. versionchanged:: 3.6 - This method no longer relies on the platform C :c:func:`mktime` - function to perform conversions. - .. note:: There is no method to obtain the POSIX timestamp directly from a @@ -1547,6 +1557,17 @@ Instance methods: timestamp = (dt - datetime(1970, 1, 1)) / timedelta(seconds=1) + .. versionadded:: 3.3 + + .. versionchanged:: 3.6 + The :meth:`timestamp` method uses the :attr:`.fold` attribute to + disambiguate the times during a repeated interval. + + .. versionchanged:: 3.6 + This method no longer relies on the platform C :c:func:`mktime` + function to perform conversions. + + .. method:: datetime.weekday() Return the day of the week as an integer, where Monday is 0 and Sunday is 6. @@ -1675,7 +1696,7 @@ Instance methods: See also :ref:`strftime-strptime-behavior` and :meth:`datetime.isoformat`. -Examples of Usage: :class:`.datetime` +Examples of usage: :class:`!datetime` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Examples of working with :class:`.datetime` objects: @@ -1801,9 +1822,10 @@ Usage of ``KabulTz`` from above:: >>> dt2 == dt3 True + .. _datetime-time: -:class:`.time` Objects +:class:`!time` objects ---------------------- A :class:`.time` object represents a (local) time of day, independent of any particular @@ -1824,6 +1846,7 @@ day, and subject to adjustment via a :class:`tzinfo` object. If an argument outside those ranges is given, :exc:`ValueError` is raised. All default to 0 except *tzinfo*, which defaults to ``None``. + Class attributes: @@ -1882,6 +1905,7 @@ Instance attributes (read-only): .. versionadded:: 3.6 + :class:`.time` objects support equality and order comparisons, where ``a`` is considered less than ``b`` when ``a`` precedes ``b`` in time. @@ -1904,8 +1928,8 @@ In Boolean contexts, a :class:`.time` object is always considered to be true. .. versionchanged:: 3.5 Before Python 3.5, a :class:`.time` object was considered to be false if it represented midnight in UTC. This behavior was considered obscure and - error-prone and has been removed in Python 3.5. See :issue:`13936` for full - details. + error-prone and has been removed in Python 3.5. See :issue:`13936` for more + information. Other constructors: @@ -1950,6 +1974,7 @@ Other constructors: Previously, this method only supported formats that could be emitted by :meth:`time.isoformat`. + .. classmethod:: time.strptime(date_string, format) Return a :class:`.time` corresponding to *date_string*, parsed according to @@ -2066,13 +2091,15 @@ Instance methods: .. versionchanged:: 3.7 The DST offset is not restricted to a whole number of minutes. + .. method:: time.tzname() If :attr:`.tzinfo` is ``None``, returns ``None``, else returns ``self.tzinfo.tzname(None)``, or raises an exception if the latter doesn't return ``None`` or a string object. -Examples of Usage: :class:`.time` + +Examples of usage: :class:`!time` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Examples of working with a :class:`.time` object:: @@ -2105,12 +2132,12 @@ Examples of working with a :class:`.time` object:: .. _datetime-tzinfo: -:class:`tzinfo` Objects ------------------------ +:class:`!tzinfo` objects +------------------------ .. class:: tzinfo() - This is an abstract base class, meaning that this class should not be + This is an :term:`abstract base class`, meaning that this class should not be instantiated directly. Define a subclass of :class:`tzinfo` to capture information about a particular time zone. @@ -2381,8 +2408,8 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. _datetime-timezone: -:class:`timezone` Objects -------------------------- +:class:`!timezone` objects +-------------------------- The :class:`timezone` class is a subclass of :class:`tzinfo`, each instance of which represents a time zone defined by a fixed offset from @@ -2420,6 +2447,7 @@ where historical changes have been made to civil time. .. versionchanged:: 3.7 The UTC offset is not restricted to a whole number of minutes. + .. method:: timezone.tzname(dt) Return the fixed value specified when the :class:`timezone` instance @@ -2440,11 +2468,13 @@ where historical changes have been made to civil time. Always returns ``None``. + .. method:: timezone.fromutc(dt) Return ``dt + offset``. The *dt* argument must be an aware :class:`.datetime` instance, with ``tzinfo`` set to ``self``. + Class attributes: .. attribute:: timezone.utc @@ -2457,8 +2487,8 @@ Class attributes: .. _strftime-strptime-behavior: -:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Behavior --------------------------------------------------------------------- +:meth:`!strftime` and :meth:`!strptime` behavior +------------------------------------------------ :class:`date`, :class:`.datetime`, and :class:`.time` objects all support a ``strftime(format)`` method, to create a string representing the time under the @@ -2484,8 +2514,8 @@ versus :meth:`~.datetime.strptime`: .. _format-codes: -:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Format Codes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:meth:`!strftime` and :meth:`!strptime` format codes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ These methods accept format codes that can be used to parse and format dates:: @@ -2678,7 +2708,8 @@ differences between platforms in handling of unsupported format specifiers. .. versionadded:: 3.15 ``%:z``, ``%F``, and ``%D`` were added for :meth:`~.datetime.strptime`. -Technical Detail + +Technical detail ^^^^^^^^^^^^^^^^ Broadly speaking, ``d.strftime(fmt)`` acts like the :mod:`time` module's @@ -2701,7 +2732,6 @@ in the format string will be pulled from the default value. the default year of 1900 is *not* a leap year. Always add a default leap year to partial date strings before parsing. - .. testsetup:: # doctest seems to turn the warning into an error which makes it diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index a8f693f4879025..b9b81a7d469d0d 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -265,9 +265,17 @@ The constructors :func:`int`, :func:`float`, and pair: operator; % (percent) pair: operator; ** +.. _stdtypes-mixed-arithmetic: + Python fully supports mixed arithmetic: when a binary arithmetic operator has -operands of different numeric types, the operand with the "narrower" type is -widened to that of the other, where integer is narrower than floating point. +operands of different built-in numeric types, the operand with the "narrower" +type is widened to that of the other: + +* If both arguments are complex numbers, no conversion is performed; +* if either argument is a complex or a floating-point number, the other is + converted to a floating-point number; +* otherwise, both must be integers and no conversion is necessary. + Arithmetic with complex and real operands is defined by the usual mathematical formula, for example:: diff --git a/Doc/library/token.rst b/Doc/library/token.rst index c228006d4c1e1d..fb826f5465bd80 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -50,8 +50,7 @@ The token constants are: .. data:: NAME - Token value that indicates an :ref:`identifier `. - Note that keywords are also initially tokenized as ``NAME`` tokens. + Token value that indicates an :ref:`identifier or keyword `. .. data:: NUMBER diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 54384a8cf3fb90..68dcfc00bbd99c 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -9,9 +9,11 @@ Expressions This chapter explains the meaning of the elements of expressions in Python. -**Syntax Notes:** In this and the following chapters, extended BNF notation will -be used to describe syntax, not lexical analysis. When (one alternative of) a -syntax rule has the form +**Syntax Notes:** In this and the following chapters, +:ref:`grammar notation ` will be used to describe syntax, +not lexical analysis. + +When (one alternative of) a syntax rule has the form: .. productionlist:: python-grammar name: othername @@ -29,17 +31,13 @@ Arithmetic conversions When a description of an arithmetic operator below uses the phrase "the numeric arguments are converted to a common real type", this means that the operator -implementation for built-in types works as follows: - -* If both arguments are complex numbers, no conversion is performed; - -* if either argument is a complex or a floating-point number, the other is converted to a floating-point number; - -* otherwise, both must be integers and no conversion is necessary. +implementation for built-in numeric types works as described in the +:ref:`Numeric Types ` section of the standard +library documentation. -Some additional rules apply for certain operators (e.g., a string as a left -argument to the '%' operator). Extensions must define their own conversion -behavior. +Some additional rules apply for certain operators and non-numeric operands +(for example, a string as a left argument to the ``%`` operator). +Extensions must define their own conversion behavior. .. _atoms: @@ -49,15 +47,57 @@ Atoms .. index:: atom -Atoms are the most basic elements of expressions. The simplest atoms are -identifiers or literals. Forms enclosed in parentheses, brackets or braces are -also categorized syntactically as atoms. The syntax for atoms is: +Atoms are the most basic elements of expressions. +The simplest atoms are :ref:`names ` or literals. +Forms enclosed in parentheses, brackets or braces are also categorized +syntactically as atoms. -.. productionlist:: python-grammar - atom: `identifier` | `literal` | `enclosure` - enclosure: `parenth_form` | `list_display` | `dict_display` | `set_display` - : | `generator_expression` | `yield_atom` +Formally, the syntax for atoms is: + +.. grammar-snippet:: + :group: python-grammar + + atom: + | 'True' + | 'False' + | 'None' + | '...' + | `identifier` + | `literal` + | `enclosure` + enclosure: + | `parenth_form` + | `list_display` + | `dict_display` + | `set_display` + | `generator_expression` + | `yield_atom` + + +.. _atom-singletons: + +Built-in constants +------------------ + +The keywords ``True``, ``False``, and ``None`` name +:ref:`built-in constants `. +The token ``...`` names the :py:data:`Ellipsis` constant. +Evaluation of these atoms yields the corresponding value. + +.. note:: + + Several more built-in constants are available as global variables, + but only the ones mentioned here are :ref:`keywords `. + In particular, these names cannot be reassigned or used as attributes: + + .. code-block:: pycon + + >>> False = 123 + File "", line 1 + False = 123 + ^^^^^ + SyntaxError: cannot assign to False .. _atom-identifiers: @@ -131,51 +171,104 @@ Literals .. index:: single: literal -Python supports string and bytes literals and various numeric literals: +A :dfn:`literal` is a textual representation of a value. +Python supports numeric, string and bytes literals. +:ref:`Format strings ` and :ref:`template strings ` +are treated as string literals. + +Numeric literals consist of a single :token:`NUMBER ` +token, which names an integer, floating-point number, or an imaginary number. +See the :ref:`numbers` section in Lexical analysis documentation for details. + +String and bytes literals may consist of several tokens. +See section :ref:`string-concatenation` for details. + +Note that negative and complex numbers, like ``-3`` or ``3+4.2j``, +are syntactically not literals, but :ref:`unary ` or +:ref:`binary ` arithmetic operations involving the ``-`` or ``+`` +operator. + +Evaluation of a literal yields an object of the given type +(:class:`int`, :class:`float`, :class:`complex`, :class:`str`, +:class:`bytes`, or :class:`~string.templatelib.Template`) with the given value. +The value may be approximated in the case of floating-point +and imaginary literals. + +The formal grammar for literals is: .. grammar-snippet:: :group: python-grammar literal: `strings` | `NUMBER` -Evaluation of a literal yields an object of the given type (string, bytes, -integer, floating-point number, complex number) with the given value. The value -may be approximated in the case of floating-point and imaginary (complex) -literals. -See section :ref:`literals` for details. -See section :ref:`string-concatenation` for details on ``strings``. - .. index:: triple: immutable; data; type pair: immutable; object +Literals and object identity +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + All literals correspond to immutable data types, and hence the object's identity is less important than its value. Multiple evaluations of literals with the same value (either the same occurrence in the program text or a different occurrence) may obtain the same object or a different object with the same value. +.. admonition:: CPython implementation detail + + For example, in CPython, *small* integers with the same value evaluate + to the same object:: + + >>> x = 7 + >>> y = 7 + >>> x is y + True + + However, large integers evaluate to different objects:: + + >>> x = 123456789 + >>> y = 123456789 + >>> x is y + False + + This behavior may change in future versions of CPython. + In particular, the boundary between "small" and "large" integers has + already changed in the past. + + CPython will emit a :py:exc:`SyntaxWarning` when you compare literals + using ``is``:: + + >>> x = 7 + >>> x is 7 + :1: SyntaxWarning: "is" with 'int' literal. Did you mean "=="? + True + + See :ref:`faq-identity-with-is` for more information. + +:ref:`Template strings ` are immutable but may reference mutable +objects as :class:`~string.templatelib.Interpolation` values. +For the purposes of this section, two t-strings have the "same value" if +both their structure and the *identity* of the values match. + +.. impl-detail:: + + Currently, each evaluation of a template string results in + a different object. + .. _string-concatenation: String literal concatenation ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Multiple adjacent string or bytes literals (delimited by whitespace), possibly +Multiple adjacent string or bytes literals, possibly using different quoting conventions, are allowed, and their meaning is the same as their concatenation:: >>> "hello" 'world' "helloworld" -Formally: - -.. grammar-snippet:: - :group: python-grammar - - strings: ( `STRING` | `fstring`)+ | `tstring`+ - This feature is defined at the syntactical level, so it only works with literals. To concatenate string expressions at run time, the '+' operator may be used:: @@ -208,6 +301,13 @@ string literals:: >>> t"Hello" t"{name}!" Template(strings=('Hello', '!'), interpolations=(...)) +Formally: + +.. grammar-snippet:: + :group: python-grammar + + strings: (`STRING` | `fstring`)+ | `tstring`+ + .. _parenthesized: @@ -1390,8 +1490,9 @@ for the operands): ``-1**2`` results in ``-1``. The power operator has the same semantics as the built-in :func:`pow` function, when called with two arguments: it yields its left argument raised to the power -of its right argument. The numeric arguments are first converted to a common -type, and the result is of that type. +of its right argument. +Numeric arguments are first :ref:`converted to a common type `, +and the result is of that type. For int operands, the result has the same type as the operands unless the second argument is negative; in that case, all arguments are converted to float and a @@ -1477,9 +1578,10 @@ operators and one for additive operators: The ``*`` (multiplication) operator yields the product of its arguments. The arguments must either both be numbers, or one argument must be an integer and -the other must be a sequence. In the former case, the numbers are converted to a -common real type and then multiplied together. In the latter case, sequence -repetition is performed; a negative repetition factor yields an empty sequence. +the other must be a sequence. In the former case, the numbers are +:ref:`converted to a common real type ` and then +multiplied together. In the latter case, sequence repetition is performed; +a negative repetition factor yields an empty sequence. This operation can be customized using the special :meth:`~object.__mul__` and :meth:`~object.__rmul__` methods. @@ -1507,7 +1609,8 @@ This operation can be customized using the special :meth:`~object.__matmul__` an pair: operator; // The ``/`` (division) and ``//`` (floor division) operators yield the quotient of -their arguments. The numeric arguments are first converted to a common type. +their arguments. The numeric arguments are first +:ref:`converted to a common type `. Division of integers yields a float, while floor division of integers results in an integer; the result is that of mathematical division with the 'floor' function applied to the result. Division by zero raises the :exc:`ZeroDivisionError` @@ -1523,8 +1626,9 @@ The floor division operation can be customized using the special pair: operator; % (percent) The ``%`` (modulo) operator yields the remainder from the division of the first -argument by the second. The numeric arguments are first converted to a common -type. A zero right argument raises the :exc:`ZeroDivisionError` exception. The +argument by the second. The numeric arguments are first +:ref:`converted to a common type `. +A zero right argument raises the :exc:`ZeroDivisionError` exception. The arguments may be floating-point numbers, e.g., ``3.14%0.7`` equals ``0.34`` (since ``3.14`` equals ``4*0.7 + 0.34``.) The modulo operator always yields a result with the same sign as its second operand (or zero); the absolute value of @@ -1555,7 +1659,9 @@ floating-point number using the :func:`abs` function if appropriate. The ``+`` (addition) operator yields the sum of its arguments. The arguments must either both be numbers or both be sequences of the same type. In the -former case, the numbers are converted to a common real type and then added together. +former case, the numbers are +:ref:`converted to a common real type ` and then +added together. In the latter case, the sequences are concatenated. This operation can be customized using the special :meth:`~object.__add__` and @@ -1570,8 +1676,9 @@ This operation can be customized using the special :meth:`~object.__add__` and single: operator; - (minus) single: - (minus); binary operator -The ``-`` (subtraction) operator yields the difference of its arguments. The -numeric arguments are first converted to a common real type. +The ``-`` (subtraction) operator yields the difference of its arguments. +The numeric arguments are first +:ref:`converted to a common real type `. This operation can be customized using the special :meth:`~object.__sub__` and :meth:`~object.__rsub__` methods. diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 3ebc8967c3dc7f..1e69c64bcd1fc0 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -496,7 +496,7 @@ struct _py_func_state { /* For now we hard-code this to a value for which we are confident all the static builtin types will fit (for all builds). */ -#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES 200 +#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES 201 #define _Py_MAX_MANAGED_STATIC_EXT_TYPES 10 #define _Py_MAX_MANAGED_STATIC_TYPES \ (_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES + _Py_MAX_MANAGED_STATIC_EXT_TYPES) diff --git a/Lib/copy.py b/Lib/copy.py index 4c024ab5311d2d..33dabb3395a7c0 100644 --- a/Lib/copy.py +++ b/Lib/copy.py @@ -101,7 +101,7 @@ def copy(x): _copy_atomic_types = frozenset({types.NoneType, int, float, bool, complex, str, tuple, - bytes, frozenset, type, range, slice, property, + bytes, frozendict, frozenset, type, range, slice, property, types.BuiltinFunctionType, types.EllipsisType, types.NotImplementedType, types.FunctionType, types.CodeType, weakref.ref, super}) @@ -203,6 +203,11 @@ def _deepcopy_dict(x, memo, deepcopy=deepcopy): return y d[dict] = _deepcopy_dict +def _deepcopy_frozendict(x, memo, deepcopy=deepcopy): + y = _deepcopy_dict(x, memo, deepcopy) + return frozendict(y) +d[frozendict] = _deepcopy_frozendict + def _deepcopy_method(x, memo): # Copy instance methods return type(x)(x.__func__, deepcopy(x.__self__, memo)) d[types.MethodType] = _deepcopy_method diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index c4460c2e44d578..3d4ed8a2b6ee40 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1484,6 +1484,29 @@ def __hash__(self): # bad hashable dict key self.check_unpickling_error(CustomError, base + b'}c__main__\nBadKey1\n)\x81Nsb.') + def test_bad_types(self): + # APPEND + self.assertEqual(self.loads(b']Na.'), [None]) + self.check_unpickling_error(AttributeError, b'NNa.') # non-list + # APPENDS + self.assertEqual(self.loads(b'](Ne.'), [None]) + self.check_unpickling_error(AttributeError, b'N(Ne.') # non-list + self.check_unpickling_error(AttributeError, b'N(e.') + # SETITEM + self.assertEqual(self.loads(b'}NNs.'), {None: None}) + self.check_unpickling_error(TypeError, b'NNNs.') # non-dict + self.check_unpickling_error(TypeError, b'}]Ns.') # non-hashable key + # SETITEMS + self.assertEqual(self.loads(b'}(NNu.'), {None: None}) + self.check_unpickling_error(TypeError, b'N(NNu.') # non-dict + self.assertEqual(self.loads(b'N(u.'), None) # no validation for empty items + self.check_unpickling_error(TypeError, b'}(]Nu.') # non-hashable key + # ADDITEMS + self.assertEqual(self.loads(b'\x8f(N\x90.'), {None}) + self.check_unpickling_error(AttributeError, b'N(N\x90.') # non-set + self.check_unpickling_error(AttributeError, b'N(\x90.') + self.check_unpickling_error(TypeError, b'\x8f(]\x90.') # non-hashable element + def test_bad_stack(self): badpickles = [ b'.', # STOP diff --git a/Lib/test/test__interpchannels.py b/Lib/test/test__interpchannels.py index d7cf77368ef9f2..2b0aba42896c06 100644 --- a/Lib/test/test__interpchannels.py +++ b/Lib/test/test__interpchannels.py @@ -382,6 +382,38 @@ def test_sequential_ids(self): self.assertEqual(id3, int(id2) + 1) self.assertEqual(set(after) - set(before), {id1, id2, id3}) + def test_channel_list_all_closed(self): + id1 = _channels.create() + id2 = _channels.create() + id3 = _channels.create() + before = _channels.list_all() + expected = [info for info in before if info[0] != id2] + _channels.close(id2, force=True) + after = _channels.list_all() + self.assertEqual(set(after), set(expected)) + self.assertEqual(len(after), len(before) - 1) + + def test_channel_list_all_destroyed(self): + id1 = _channels.create() + id2 = _channels.create() + id3 = _channels.create() + before = _channels.list_all() + expected = [info for info in before if info[0] != id2] + _channels.destroy(id2) + after = _channels.list_all() + self.assertEqual(set(after), set(expected)) + self.assertEqual(len(after), len(before) - 1) + + def test_channel_list_all_released(self): + id1 = _channels.create() + id2 = _channels.create() + id3 = _channels.create() + before = _channels.list_all() + _channels.release(id2, send=True, recv=True) + after = _channels.list_all() + self.assertEqual(set(after), set(before)) + self.assertEqual(len(after), len(before)) + def test_ids_global(self): id1 = _interpreters.create() out = _run_output(id1, dedent(""" diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index cfef24727e8c82..858e5e089d5aba 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -133,6 +133,12 @@ def test_copy_dict(self): self.assertEqual(y, x) self.assertIsNot(y, x) + def test_copy_frozendict(self): + x = frozendict(x=1, y=2) + self.assertIs(copy.copy(x), x) + x = frozendict() + self.assertIs(copy.copy(x), x) + def test_copy_set(self): x = {1, 2, 3} y = copy.copy(x) @@ -419,6 +425,13 @@ def test_deepcopy_dict(self): self.assertIsNot(x, y) self.assertIsNot(x["foo"], y["foo"]) + def test_deepcopy_frozendict(self): + x = frozendict({"foo": [1, 2], "bar": 3}) + y = copy.deepcopy(x) + self.assertEqual(y, x) + self.assertIsNot(x, y) + self.assertIsNot(x["foo"], y["foo"]) + @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() def test_deepcopy_reflexive_dict(self): diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 21f8bb11071c90..71f72cb2557670 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -1787,6 +1787,44 @@ def test_hash(self): with self.assertRaisesRegex(TypeError, "unhashable type: 'list'"): hash(fd) + def test_fromkeys(self): + self.assertEqual(frozendict.fromkeys('abc'), + frozendict(a=None, b=None, c=None)) + + # Subclass which overrides the constructor + created = frozendict(x=1) + class FrozenDictSubclass(frozendict): + def __new__(self): + return created + + fd = FrozenDictSubclass.fromkeys("abc") + self.assertEqual(fd, frozendict(x=1, a=None, b=None, c=None)) + self.assertEqual(type(fd), FrozenDictSubclass) + self.assertEqual(created, frozendict(x=1)) + + fd = FrozenDictSubclass.fromkeys(frozendict(y=2)) + self.assertEqual(fd, frozendict(x=1, y=None)) + self.assertEqual(type(fd), FrozenDictSubclass) + self.assertEqual(created, frozendict(x=1)) + + # Subclass which doesn't override the constructor + class FrozenDictSubclass2(frozendict): + pass + + fd = FrozenDictSubclass2.fromkeys("abc") + self.assertEqual(fd, frozendict(a=None, b=None, c=None)) + self.assertEqual(type(fd), FrozenDictSubclass2) + + # Dict subclass which overrides the constructor + class DictSubclass(dict): + def __new__(self): + return created + + fd = DictSubclass.fromkeys("abc") + self.assertEqual(fd, frozendict(x=1, a=None, b=None, c=None)) + self.assertEqual(type(fd), DictSubclass) + self.assertEqual(created, frozendict(x=1)) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index 52827357078b85..5437792b5a7014 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -47,6 +47,12 @@ def test_list_all(self): after = set(channels.list_all()) self.assertEqual(after, created) + def test_list_all_closed(self): + created = [channels.create() for _ in range(3)] + rch, sch = created.pop(1) + rch.close() + self.assertEqual(set(channels.list_all()), set(created)) + def test_shareable(self): interp = interpreters.create() rch, sch = channels.create() diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 343711ce3a9cef..1f5b0596107704 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1548,6 +1548,27 @@ def contains_op_dict(): self.assert_specialized(contains_op_dict, "CONTAINS_OP_DICT") self.assert_no_opcode(contains_op_dict, "CONTAINS_OP") + def contains_op_frozen_dict(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = 1, frozendict({1: 2, 2: 5}) + self.assertTrue(a in b) + self.assertFalse(3 in b) + + contains_op_frozen_dict() + self.assert_specialized(contains_op_frozen_dict, "CONTAINS_OP_DICT") + self.assert_no_opcode(contains_op_frozen_dict, "CONTAINS_OP") + + def contains_op_frozen_dict_subclass(): + class MyFrozenDict(frozendict): + pass + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a, b = 1, MyFrozenDict({1: 2, 2: 5}) + self.assertTrue(a in b) + self.assertFalse(3 in b) + + contains_op_frozen_dict_subclass() + self.assert_no_opcode(contains_op_frozen_dict_subclass, "CONTAINS_OP_DICT") + def contains_op_set(): for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a, b = 1, {1, 2} @@ -1808,6 +1829,27 @@ def binary_subscr_dict(): self.assert_specialized(binary_subscr_dict, "BINARY_OP_SUBSCR_DICT") self.assert_no_opcode(binary_subscr_dict, "BINARY_OP") + def binary_subscr_frozen_dict(): + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = frozendict({1: 2, 2: 3}) + self.assertEqual(a[1], 2) + self.assertEqual(a[2], 3) + + binary_subscr_frozen_dict() + self.assert_specialized(binary_subscr_frozen_dict, "BINARY_OP_SUBSCR_DICT") + self.assert_no_opcode(binary_subscr_frozen_dict, "BINARY_OP") + + def binary_subscr_frozen_dict_subclass(): + class MyFrozenDict(frozendict): + pass + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + a = MyFrozenDict({1: 2, 2: 3}) + self.assertEqual(a[1], 2) + self.assertEqual(a[2], 3) + + binary_subscr_frozen_dict_subclass() + self.assert_no_opcode(binary_subscr_frozen_dict_subclass, "BINARY_OP_SUBSCR_DICT") + def binary_subscr_str_int(): for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): a = "foobar" diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index 554716aed1e98b..9bfd4bc7d63669 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -188,7 +188,10 @@ def test_symmetric_difference(self): self.assertEqual(type(i), self.basetype) self.assertRaises(PassThru, self.s.symmetric_difference, check_pass_thru()) self.assertRaises(TypeError, self.s.symmetric_difference, [[]]) - for C in set, frozenset, dict.fromkeys, str, list, tuple: + constructors = (set, frozenset, + dict.fromkeys, frozendict.fromkeys, + str, list, tuple) + for C in constructors: self.assertEqual(self.thetype('abcba').symmetric_difference(C('cdc')), set('abd')) self.assertEqual(self.thetype('abcba').symmetric_difference(C('efgfe')), set('abcefg')) self.assertEqual(self.thetype('abcba').symmetric_difference(C('ccb')), set('a')) @@ -1591,6 +1594,14 @@ def setUp(self): #------------------------------------------------------------------------------ +class TestOnlySetsFrozenDict(TestOnlySetsInBinaryOps, unittest.TestCase): + def setUp(self): + self.set = set((1, 2, 3)) + self.other = frozendict({1:2, 3:4}) + self.otherIsIterable = True + +#------------------------------------------------------------------------------ + class TestOnlySetsOperator(TestOnlySetsInBinaryOps, unittest.TestCase): def setUp(self): self.set = set((1, 2, 3)) diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 47f6b46061ac30..b187643e84521c 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -1815,6 +1815,11 @@ def test_weak_valued_union_operators(self): def test_weak_keyed_dict_update(self): self.check_update(weakref.WeakKeyDictionary, {C(): 1, C(): 2, C(): 3}) + d = weakref.WeakKeyDictionary() + msg = ("Keyword arguments are not supported: " + "cannot create weak reference to 'str' object") + with self.assertRaisesRegex(TypeError, msg): + d.update(k='v') def test_weak_keyed_delitem(self): d = weakref.WeakKeyDictionary() diff --git a/Lib/weakref.py b/Lib/weakref.py index 94e4278143c987..af7244553c908c 100644 --- a/Lib/weakref.py +++ b/Lib/weakref.py @@ -408,14 +408,16 @@ def setdefault(self, key, default=None): return self.data.setdefault(ref(key, self._remove),default) def update(self, dict=None, /, **kwargs): + if kwargs: + msg = ("Keyword arguments are not supported: " + "cannot create weak reference to 'str' object") + raise TypeError(msg) d = self.data if dict is not None: if not hasattr(dict, "items"): dict = type({})(dict) for key, value in dict.items(): d[ref(key, self._remove)] = value - if len(kwargs): - self.update(kwargs) def __ior__(self, other): self.update(other) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-02-18-21-44-39.gh-issue-141510.7LST2O.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-02-18-21-44-39.gh-issue-141510.7LST2O.rst new file mode 100644 index 00000000000000..87d6a2a6df96a1 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-02-18-21-44-39.gh-issue-141510.7LST2O.rst @@ -0,0 +1 @@ +Update specializer to support frozendict. Patch by Donghee Na. diff --git a/Misc/NEWS.d/next/Library/2024-09-30-15-31-59.gh-issue-124748.KYzYFp.rst b/Misc/NEWS.d/next/Library/2024-09-30-15-31-59.gh-issue-124748.KYzYFp.rst new file mode 100644 index 00000000000000..5067db357fc577 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-30-15-31-59.gh-issue-124748.KYzYFp.rst @@ -0,0 +1,2 @@ +Improve :exc:`TypeError` error message when :meth:`!weakref.WeakKeyDictionary.update` +is used with keyword-only parameters. diff --git a/Misc/NEWS.d/next/Library/2026-01-12-19-39-57.gh-issue-140652.HvM9Bl.rst b/Misc/NEWS.d/next/Library/2026-01-12-19-39-57.gh-issue-140652.HvM9Bl.rst new file mode 100644 index 00000000000000..bed126f02f8714 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-01-12-19-39-57.gh-issue-140652.HvM9Bl.rst @@ -0,0 +1 @@ +Fix a crash in :func:`!_interpchannels.list_all` after closing a channel. diff --git a/Misc/NEWS.d/next/Library/2026-02-13-11-14-18.gh-issue-144763.cDwnEE.rst b/Misc/NEWS.d/next/Library/2026-02-13-11-14-18.gh-issue-144763.cDwnEE.rst new file mode 100644 index 00000000000000..14eb4f49c8ad3c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-02-13-11-14-18.gh-issue-144763.cDwnEE.rst @@ -0,0 +1,2 @@ +Fix a race condition in :mod:`tracemalloc`: it no longer detaches the attached +thread state to acquire its internal lock. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2026-02-17-11-28-37.gh-issue-141510.OpAz0M.rst b/Misc/NEWS.d/next/Library/2026-02-17-11-28-37.gh-issue-141510.OpAz0M.rst new file mode 100644 index 00000000000000..5b604124c6d7cc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-02-17-11-28-37.gh-issue-141510.OpAz0M.rst @@ -0,0 +1,2 @@ +The :mod:`copy` module now supports the :class:`frozendict` type. Patch by +Pieter Eendebak based on work by Victor Stinner. diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index ef9cf01ecbec5e..2933332ad465d4 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -1644,14 +1644,16 @@ _channels_list_all(_channels *channels, int64_t *count) if (ids == NULL) { goto done; } - _channelref *ref = channels->head; - for (int64_t i=0; ref != NULL; ref = ref->next, i++) { - ids[i] = (struct channel_id_and_info){ - .id = ref->cid, - .defaults = ref->chan->defaults, - }; + int64_t i = 0; + for (_channelref *ref = channels->head; ref != NULL; ref = ref->next) { + if (ref->chan != NULL) { + ids[i++] = (struct channel_id_and_info){ + .id = ref->cid, + .defaults = ref->chan->defaults, + }; + } } - *count = channels->numopen; + *count = i; cids = ids; done: diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 24d3443dd8abfe..65facaa6db2036 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -6663,8 +6663,6 @@ do_append(PickleState *state, UnpicklerObject *self, Py_ssize_t x) len = Py_SIZE(self->stack); if (x > len || x <= self->stack->fence) return Pdata_stack_underflow(state, self->stack); - if (len == x) /* nothing to do */ - return 0; list = self->stack->data[x - 1]; @@ -6754,8 +6752,6 @@ do_setitems(PickleState *st, UnpicklerObject *self, Py_ssize_t x) len = Py_SIZE(self->stack); if (x > len || x <= self->stack->fence) return Pdata_stack_underflow(st, self->stack); - if (len == x) /* nothing to do */ - return 0; if ((len - x) % 2 != 0) { /* Corrupt or hostile pickle -- we never write one like this. */ PyErr_SetString(st->UnpicklingError, @@ -6807,8 +6803,6 @@ load_additems(PickleState *state, UnpicklerObject *self) len = Py_SIZE(self->stack); if (mark > len || mark <= self->stack->fence) return Pdata_stack_underflow(state, self->stack); - if (len == mark) /* nothing to do */ - return 0; set = self->stack->data[mark - 1]; diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index ddd8fcdc231bf1..a9cd0574a596a1 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -642,7 +642,7 @@ { nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { + if (!PyAnyDict_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -655,7 +655,7 @@ dict_st = nos; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); + assert(PyAnyDict_CheckExact(dict)); STAT_INC(BINARY_OP, hit); PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5139,7 +5139,7 @@ { tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyDict_CheckExact(o)) { + if (!PyAnyDict_CheckExact(o)) { UPDATE_MISS_STATS(CONTAINS_OP); assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); JUMP_TO_PREDICTED(CONTAINS_OP); @@ -5152,7 +5152,7 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyDict_CheckExact(right_o)); + assert(PyAnyDict_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); @@ -11482,7 +11482,7 @@ { nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { + if (!PyAnyDict_CheckExact(o)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); JUMP_TO_PREDICTED(STORE_SUBSCR); diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 0959e2c78a3289..af3fcca7455470 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -138,6 +138,8 @@ As a consequence of this, split keys have a maximum size of 16. // Forward declarations static PyObject* frozendict_new(PyTypeObject *type, PyObject *args, PyObject *kwds); +static PyObject* dict_new(PyTypeObject *type, PyObject *args, PyObject *kwds); +static int dict_merge(PyObject *a, PyObject *b, int override); /*[clinic input] @@ -282,6 +284,26 @@ load_keys_nentries(PyDictObject *mp) #endif +#ifndef NDEBUG +// Check if it's possible to modify a dictionary. +// Usage: assert(can_modify_dict(mp)). +static inline int +can_modify_dict(PyDictObject *mp) +{ + if (PyFrozenDict_Check(mp)) { + // No locking required to modify a newly created frozendict + // since it's only accessible from the current thread. + return PyUnstable_Object_IsUniquelyReferenced(_PyObject_CAST(mp)); + } + else { + // Locking is only required if the dictionary is not + // uniquely referenced. + ASSERT_DICT_LOCKED(mp); + return 1; + } +} +#endif + #define _PyAnyDict_CAST(op) \ (assert(PyAnyDict_Check(op)), _Py_CAST(PyDictObject*, op)) @@ -1867,8 +1889,9 @@ insert_split_key(PyDictKeysObject *keys, PyObject *key, Py_hash_t hash) static void insert_split_value(PyDictObject *mp, PyObject *key, PyObject *value, Py_ssize_t ix) { + assert(can_modify_dict(mp)); assert(PyUnicode_CheckExact(key)); - ASSERT_DICT_LOCKED(mp); + PyObject *old_value = mp->ma_values->values[ix]; if (old_value == NULL) { _PyDict_NotifyEvent(PyDict_EVENT_ADDED, mp, key, value); @@ -1896,11 +1919,11 @@ static int insertdict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) { + assert(can_modify_dict(mp)); + PyObject *old_value = NULL; Py_ssize_t ix; - ASSERT_DICT_LOCKED(mp); - if (_PyDict_HasSplitTable(mp) && PyUnicode_CheckExact(key)) { ix = insert_split_key(mp->ma_keys, key, hash); if (ix != DKIX_EMPTY) { @@ -1967,8 +1990,8 @@ static int insert_to_emptydict(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) { + assert(can_modify_dict(mp)); assert(mp->ma_keys == Py_EMPTY_KEYS); - ASSERT_DICT_LOCKED(mp); int unicode = PyUnicode_CheckExact(key); PyDictKeysObject *newkeys = new_keys_object(PyDict_LOG_MINSIZE, unicode); @@ -2069,11 +2092,11 @@ static int dictresize(PyDictObject *mp, uint8_t log2_newsize, int unicode) { + assert(can_modify_dict(mp)); + PyDictKeysObject *oldkeys, *newkeys; PyDictValues *oldvalues; - ASSERT_DICT_LOCKED(mp); - if (log2_newsize >= SIZEOF_SIZE_T*8) { PyErr_NoMemory(); return -1; @@ -2671,11 +2694,13 @@ _PyDict_LoadBuiltinsFromGlobals(PyObject *globals) /* Consumes references to key and value */ static int -anydict_setitem_take2(PyDictObject *mp, PyObject *key, PyObject *value) +setitem_take2_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) { + assert(PyAnyDict_Check(mp)); + assert(can_modify_dict(mp)); assert(key); assert(value); - assert(PyAnyDict_Check(mp)); + Py_hash_t hash = _PyObject_HashFast(key); if (hash == -1) { dict_unhashable_type(key); @@ -2691,14 +2716,6 @@ anydict_setitem_take2(PyDictObject *mp, PyObject *key, PyObject *value) return insertdict(mp, key, hash, value); } -/* Consumes references to key and value */ -static int -setitem_take2_lock_held(PyDictObject *mp, PyObject *key, PyObject *value) -{ - ASSERT_DICT_LOCKED(mp); - return anydict_setitem_take2(mp, key, value); -} - int _PyDict_SetItem_Take2(PyDictObject *mp, PyObject *key, PyObject *value) { @@ -2800,9 +2817,9 @@ static void delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, PyObject *old_value) { - PyObject *old_key; + assert(can_modify_dict(mp)); - ASSERT_DICT_LOCKED(mp); + PyObject *old_key; Py_ssize_t hashpos = lookdict_index(mp->ma_keys, hash, ix); assert(hashpos >= 0); @@ -2856,19 +2873,17 @@ int _PyDict_DelItem_KnownHash_LockHeld(PyObject *op, PyObject *key, Py_hash_t hash) { Py_ssize_t ix; - PyDictObject *mp; PyObject *old_value; if (!PyDict_Check(op)) { PyErr_BadInternalCall(); return -1; } - - ASSERT_DICT_LOCKED(op); + PyDictObject *mp = (PyDictObject *)op; + assert(can_modify_dict(mp)); assert(key); assert(hash != -1); - mp = (PyDictObject *)op; ix = _Py_dict_lookup(mp, key, hash, &old_value); if (ix == DKIX_ERROR) return -1; @@ -2897,19 +2912,18 @@ delitemif_lock_held(PyObject *op, PyObject *key, int (*predicate)(PyObject *value, void *arg), void *arg) { + PyDictObject *mp = _PyAnyDict_CAST(op); + assert(can_modify_dict(mp)); + Py_ssize_t ix; - PyDictObject *mp; Py_hash_t hash; PyObject *old_value; int res; - ASSERT_DICT_LOCKED(op); - assert(key); hash = PyObject_Hash(key); if (hash == -1) return -1; - mp = (PyDictObject *)op; ix = _Py_dict_lookup(mp, key, hash, &old_value); if (ix == DKIX_ERROR) { return -1; @@ -2951,16 +2965,16 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, static void clear_lock_held(PyObject *op) { - PyDictObject *mp; + if (!PyDict_Check(op)) { + return; + } + PyDictObject *mp = (PyDictObject *)op; + assert(can_modify_dict(mp)); + PyDictKeysObject *oldkeys; PyDictValues *oldvalues; Py_ssize_t i, n; - ASSERT_DICT_LOCKED(op); - - if (!PyDict_Check(op)) - return; - mp = ((PyDictObject *)op); oldkeys = mp->ma_keys; oldvalues = mp->ma_values; if (oldkeys == Py_EMPTY_KEYS) { @@ -3106,8 +3120,7 @@ _PyDict_Pop_KnownHash(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **result) { assert(PyDict_Check(mp)); - - ASSERT_DICT_LOCKED(mp); + assert(can_modify_dict(mp)); if (mp->ma_used == 0) { if (result) { @@ -3149,8 +3162,6 @@ _PyDict_Pop_KnownHash(PyDictObject *mp, PyObject *key, Py_hash_t hash, static int pop_lock_held(PyObject *op, PyObject *key, PyObject **result) { - ASSERT_DICT_LOCKED(op); - if (!PyDict_Check(op)) { if (result) { *result = NULL; @@ -3159,6 +3170,7 @@ pop_lock_held(PyObject *op, PyObject *key, PyObject **result) return -1; } PyDictObject *dict = (PyDictObject *)op; + assert(can_modify_dict(dict)); if (dict->ma_used == 0) { if (result) { @@ -3230,6 +3242,8 @@ _PyDict_Pop(PyObject *dict, PyObject *key, PyObject *default_value) static PyDictObject * dict_dict_fromkeys(PyDictObject *mp, PyObject *iterable, PyObject *value) { + assert(can_modify_dict(mp)); + PyObject *oldvalue; Py_ssize_t pos = 0; PyObject *key; @@ -3255,6 +3269,8 @@ dict_dict_fromkeys(PyDictObject *mp, PyObject *iterable, PyObject *value) static PyDictObject * dict_set_fromkeys(PyDictObject *mp, PyObject *iterable, PyObject *value) { + assert(can_modify_dict(mp)); + Py_ssize_t pos = 0; PyObject *key; Py_hash_t hash; @@ -3286,9 +3302,34 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) int status; d = _PyObject_CallNoArgs(cls); - if (d == NULL) + if (d == NULL) { return NULL; + } + // If cls is a dict or frozendict subclass with overridden constructor, + // copy the frozendict. + PyTypeObject *cls_type = _PyType_CAST(cls); + if (PyFrozenDict_Check(d) && cls_type->tp_new != frozendict_new) { + // Subclass-friendly copy + PyObject *copy; + if (PyObject_IsSubclass(cls, (PyObject*)&PyFrozenDict_Type)) { + copy = frozendict_new(cls_type, NULL, NULL); + } + else { + copy = dict_new(cls_type, NULL, NULL); + } + if (copy == NULL) { + Py_DECREF(d); + return NULL; + } + if (dict_merge(copy, d, 1) < 0) { + Py_DECREF(d); + Py_DECREF(copy); + return NULL; + } + Py_SETREF(d, copy); + } + assert(!PyFrozenDict_Check(d) || can_modify_dict((PyDictObject*)d)); if (PyDict_CheckExact(d)) { if (PyDict_CheckExact(iterable)) { @@ -3359,11 +3400,11 @@ _PyDict_FromKeys(PyObject *cls, PyObject *iterable, PyObject *value) dict_iter_exit:; Py_END_CRITICAL_SECTION(); } - else if (PyFrozenDict_CheckExact(d)) { + else if (PyFrozenDict_Check(d)) { while ((key = PyIter_Next(it)) != NULL) { - // anydict_setitem_take2 consumes a reference to key - status = anydict_setitem_take2((PyDictObject *)d, - key, Py_NewRef(value)); + // setitem_take2_lock_held consumes a reference to key + status = setitem_take2_lock_held((PyDictObject *)d, + key, Py_NewRef(value)); if (status < 0) { assert(PyErr_Occurred()); goto Fail; @@ -3933,7 +3974,7 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) static int dict_dict_merge(PyDictObject *mp, PyDictObject *other, int override) { - ASSERT_DICT_LOCKED(mp); + assert(can_modify_dict(mp)); ASSERT_DICT_LOCKED(other); if (other == mp || other->ma_used == 0) @@ -4474,8 +4515,6 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu Py_hash_t hash; Py_ssize_t ix; - ASSERT_DICT_LOCKED(d); - if (!PyDict_Check(d)) { PyErr_BadInternalCall(); if (result) { @@ -4483,6 +4522,7 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } return -1; } + assert(can_modify_dict((PyDictObject*)d)); hash = _PyObject_HashFast(key); if (hash == -1) { @@ -4657,11 +4697,11 @@ static PyObject * dict_popitem_impl(PyDictObject *self) /*[clinic end generated code: output=e65fcb04420d230d input=ef28b4da5f0f762e]*/ { + assert(can_modify_dict(self)); + Py_ssize_t i, j; PyObject *res; - ASSERT_DICT_LOCKED(self); - /* Allocate the result tuple before checking the size. Believe it * or not, this allocation could trigger a garbage collection which * could empty the dict, so if we checked the size first and that @@ -4799,11 +4839,12 @@ _PyDict_SizeOf_LockHeld(PyDictObject *mp) } void -_PyDict_ClearKeysVersionLockHeld(PyObject *mp) +_PyDict_ClearKeysVersionLockHeld(PyObject *op) { - ASSERT_DICT_LOCKED(mp); + PyDictObject *mp = _PyAnyDict_CAST(op); + assert(can_modify_dict(mp)); - FT_ATOMIC_STORE_UINT32_RELAXED(((PyDictObject *)mp)->ma_keys->dk_version, 0); + FT_ATOMIC_STORE_UINT32_RELAXED(mp->ma_keys->dk_version, 0); } Py_ssize_t @@ -7994,6 +8035,8 @@ frozendict_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (d == NULL) { return NULL; } + assert(can_modify_dict(_PyAnyDict_CAST(d))); + PyFrozenDictObject *self = _PyFrozenDictObject_CAST(d); self->ma_hash = -1; diff --git a/Objects/setobject.c b/Objects/setobject.c index f8713bf3d1a432..ae6c1d1248d2fc 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1186,10 +1186,14 @@ set_iter(PyObject *so) static int set_update_dict_lock_held(PySetObject *so, PyObject *other) { - assert(PyDict_CheckExact(other)); + assert(PyAnyDict_CheckExact(other)); _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(other); +#ifdef Py_DEBUG + if (!PyFrozenDict_CheckExact(other)) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(other); + } +#endif /* Do one big resize at the start, rather than * incrementally resizing as we insert new keys. Expect @@ -1245,7 +1249,7 @@ set_update_lock_held(PySetObject *so, PyObject *other) if (PyAnySet_Check(other)) { return set_merge_lock_held(so, other); } - else if (PyDict_CheckExact(other)) { + else if (PyAnyDict_CheckExact(other)) { return set_update_dict_lock_held(so, other); } return set_update_iterable_lock_held(so, other); @@ -1270,6 +1274,9 @@ set_update_local(PySetObject *so, PyObject *other) Py_END_CRITICAL_SECTION(); return rv; } + else if (PyFrozenDict_CheckExact(other)) { + return set_update_dict_lock_held(so, other); + } return set_update_iterable_lock_held(so, other); } @@ -1293,6 +1300,13 @@ set_update_internal(PySetObject *so, PyObject *other) Py_END_CRITICAL_SECTION2(); return rv; } + else if (PyFrozenDict_CheckExact(other)) { + int rv; + Py_BEGIN_CRITICAL_SECTION(so); + rv = set_update_dict_lock_held(so, other); + Py_END_CRITICAL_SECTION(); + return rv; + } else { int rv; Py_BEGIN_CRITICAL_SECTION(so); @@ -2033,7 +2047,7 @@ set_difference(PySetObject *so, PyObject *other) if (PyAnySet_Check(other)) { other_size = PySet_GET_SIZE(other); } - else if (PyDict_CheckExact(other)) { + else if (PyAnyDict_CheckExact(other)) { other_size = PyDict_GET_SIZE(other); } else { @@ -2050,7 +2064,7 @@ set_difference(PySetObject *so, PyObject *other) if (result == NULL) return NULL; - if (PyDict_CheckExact(other)) { + if (PyAnyDict_CheckExact(other)) { while (set_next(so, &pos, &entry)) { key = entry->key; hash = entry->hash; @@ -2172,7 +2186,11 @@ static int set_symmetric_difference_update_dict(PySetObject *so, PyObject *other) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(other); +#ifdef Py_DEBUG + if (!PyFrozenDict_CheckExact(other)) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(other); + } +#endif Py_ssize_t pos = 0; PyObject *key, *value; @@ -2246,6 +2264,11 @@ set_symmetric_difference_update_impl(PySetObject *so, PyObject *other) rv = set_symmetric_difference_update_dict(so, other); Py_END_CRITICAL_SECTION2(); } + else if (PyFrozenDict_CheckExact(other)) { + Py_BEGIN_CRITICAL_SECTION(so); + rv = set_symmetric_difference_update_dict(so, other); + Py_END_CRITICAL_SECTION(); + } else if (PyAnySet_Check(other)) { Py_BEGIN_CRITICAL_SECTION2(so, other); rv = set_symmetric_difference_update_set(so, (PySetObject *)other); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b461f9b5bea8a6..63a4222264985a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1058,12 +1058,12 @@ dummy_func( op(_GUARD_NOS_DICT, (nos, unused -- nos, unused)) { PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - EXIT_IF(!PyDict_CheckExact(o)); + EXIT_IF(!PyAnyDict_CheckExact(o)); } op(_GUARD_TOS_DICT, (tos -- tos)) { PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - EXIT_IF(!PyDict_CheckExact(o)); + EXIT_IF(!PyAnyDict_CheckExact(o)); } macro(BINARY_OP_SUBSCR_DICT) = @@ -1073,7 +1073,7 @@ dummy_func( PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); + assert(PyAnyDict_CheckExact(dict)); STAT_INC(BINARY_OP, hit); PyObject *res_o; int rc = PyDict_GetItemRef(dict, sub, &res_o); @@ -2940,7 +2940,7 @@ dummy_func( PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyDict_CheckExact(right_o)); + assert(PyAnyDict_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); int res = PyDict_Contains(right_o, left_o); if (res < 0) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 9dead4eecc7826..1b3de80e4443b1 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -5920,7 +5920,7 @@ _PyStackRef nos; nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { + if (!PyAnyDict_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); @@ -5941,7 +5941,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; nos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { + if (!PyAnyDict_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); _tos_cache0 = _stack_item_0; SET_CURRENT_CACHED_VALUES(1); @@ -5964,7 +5964,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; nos = _stack_item_0; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { + if (!PyAnyDict_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); _tos_cache1 = _stack_item_1; _tos_cache0 = nos; @@ -5987,7 +5987,7 @@ _PyStackRef _stack_item_2 = _tos_cache2; nos = _stack_item_1; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { + if (!PyAnyDict_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); _tos_cache2 = _stack_item_2; _tos_cache1 = nos; @@ -6009,7 +6009,7 @@ _PyStackRef tos; tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyDict_CheckExact(o)) { + if (!PyAnyDict_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); SET_CURRENT_CACHED_VALUES(0); JUMP_TO_JUMP_TARGET(); @@ -6029,7 +6029,7 @@ _PyStackRef _stack_item_0 = _tos_cache0; tos = _stack_item_0; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyDict_CheckExact(o)) { + if (!PyAnyDict_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); _tos_cache0 = tos; SET_CURRENT_CACHED_VALUES(1); @@ -6049,7 +6049,7 @@ _PyStackRef _stack_item_1 = _tos_cache1; tos = _stack_item_1; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyDict_CheckExact(o)) { + if (!PyAnyDict_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); _tos_cache1 = tos; _tos_cache0 = _stack_item_0; @@ -6072,7 +6072,7 @@ _PyStackRef _stack_item_2 = _tos_cache2; tos = _stack_item_2; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyDict_CheckExact(o)) { + if (!PyAnyDict_CheckExact(o)) { UOP_STAT_INC(uopcode, miss); _tos_cache2 = tos; _tos_cache1 = _stack_item_1; @@ -6102,7 +6102,7 @@ dict_st = _stack_item_0; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); + assert(PyAnyDict_CheckExact(dict)); STAT_INC(BINARY_OP, hit); PyObject *res_o; stack_pointer[0] = dict_st; @@ -10393,7 +10393,7 @@ left = _stack_item_0; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyDict_CheckExact(right_o)); + assert(PyAnyDict_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); stack_pointer[0] = left; stack_pointer[1] = right; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 37fa6d679190dd..829a6988954e5f 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -642,7 +642,7 @@ { nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { + if (!PyAnyDict_CheckExact(o)) { UPDATE_MISS_STATS(BINARY_OP); assert(_PyOpcode_Deopt[opcode] == (BINARY_OP)); JUMP_TO_PREDICTED(BINARY_OP); @@ -655,7 +655,7 @@ dict_st = nos; PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); PyObject *dict = PyStackRef_AsPyObjectBorrow(dict_st); - assert(PyDict_CheckExact(dict)); + assert(PyAnyDict_CheckExact(dict)); STAT_INC(BINARY_OP, hit); PyObject *res_o; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -5139,7 +5139,7 @@ { tos = stack_pointer[-1]; PyObject *o = PyStackRef_AsPyObjectBorrow(tos); - if (!PyDict_CheckExact(o)) { + if (!PyAnyDict_CheckExact(o)) { UPDATE_MISS_STATS(CONTAINS_OP); assert(_PyOpcode_Deopt[opcode] == (CONTAINS_OP)); JUMP_TO_PREDICTED(CONTAINS_OP); @@ -5152,7 +5152,7 @@ left = stack_pointer[-2]; PyObject *left_o = PyStackRef_AsPyObjectBorrow(left); PyObject *right_o = PyStackRef_AsPyObjectBorrow(right); - assert(PyDict_CheckExact(right_o)); + assert(PyAnyDict_CheckExact(right_o)); STAT_INC(CONTAINS_OP, hit); _PyFrame_SetStackPointer(frame, stack_pointer); int res = PyDict_Contains(right_o, left_o); @@ -11479,7 +11479,7 @@ { nos = stack_pointer[-2]; PyObject *o = PyStackRef_AsPyObjectBorrow(nos); - if (!PyDict_CheckExact(o)) { + if (!PyAnyDict_CheckExact(o)) { UPDATE_MISS_STATS(STORE_SUBSCR); assert(_PyOpcode_Deopt[opcode] == (STORE_SUBSCR)); JUMP_TO_PREDICTED(STORE_SUBSCR); diff --git a/Python/specialize.c b/Python/specialize.c index 5ba016f83ea077..4d3ba4acbbf038 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -2287,7 +2287,7 @@ _Py_Specialize_BinaryOp(_PyStackRef lhs_st, _PyStackRef rhs_st, _Py_CODEUNIT *in } } } - if (PyDict_CheckExact(lhs)) { + if (PyAnyDict_CheckExact(lhs)) { specialize(instr, BINARY_OP_SUBSCR_DICT); return; } @@ -2767,7 +2767,7 @@ _Py_Specialize_ContainsOp(_PyStackRef value_st, _Py_CODEUNIT *instr) assert(ENABLE_SPECIALIZATION); assert(_PyOpcode_Caches[CONTAINS_OP] == INLINE_CACHE_ENTRIES_COMPARE_OP); - if (PyDict_CheckExact(value)) { + if (PyAnyDict_CheckExact(value)) { specialize(instr, CONTAINS_OP_DICT); return; } diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index cdd96925d1f27a..0afc84e021817c 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -36,7 +36,7 @@ static int _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, the GIL held from PyMem_RawFree(). It cannot acquire the lock because it would introduce a deadlock in _PyThreadState_DeleteCurrent(). */ #define tables_lock _PyRuntime.tracemalloc.tables_lock -#define TABLES_LOCK() PyMutex_Lock(&tables_lock) +#define TABLES_LOCK() PyMutex_LockFlags(&tables_lock, _Py_LOCK_DONT_DETACH) #define TABLES_UNLOCK() PyMutex_Unlock(&tables_lock) @@ -224,13 +224,20 @@ tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) assert(PyStackRef_CodeCheck(pyframe->f_executable)); frame->filename = &_Py_STR(anon_unknown); - int lineno = PyUnstable_InterpreterFrame_GetLine(pyframe); + int lineno = -1; + PyCodeObject *code = _PyFrame_GetCode(pyframe); + // PyUnstable_InterpreterFrame_GetLine() cannot but used, since it uses + // a critical section which can trigger a deadlock. + int lasti = _PyFrame_SafeGetLasti(pyframe); + if (lasti >= 0) { + lineno = _PyCode_SafeAddr2Line(code, lasti); + } if (lineno < 0) { lineno = 0; } frame->lineno = (unsigned int)lineno; - PyObject *filename = _PyFrame_GetCode(pyframe)->co_filename; + PyObject *filename = code->co_filename; if (filename == NULL) { #ifdef TRACE_DEBUG tracemalloc_error("failed to get the filename of the code object"); @@ -863,7 +870,8 @@ _PyTraceMalloc_Stop(void) TABLES_LOCK(); if (!tracemalloc_config.tracing) { - goto done; + TABLES_UNLOCK(); + return; } /* stop tracing Python memory allocations */ @@ -880,10 +888,12 @@ _PyTraceMalloc_Stop(void) raw_free(tracemalloc_traceback); tracemalloc_traceback = NULL; - (void)PyRefTracer_SetTracer(NULL, NULL); - -done: TABLES_UNLOCK(); + + // Call it after TABLES_UNLOCK() since it calls _PyEval_StopTheWorldAll() + // which would lead to a deadlock with TABLES_LOCK() which doesn't detach + // the thread state. + (void)PyRefTracer_SetTracer(NULL, NULL); }