From 9e19c56ae85448094bf5a475dd640fb352533744 Mon Sep 17 00:00:00 2001 From: Joel Ray Holveck Date: Tue, 5 May 2026 03:01:16 -0700 Subject: [PATCH] Change the ScreenShot API to be buffer-oriented. This implements many of the changes discussed in #476, part 1. It does not make the ScreenShot object itself accessible as a buffer; that's a separate matter. This change makes `_raw` (previously `raw`) private. `_raw` is now the original object passed in. The `bgra` and `rgb` properties now return `memoryview` objects, rather than `bytes` or `bytearray` objects. This means that the backends can now capture data to an OS-provided buffer, such as one provided by `mmap` or `CreateDIBSection`. The `ScreenShot` object can then give the user direct access to this buffer, without ever needing to copy the data. Similarly, ScreenShot objects now can be given their data as any buffer type, not just a bytearray. The backends haven't yet been changed to take advantage of these new features. Docs, examples, and demos are updated. --- demos/cat-detector.py | 2 +- demos/tinytv-stream-simple.py | 2 +- demos/tinytv-stream.py | 2 +- demos/video-capture-simple.py | 2 +- docs/source/examples.rst | 10 ++-- docs/source/examples/pil.py | 4 +- docs/source/examples/pil_pixels.py | 2 +- docs/source/release-history/v11.0.0.md | 13 ++++ docs/source/support.rst | 9 ++- pyproject.toml | 1 + src/mss/base.py | 15 ++--- src/mss/screenshot.py | 82 ++++++++++++++++++-------- src/tests/test_bgra_to_rgb.py | 18 +++--- src/tests/test_get_pixels.py | 4 +- src/tests/test_implementation.py | 2 +- src/tests/third_party/test_pil.py | 2 +- 16 files changed, 103 insertions(+), 67 deletions(-) diff --git a/demos/cat-detector.py b/demos/cat-detector.py index 596dd997..c274909f 100755 --- a/demos/cat-detector.py +++ b/demos/cat-detector.py @@ -281,7 +281,7 @@ def main() -> None: # We transfer the image from MSS to PyTorch via a Pillow Image. Faster approaches exist (see # screenshot_to_tensor), but PIL is more readable. The bulk of the time in this program is spent doing # the AI work, so we just use the most convenient mechanism. - img = Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX") + img = Image.frombuffer("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX") # We explicitly convert it to a tensor here, even though Torchvision can also convert it in the preprocess # step. This is so that we send it to the GPU before we do the preprocessing: PIL Images are always on diff --git a/demos/tinytv-stream-simple.py b/demos/tinytv-stream-simple.py index 1046a5d3..4fe9ef13 100755 --- a/demos/tinytv-stream-simple.py +++ b/demos/tinytv-stream-simple.py @@ -97,7 +97,7 @@ def main() -> None: # The next step is to resize the image to fit the TinyTV's screen. There's a great image # manipulation library called PIL, or Pillow, that can do that. Let's transfer the raw pixels in # the ScreenShot object into a PIL Image. - original_image = Image.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX") + original_image = Image.frombuffer("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX") # Now, we can resize it. The resize method may stretch the image to make it match the TinyTV's # screen; the advanced demo gives other options. Using a reducing gap is optional, but speeds up diff --git a/demos/tinytv-stream.py b/demos/tinytv-stream.py index 351eebfd..83d62425 100755 --- a/demos/tinytv-stream.py +++ b/demos/tinytv-stream.py @@ -348,7 +348,7 @@ def capture_image( while True: sct_img = sct.grab(rect) - pil_img = Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX") + pil_img = Image.frombuffer("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX") yield pil_img diff --git a/demos/video-capture-simple.py b/demos/video-capture-simple.py index 7bfc4a05..ad39d32b 100755 --- a/demos/video-capture-simple.py +++ b/demos/video-capture-simple.py @@ -129,7 +129,7 @@ def main() -> None: # use PIL: you can create an Image from the screenshot, and create a VideoFrame from that. That said, # if you want to boost the fps rate by about 50%, check out the full demo, and search for # from_numpy_buffer. - img = Image.frombytes("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX") + img = Image.frombuffer("RGB", screenshot.size, screenshot.bgra, "raw", "BGRX") frame = av.VideoFrame.from_image(img) # When we encode frames, we get back a list of packets. Often, we'll get no packets at first: the diff --git a/docs/source/examples.rst b/docs/source/examples.rst index e10c6bb3..0085caeb 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -123,7 +123,7 @@ PIL === You can use the Python Image Library (aka Pillow) to do whatever you want with raw pixels. -This is an example using `frombytes() `_: +This is an example using `frombuffer() `_: .. literalinclude:: examples/pil.py :lines: 7- @@ -188,7 +188,7 @@ Different possibilities to convert raw BGRA values to RGB:: def numpy_flip(im): - """ Most efficient Numpy version as of now. """ + """ Most efficient Numpy version as of MSS 10.1. """ frame = numpy.array(im, dtype=numpy.uint8) return numpy.flip(frame[:, :, :3], 2).tobytes() @@ -198,14 +198,14 @@ Different possibilities to convert raw BGRA values to RGB:: return numpy.array(im, dtype=numpy.uint8)[..., [2, 1, 0]].tobytes() - def pil_frombytes(im): + def pil_frombuffer(im): """ Efficient Pillow version. """ - return Image.frombytes('RGB', im.size, im.bgra, 'raw', 'BGRX').tobytes() + return Image.frombuffer('RGB', im.size, im.bgra, 'raw', 'BGRX').tobytes() with mss.MSS() as sct: im = sct.grab(sct.monitors[1]) - rgb = pil_frombytes(im) + rgb = pil_frombuffer(im) ... .. versionadded:: 3.2.0 diff --git a/docs/source/examples/pil.py b/docs/source/examples/pil.py index dfb706b4..557e869c 100644 --- a/docs/source/examples/pil.py +++ b/docs/source/examples/pil.py @@ -15,9 +15,9 @@ sct_img = sct.grab(monitor) # Create the Image - img = Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX") + img = Image.frombuffer("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX") # The same, but less efficient: - # img = Image.frombytes('RGB', sct_img.size, sct_img.rgb) + # img = Image.frombuffer('RGB', sct_img.size, sct_img.rgb, 'raw', 'RGB') # And save it! output = f"monitor-{num}.png" diff --git a/docs/source/examples/pil_pixels.py b/docs/source/examples/pil_pixels.py index 52973534..4398192d 100644 --- a/docs/source/examples/pil_pixels.py +++ b/docs/source/examples/pil_pixels.py @@ -16,7 +16,7 @@ img = Image.new("RGB", sct_img.size) # Best solution: create a list(tuple(R, G, B), ...) for putdata() - pixels = zip(sct_img.raw[2::4], sct_img.raw[1::4], sct_img.raw[::4], strict=False) + pixels = zip(sct_img.bgra[2::4], sct_img.bgra[1::4], sct_img.bgra[::4], strict=False) img.putdata(list(pixels)) # But you can set individual pixels too (slower) diff --git a/docs/source/release-history/v11.0.0.md b/docs/source/release-history/v11.0.0.md index 7a13ef06..892493ab 100644 --- a/docs/source/release-history/v11.0.0.md +++ b/docs/source/release-history/v11.0.0.md @@ -8,6 +8,19 @@ Release date: 2026-xx-x ## Highlights +### API changes + +The API changes discussed in the 10.2 release notes are now implemented. (They're not all implemented as of this +writing, but should be by the time we release 11.0!) + +#### ScreenShot attributes + +The :py:attr:`mss.ScreenShot.raw` attribute has been removed. Use the :py:attr:`mss.ScreenShot.bgra` property instead. + +The :py:attr:`mss.ScreenShot.bgra` and :py:attr:`mss.ScreenShot.rgb` properties now will return read-only bytes-like +:py:type:`memoryview` objects, not necessarily :py:type:`bytes` or :py:type:`bytearray` objects. For practical use +cases, this should not be noticible. This change was allows faster access to screenshot data, with fewer memory copies. + ### Python 3.9 EOL Python 3.9 reached [end-of-life](https://devguide.python.org/developer-workflow/development-cycle/index.html#end-of-life-branches) on [October 31, 2025](https://devguide.python.org/versions/). It is no longer receiving any updates, even security updates. diff --git a/docs/source/support.rst b/docs/source/support.rst index b2b3f240..7a69b88b 100644 --- a/docs/source/support.rst +++ b/docs/source/support.rst @@ -5,7 +5,9 @@ Support Feel free to try MSS on a system we had not tested, and let us know by creating an `issue `_. - OS: GNU/Linux, macOS, and Windows - - Python: 3.10 and newer + - Python: CPython 3.10 and newer + +Python implementations other than CPython are unlikely to ever be supported, due to MSS's extensive use of ctypes. Future @@ -18,8 +20,9 @@ Future Others ====== -Tested successfully on Pypy 5.1.0 on Windows, but speed is terrible. - +Previous version of MSS were tested successfully on PyPy 5.1.0 on Windows, but speed is terrible. In general, PyPy +support is not a priority, but if you want to help, please create an +`issue `_. Abandoned ========= diff --git a/pyproject.toml b/pyproject.toml index 87a00e24..da599109 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,6 +75,7 @@ dev = [ "mypy==1.20.2", "ruff==0.15.12", "twine==6.2.0", + "typing_extensions==4.15.0", ] docs = [ "myst-parser==5.0.0 ; python_version >= '3.12'", diff --git a/src/mss/base.py b/src/mss/base.py index f5f373bf..963eb4cd 100644 --- a/src/mss/base.py +++ b/src/mss/base.py @@ -18,16 +18,9 @@ from collections.abc import Callable, Iterator from types import TracebackType - from mss.models import Monitor, Monitors, Size + from typing_extensions import Self - # Prior to 3.11, Python didn't have the Self type. typing_extensions does, but we don't want to depend on it. - try: - from typing import Self - except ImportError: - try: - from typing_extensions import Self - except ImportError: - Self = Any # type: ignore[assignment] + from mss.models import Monitor, Monitors, Size try: from datetime import UTC @@ -460,8 +453,8 @@ def _merge(screenshot: ScreenShot, cursor: ScreenShot, /) -> ScreenShot: if not overlap: return screenshot - screen_raw = screenshot.raw - cursor_raw = cursor.raw + screen_raw = screenshot._raw # noqa: SLF001 + cursor_raw = cursor.bgra cy, cy2 = (cy - y) * 4, (cy2 - y2) * 4 cx, cx2 = (cx - x) * 4, (cx2 - x2) * 4 diff --git a/src/mss/screenshot.py b/src/mss/screenshot.py index c8382e18..b7b0f6cf 100644 --- a/src/mss/screenshot.py +++ b/src/mss/screenshot.py @@ -12,25 +12,23 @@ from collections.abc import Iterator from typing import Any + from typing_extensions import Buffer + class ScreenShot: """Screenshot object. .. note:: - - A better name would have been *Image*, but to prevent collisions + A better name would have been *Image*, but to prevent collisions with PIL.Image, it has been decided to use *ScreenShot*. """ - __slots__ = {"__pixels", "__rgb", "pos", "raw", "size"} + __slots__ = {"__bgra", "__pixels", "__rgb", "_raw", "pos", "size"} - def __init__(self, data: bytearray, monitor: Monitor, /, *, size: Size | None = None) -> None: + def __init__(self, data: Buffer, monitor: Monitor, /, *, size: Size | None = None) -> None: + self.__bgra: memoryview | None = None self.__pixels: Pixels | None = None - self.__rgb: bytes | None = None - - #: Bytearray of the raw BGRA pixels retrieved by ctypes - #: OS independent implementations. - self.raw: bytearray = data + self.__rgb: memoryview | None = None #: NamedTuple of the screenshot coordinates. self.pos: Pos = Pos(monitor["left"], monitor["top"]) @@ -38,6 +36,19 @@ def __init__(self, data: bytearray, monitor: Monitor, /, *, size: Size | None = #: NamedTuple of the screenshot size. self.size: Size = Size(monitor["width"], monitor["height"]) if size is None else size + # Buffer of the raw BGRA pixels, retrieved by the + # platform-specific implementations. This is kept read-write + # if it was originally so, in order for _merge to work. + # However, it should be made read-only before returning to the + # user (via bgra), so that the cached values for __pixels and + # __rgb aren't potentially inconsistent if the user changes + # data. + self._raw: memoryview = memoryview(data) + assert self._raw.nbytes == self.size.width * self.size.height * 4, ( # noqa: S101 + "Data size does not match screenshot dimensions." + ) + assert self._raw.format == "B", "Data format is not bytes." # noqa: S101 + def __repr__(self) -> str: return f"<{type(self).__name__} pos={self.left},{self.top} size={self.width}x{self.height}>" @@ -63,28 +74,38 @@ def __array_interface__(self) -> dict[str, Any]: "version": 3, "shape": (self.height, self.width, 4), "typestr": "|u1", - "data": self.raw, + "data": self.bgra, } @classmethod - def from_size(cls: type[ScreenShot], data: bytearray, width: int, height: int, /) -> ScreenShot: + def from_size(cls: type[ScreenShot], data: Buffer, width: int, height: int, /) -> ScreenShot: """Instantiate a new class given only screenshot's data and size.""" monitor = {"left": 0, "top": 0, "width": width, "height": height} return cls(data, monitor) @property - def bgra(self) -> bytes: + def bgra(self) -> memoryview: """BGRx values from the BGRx raw pixels. - The format is a bytes object with BGRxBGRx... sequence. A specific - pixel can be accessed as + The format is a memoryview object of bytes. These are in a + BGRxBGRx... sequence. A specific pixel can be accessed as ``bgra[(y * width + x) * 4:(y * width + x) * 4 + 4].`` + The memoryview is read-only. PyTorch will issue a warning + when given a read-only buffer, but will still work. However, + actually modifying the data may cause undefined behavior. + .. note:: - While the name is ``bgra``, the alpha channel may or may not be - valid. + While the name is ``bgra``, the alpha channel may or may + not be valid. """ - return bytes(self.raw) + # Making a read-only copy of a memoryview is very cheap. But + # we still always return the same memoryview: somebody using a + # property may expect it to be identical (under the `is` + # operator) every time. + if self.__bgra is None: + self.__bgra = self._raw.toreadonly() + return self.__bgra @property def pixels(self) -> Pixels: @@ -93,8 +114,8 @@ def pixels(self) -> Pixels: The format is a list of rows. Each row is a list of pixels. Each pixel is a tuple of (R, G, B). """ - if not self.__pixels: - rgb_tuples: Iterator[Pixel] = zip(self.raw[2::4], self.raw[1::4], self.raw[::4], strict=False) + if self.__pixels is None: + rgb_tuples: Iterator[Pixel] = zip(self._raw[2::4], self._raw[1::4], self._raw[::4], strict=False) self.__pixels = list(zip(*[iter(rgb_tuples)] * self.width, strict=False)) return self.__pixels @@ -111,20 +132,29 @@ def pixel(self, coord_x: int, coord_y: int) -> Pixel: raise ScreenShotError(msg) from exc @property - def rgb(self) -> bytes: + def rgb(self) -> memoryview: """Compute RGB values from the BGRA raw pixels. - The format is a bytes object with BGRBGR... sequence. A specific - pixel can be accessed as - ``rgb[(y * width + x) * 3:(y * width + x) * 3 + 3]``. + The format is a memoryview object of bytes. These are in a + RGBRGB... sequence. A specific pixel can be accessed as + ``rgb[(y * width + x) * 4:(y * width + x) * 4 + 4].`` + + The memoryview is read-only. PyTorch will issue a warning + when given a read-only buffer, but will still work. However, + actually modifying the data may cause undefined behavior. + + :: note:: + This is a computed property. If possible, using the + :py:attr:`bgra` property directly is usually more + efficient. """ - if not self.__rgb: + if self.__rgb is None: rgb = bytearray(self.height * self.width * 3) - raw = self.raw + raw = self._raw rgb[::3] = raw[2::4] rgb[1::3] = raw[1::4] rgb[2::3] = raw[::4] - self.__rgb = bytes(rgb) + self.__rgb = memoryview(rgb).toreadonly() return self.__rgb diff --git a/src/tests/test_bgra_to_rgb.py b/src/tests/test_bgra_to_rgb.py index a481c1f1..deed4b60 100644 --- a/src/tests/test_bgra_to_rgb.py +++ b/src/tests/test_bgra_to_rgb.py @@ -2,19 +2,15 @@ Source: https://github.com/BoboTiG/python-mss. """ -import pytest - from mss.base import ScreenShot -def test_bad_length() -> None: - data = bytearray(b"789c626001000000ffff030000060005") - image = ScreenShot.from_size(data, 1024, 768) - with pytest.raises(ValueError, match="attempt to assign"): - _ = image.rgb - - def test_good_types(raw: bytes) -> None: image = ScreenShot.from_size(bytearray(raw), 1024, 768) - assert isinstance(image.raw, bytearray) - assert isinstance(image.rgb, bytes) + assert isinstance(image.rgb, memoryview) + assert image.rgb.readonly + + +def test_contents() -> None: + image = ScreenShot.from_size(b"BGRA" * 1024 * 768, 1024, 768) + assert bytes(image.rgb) == b"RGB" * 1024 * 768 diff --git a/src/tests/test_get_pixels.py b/src/tests/test_get_pixels.py index cc94aba7..c784a6bd 100644 --- a/src/tests/test_get_pixels.py +++ b/src/tests/test_get_pixels.py @@ -16,8 +16,8 @@ def test_grab_monitor(mss_impl: Callable[..., MSS]) -> None: for mon in sct.monitors: image = sct.grab(mon) assert isinstance(image, ScreenShot) - assert isinstance(image.raw, bytearray) - assert isinstance(image.rgb, bytes) + assert isinstance(image.bgra, memoryview) + assert image.bgra.readonly def test_grab_part_of_screen(mss_impl: Callable[..., MSS]) -> None: diff --git a/src/tests/test_implementation.py b/src/tests/test_implementation.py index 278fe3ca..29c4ce5e 100644 --- a/src/tests/test_implementation.py +++ b/src/tests/test_implementation.py @@ -114,7 +114,7 @@ def test_repr(mss_impl: Callable[..., MSS]) -> None: expected_box = {"top": 0, "left": 0, "width": 10, "height": 10} with mss_impl() as sct: img = sct.grab(box) - ref = ScreenShot(bytearray(b"42"), expected_box) + ref = ScreenShot(bytearray(b"BGRA" * 100), expected_box) assert repr(img) == repr(ref) diff --git a/src/tests/third_party/test_pil.py b/src/tests/third_party/test_pil.py index 274fd049..586e1425 100644 --- a/src/tests/third_party/test_pil.py +++ b/src/tests/third_party/test_pil.py @@ -37,7 +37,7 @@ def test_pil_bgra(mss_impl: Callable[..., MSS]) -> None: with mss_impl() as sct: sct_img = sct.grab(box) - img = Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX") + img = Image.frombuffer("RGB", sct_img.size, sct_img.bgra, "raw", "BGRX") assert img.mode == "RGB" assert img.size == sct_img.size