From fda3c889f65d2bcc8cb3284c324fe589771d3f7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20D=C3=B6rrer?= <d-chris@web.de>
Date: Wed, 19 Mar 2025 22:12:49 +0100
Subject: [PATCH 1/9] implemented pytest for makepyfile

see: fixture `testdir.makepyfile` does not encode files properly #285
---
 tests/test_makepyfile.py | 274 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 274 insertions(+)
 create mode 100644 tests/test_makepyfile.py

diff --git a/tests/test_makepyfile.py b/tests/test_makepyfile.py
new file mode 100644
index 0000000..0347f7e
--- /dev/null
+++ b/tests/test_makepyfile.py
@@ -0,0 +1,274 @@
+from encodings.aliases import aliases
+from pathlib import Path
+from textwrap import dedent
+from typing import Tuple
+
+import pytest
+
+pytest_plugins = ["pytester"]
+
+
+def enc(enc):
+    try:
+        _ = "abc".encode(enc)
+    except Exception:
+        return False
+
+    return True
+
+
+all_encodings = set(filter(enc, aliases.values()))
+
+
+# @pytest.fixture(
+#     params=[
+#         "cp1252",
+#         "utf-8",
+#         "ascii",
+#         "latin1",
+#     ]
+# )
+@pytest.fixture(params=all_encodings)
+def encoding(request):
+    return request.param
+
+
+# UTF-8 Fixture (contains various Unicode characters)
+@pytest.fixture(
+    params=[
+        ("3", "4"),
+        ("☆", "★"),
+        ("1", "☀"),
+        ("✅", "💥"),
+    ],
+    ids=[
+        "numbers",
+        "stars",
+        "number-sun",
+        "check-explosion",
+    ],
+)
+def utf8_charset(request):
+    return request.param
+
+
+# CP-1252 Fixture (contains characters from Windows-1252 encoding)
+@pytest.fixture(
+    params=[
+        ("é", "ü"),
+        ("„", "“"),
+        ("¥", "ƒ"),
+        ("©", "®"),
+    ],
+    ids=[
+        "accented_letters_CP1252",
+        "quotes_CP1252",
+        "currency_symbols_CP1252",
+        "copyright_registered_CP1252",
+    ],
+)
+def cp1252_charset(request):
+    return request.param
+
+
+# ANSI (Strictly ASCII characters only)
+@pytest.fixture(
+    params=[
+        ("A", "B"),
+        ("1", "2"),
+        ("!", "?"),
+        ("@", "#"),
+    ],
+    ids=[
+        "letters_ANSI",
+        "numbers_ANSI",
+        "punctuation_ANSI",
+        "symbols_ANSI",
+    ],
+)
+def ansi_charset(request):
+    return request.param
+
+
+@pytest.fixture(
+    params=[
+        ("3", "4"),
+        ("A", "B"),
+        ("!", "?"),
+        ("☆", "☁"),
+        ("✅", "💥"),
+        ("é", "ü"),
+        ("Я", "ж"),  # Cyrillic characters
+        ("Ф", "Щ"),  # More Cyrillic
+        ("中", "国"),  # Chinese characters (zhong guo - China)
+        ("道", "德"),  # Chinese characters (dao de - way virtue)
+    ],
+    ids=[
+        "numbers",
+        "letters",
+        "punctuation",
+        "stars",
+        "check-explosion",
+        "accented_letters",
+        "cyrillic_basic",
+        "cyrillic_complex",
+        "chinese_country",
+        "chinese_concept",
+    ],
+)
+def any_charset(request):
+    return request.param
+
+
+@pytest.fixture
+def makepyfile_encoded(testdir, encoding, any_charset):
+    a, b = any_charset
+
+    original_file = testdir.makepyfile(
+        f"""
+        def f():
+            '''
+            >>> print({a})
+            {b}
+            '''
+            pass
+        """,
+        encoding=encoding,
+    )
+
+    expected_diff = dedent(
+        f"""
+            >>> print({a})
+        -   {b}
+        +   {a}
+        """
+    )
+
+    yield original_file, expected_diff, encoding
+
+
+def makebasicfile(tmp_path: Path, a, b, encoding: str) -> Tuple[Path, str]:
+    """alternative implementation without the use of `testdir.makepyfile`."""
+
+    original_file = Path(tmp_path).joinpath("test_basic.py")
+    code = dedent(
+        f"""
+        def f():
+            '''
+            >>> print({a})
+            {b}
+            '''
+            pass
+        """
+    )
+
+    original_file.write_text(code, encoding=encoding)
+
+    expected_diff = dedent(
+        f"""
+            >>> print({a})
+        -   {b}
+        +   {a}
+        """
+    )
+
+    return original_file, expected_diff
+
+
+@pytest.fixture
+def basic_encoded(tmp_path, encoding, any_charset):
+
+    a, b = any_charset
+
+    file, diff = makebasicfile(tmp_path, a, b, encoding)
+
+    yield file, diff, encoding
+
+
+@pytest.mark.xfail(reason="UnicodeDecodeError")
+def test_makepyfile(makepyfile_encoded):
+    """
+    Test is expected to fail because of UnicodeDecodeError.
+
+    Just to compare visually with `test_basicfile` to see the difference
+    """
+
+    file, diff, enc = makepyfile_encoded
+
+    text = Path(file).read_text(enc)
+
+    print(text, diff)
+
+    pass
+
+
+@pytest.mark.xfail(reason="UnicodeDecodeError")
+def test_basicfile(basic_encoded):
+    """
+    Test is expected to fail because of UnicodeDecodeError.
+
+    Just to compare visually with `test_makepyfile` to see the difference
+    """
+    file, diff, enc = basic_encoded
+
+    text = Path(file).read_text(enc)
+
+    print(text, diff)
+
+    pass
+
+
+def test_compare_make_basic_file(testdir, encoding, any_charset):
+    """
+    Compare testdir.makepyfile with pathlib.Path.writetext to create python files.
+
+    Expected behavior is when `testdir.makepyfile` created a file with given encoding,
+    `makebasicfile` also should be able to encode the charset.
+
+    If a UnicodeEncodeError is raised, it means `testdir.makepyfile` screwed up the
+    encoding.
+    """
+
+    a, b = any_charset
+
+    # create a python file with testdir.makepyfile in the old fashioned way
+
+    make_file = testdir.makepyfile(
+        f"""
+        def f():
+            '''
+            >>> print({a})
+            {b}
+            '''
+            pass
+        """,
+        encoding=encoding,
+    )
+
+    assert Path(
+        make_file
+    ).is_file(), f"`testdir.makepyfile` failed encode {repr((a,b))} with {encoding=}"
+
+    make_diff = dedent(
+        f"""
+            >>> print({a})
+        -   {b}
+        +   {a}
+        """
+    )
+
+    # try to check if makepyfile screwed up and create python file in a different way
+
+    try:
+        basic_file, basic_diff = makebasicfile(str(testdir), a, b, encoding)
+    except UnicodeEncodeError:
+        pytest.fail(
+            f"testdir.makepyfile encoding {repr((a,b))} with {encoding=}, but it sould have failed."
+        )
+
+    assert Path(
+        basic_file
+    ).is_file(), f"`makebasicfile` failed encode {repr((a,b))} with {encoding=}"
+    assert (
+        make_diff == basic_diff
+    ), "sanity check - diffs always should be the same, if not my test implementation is wrong."

From 6a98ec4c15b305f372b93a8f4b885e5b2178c127 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20D=C3=B6rrer?= <d-chris@web.de>
Date: Wed, 19 Mar 2025 22:58:11 +0100
Subject: [PATCH 2/9] Update tests/test_makepyfile.py

Co-authored-by: P. L. Lim <2090236+pllim@users.noreply.github.com>
---
 tests/test_makepyfile.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/test_makepyfile.py b/tests/test_makepyfile.py
index 0347f7e..a238a9e 100644
--- a/tests/test_makepyfile.py
+++ b/tests/test_makepyfile.py
@@ -199,8 +199,6 @@ def test_makepyfile(makepyfile_encoded):
 
     print(text, diff)
 
-    pass
-
 
 @pytest.mark.xfail(reason="UnicodeDecodeError")
 def test_basicfile(basic_encoded):

From 0f6d0b3e557c6b89a31ab7606f462c41aa71b86d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20D=C3=B6rrer?= <d-chris@web.de>
Date: Wed, 19 Mar 2025 22:58:17 +0100
Subject: [PATCH 3/9] Update tests/test_makepyfile.py

Co-authored-by: P. L. Lim <2090236+pllim@users.noreply.github.com>
---
 tests/test_makepyfile.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/test_makepyfile.py b/tests/test_makepyfile.py
index a238a9e..7728312 100644
--- a/tests/test_makepyfile.py
+++ b/tests/test_makepyfile.py
@@ -213,8 +213,6 @@ def test_basicfile(basic_encoded):
 
     print(text, diff)
 
-    pass
-
 
 def test_compare_make_basic_file(testdir, encoding, any_charset):
     """

From cb6ded12ee32b62e5c960d44c117353b7cc86ad9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20D=C3=B6rrer?= <d-chris@web.de>
Date: Wed, 19 Mar 2025 23:11:38 +0100
Subject: [PATCH 4/9] Update tests/test_makepyfile.py

Co-authored-by: P. L. Lim <2090236+pllim@users.noreply.github.com>
---
 tests/test_makepyfile.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/tests/test_makepyfile.py b/tests/test_makepyfile.py
index 7728312..d4cbe0a 100644
--- a/tests/test_makepyfile.py
+++ b/tests/test_makepyfile.py
@@ -20,14 +20,6 @@ def enc(enc):
 all_encodings = set(filter(enc, aliases.values()))
 
 
-# @pytest.fixture(
-#     params=[
-#         "cp1252",
-#         "utf-8",
-#         "ascii",
-#         "latin1",
-#     ]
-# )
 @pytest.fixture(params=all_encodings)
 def encoding(request):
     return request.param

From ef57b66a3241c7710f539999d72ff54ab5ce6556 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20D=C3=B6rrer?= <d-chris@web.de>
Date: Sun, 23 Mar 2025 14:19:36 +0100
Subject: [PATCH 5/9] fixed docstring content

---
 tests/test_makepyfile.py | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/tests/test_makepyfile.py b/tests/test_makepyfile.py
index d4cbe0a..1945b5b 100644
--- a/tests/test_makepyfile.py
+++ b/tests/test_makepyfile.py
@@ -11,7 +11,7 @@
 def enc(enc):
     try:
         _ = "abc".encode(enc)
-    except Exception:
+    except LookupError:
         return False
 
     return True
@@ -120,7 +120,7 @@ def makepyfile_encoded(testdir, encoding, any_charset):
         f"""
         def f():
             '''
-            >>> print({a})
+            >>> print('{a}')
             {b}
             '''
             pass
@@ -130,11 +130,11 @@ def f():
 
     expected_diff = dedent(
         f"""
-            >>> print({a})
-        -   {b}
-        +   {a}
+             >>> print('{a}')
+        -    {b}
+        +    {a}
         """
-    )
+    ).strip("\n")
 
     yield original_file, expected_diff, encoding
 
@@ -147,7 +147,7 @@ def makebasicfile(tmp_path: Path, a, b, encoding: str) -> Tuple[Path, str]:
         f"""
         def f():
             '''
-            >>> print({a})
+            >>> print('{a}')
             {b}
             '''
             pass
@@ -158,11 +158,11 @@ def f():
 
     expected_diff = dedent(
         f"""
-            >>> print({a})
-        -   {b}
-        +   {a}
+             >>> print('{a}')
+        -    {b}
+        +    {a}
         """
-    )
+    ).strip("\n")
 
     return original_file, expected_diff
 
@@ -225,7 +225,7 @@ def test_compare_make_basic_file(testdir, encoding, any_charset):
         f"""
         def f():
             '''
-            >>> print({a})
+            >>> print('{a}')
             {b}
             '''
             pass
@@ -239,11 +239,11 @@ def f():
 
     make_diff = dedent(
         f"""
-            >>> print({a})
-        -   {b}
-        +   {a}
+             >>> print('{a}')
+        -    {b}
+        +    {a}
         """
-    )
+    ).strip("\n")
 
     # try to check if makepyfile screwed up and create python file in a different way
 

From c8f55a229184862952b95994231371aa7ebedf33 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20D=C3=B6rrer?= <d-chris@web.de>
Date: Sun, 23 Mar 2025 14:48:15 +0100
Subject: [PATCH 6/9] Add debug condition for test execution and remove unused
 fixtures

---
 tests/test_makepyfile.py | 67 +++++-----------------------------------
 1 file changed, 8 insertions(+), 59 deletions(-)

diff --git a/tests/test_makepyfile.py b/tests/test_makepyfile.py
index 1945b5b..0ba48b1 100644
--- a/tests/test_makepyfile.py
+++ b/tests/test_makepyfile.py
@@ -1,3 +1,4 @@
+import os
 from encodings.aliases import aliases
 from pathlib import Path
 from textwrap import dedent
@@ -8,6 +9,11 @@
 pytest_plugins = ["pytester"]
 
 
+def not_debug():
+    """run tests only when not in CI and debug is enabled."""
+    return os.getenv("CI", False) or not os.getenv("PYTEST_DEBUG_MAKEPYFILE", False)
+
+
 def enc(enc):
     try:
         _ = "abc".encode(enc)
@@ -25,63 +31,6 @@ def encoding(request):
     return request.param
 
 
-# UTF-8 Fixture (contains various Unicode characters)
-@pytest.fixture(
-    params=[
-        ("3", "4"),
-        ("☆", "★"),
-        ("1", "☀"),
-        ("✅", "💥"),
-    ],
-    ids=[
-        "numbers",
-        "stars",
-        "number-sun",
-        "check-explosion",
-    ],
-)
-def utf8_charset(request):
-    return request.param
-
-
-# CP-1252 Fixture (contains characters from Windows-1252 encoding)
-@pytest.fixture(
-    params=[
-        ("é", "ü"),
-        ("„", "“"),
-        ("¥", "ƒ"),
-        ("©", "®"),
-    ],
-    ids=[
-        "accented_letters_CP1252",
-        "quotes_CP1252",
-        "currency_symbols_CP1252",
-        "copyright_registered_CP1252",
-    ],
-)
-def cp1252_charset(request):
-    return request.param
-
-
-# ANSI (Strictly ASCII characters only)
-@pytest.fixture(
-    params=[
-        ("A", "B"),
-        ("1", "2"),
-        ("!", "?"),
-        ("@", "#"),
-    ],
-    ids=[
-        "letters_ANSI",
-        "numbers_ANSI",
-        "punctuation_ANSI",
-        "symbols_ANSI",
-    ],
-)
-def ansi_charset(request):
-    return request.param
-
-
 @pytest.fixture(
     params=[
         ("3", "4"),
@@ -177,7 +126,7 @@ def basic_encoded(tmp_path, encoding, any_charset):
     yield file, diff, encoding
 
 
-@pytest.mark.xfail(reason="UnicodeDecodeError")
+@pytest.mark.skipif(not_debug(), reason="running in CI or debugging is not enabled.")
 def test_makepyfile(makepyfile_encoded):
     """
     Test is expected to fail because of UnicodeDecodeError.
@@ -192,7 +141,7 @@ def test_makepyfile(makepyfile_encoded):
     print(text, diff)
 
 
-@pytest.mark.xfail(reason="UnicodeDecodeError")
+@pytest.mark.skipif(not_debug(), reason="running in CI or debugging is not enabled.")
 def test_basicfile(basic_encoded):
     """
     Test is expected to fail because of UnicodeDecodeError.

From 70d67820d7b353e770de9ef216f473e92b0a3b1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20D=C3=B6rrer?= <d-chris@web.de>
Date: Sun, 23 Mar 2025 14:58:27 +0100
Subject: [PATCH 7/9] changed charset for chinese and added japanese

---
 tests/test_makepyfile.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_makepyfile.py b/tests/test_makepyfile.py
index 0ba48b1..3d56258 100644
--- a/tests/test_makepyfile.py
+++ b/tests/test_makepyfile.py
@@ -41,8 +41,8 @@ def encoding(request):
         ("é", "ü"),
         ("Я", "ж"),  # Cyrillic characters
         ("Ф", "Щ"),  # More Cyrillic
-        ("中", "国"),  # Chinese characters (zhong guo - China)
-        ("道", "德"),  # Chinese characters (dao de - way virtue)
+        ("電腦", "电脑"),  # Chinese: traditional vs simplified for "computer"
+        ("学校", "がっこう"),  # Japanese: Kanji vs Hiragana for "school"
     ],
     ids=[
         "numbers",
@@ -53,8 +53,8 @@ def encoding(request):
         "accented_letters",
         "cyrillic_basic",
         "cyrillic_complex",
-        "chinese_country",
-        "chinese_concept",
+        "chinese_combo",
+        "japanese_combo",
     ],
 )
 def any_charset(request):

From 8eccc6e0668c0b6eb9a92287cabdd3b209ef7415 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20D=C3=B6rrer?= <d-chris@web.de>
Date: Sun, 23 Mar 2025 15:24:00 +0100
Subject: [PATCH 8/9] changed description strings

---
 tests/test_makepyfile.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/test_makepyfile.py b/tests/test_makepyfile.py
index 3d56258..5941ff0 100644
--- a/tests/test_makepyfile.py
+++ b/tests/test_makepyfile.py
@@ -53,8 +53,8 @@ def encoding(request):
         "accented_letters",
         "cyrillic_basic",
         "cyrillic_complex",
-        "chinese_combo",
-        "japanese_combo",
+        "chinese_computer",
+        "japanese_school",
     ],
 )
 def any_charset(request):
@@ -126,7 +126,7 @@ def basic_encoded(tmp_path, encoding, any_charset):
     yield file, diff, encoding
 
 
-@pytest.mark.skipif(not_debug(), reason="running in CI or debugging is not enabled.")
+@pytest.mark.skipif(not_debug(), reason="skipped in CI or debugging is not enabled.")
 def test_makepyfile(makepyfile_encoded):
     """
     Test is expected to fail because of UnicodeDecodeError.
@@ -141,7 +141,7 @@ def test_makepyfile(makepyfile_encoded):
     print(text, diff)
 
 
-@pytest.mark.skipif(not_debug(), reason="running in CI or debugging is not enabled.")
+@pytest.mark.skipif(not_debug(), reason="skipped in CI or debugging is not enabled.")
 def test_basicfile(basic_encoded):
     """
     Test is expected to fail because of UnicodeDecodeError.

From a6ceec7fc414946952609b62d84bc6b41b870097 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christoph=20D=C3=B6rrer?= <d-chris@web.de>
Date: Mon, 24 Mar 2025 23:47:56 +0100
Subject: [PATCH 9/9] alternative test function using bytes object as test data

---
 tests/test_makepyfile.py | 51 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 50 insertions(+), 1 deletion(-)

diff --git a/tests/test_makepyfile.py b/tests/test_makepyfile.py
index 5941ff0..f75e2fe 100644
--- a/tests/test_makepyfile.py
+++ b/tests/test_makepyfile.py
@@ -154,7 +154,7 @@ def test_basicfile(basic_encoded):
 
     print(text, diff)
 
-
+@pytest.mark.skip(reason="makepyfile uses encoding only on bytes objects.")
 def test_compare_make_basic_file(testdir, encoding, any_charset):
     """
     Compare testdir.makepyfile with pathlib.Path.writetext to create python files.
@@ -209,3 +209,52 @@ def f():
     assert (
         make_diff == basic_diff
     ), "sanity check - diffs always should be the same, if not my test implementation is wrong."
+
+
+def test_compare_write_bytes(testdir, encoding, any_charset):
+    """
+    `makeypfile` ignores keyword arguments `encoding` if content is not bytes.
+
+    1. create test data and convert to bytes, if encoding failes the test is skipped.
+    2. use both functions to create python file
+
+    - if both functions raise the same error, everything is as expected
+    - if Path.write_bytes succeeds so should `testdir.makepyfile` and error should be None
+    - both files sould exist after the test
+    """
+
+    a, b = any_charset
+
+    try:
+        bytes_content = dedent(f"""
+                def f():
+                    '''
+                    >>> print('{a}')
+                    {b}
+                    '''
+                    pass
+                """
+        ).encode(encoding)
+    except UnicodeEncodeError:
+        # skip test do testdata could not prepared
+        pytest.xfail(f"{repr(any_charset)} cannot be encoded with {encoding=}")
+
+    try:
+        make_file = testdir.makepyfile(
+            bytes_content,
+            encoding=encoding,
+        )
+    except Exception as e:
+        error = e  # if both function fail the same, everything is as expected
+    else:
+        error = None
+
+    basic_file = Path(str(testdir)).joinpath("test_basic.py")
+    try:
+        basic_file.write_bytes(bytes_content)
+    except Exception as e:
+        assert type(error) is type(e), f"basically never should happen, but {e=} was raised."
+    else:
+        assert error is None, f"makepyfile screwed up {encoding=} and raised {error=}"
+    
+    assert Path(make_file).is_file() and Path(basic_file).is_file(), "files are missing."