Improve fuzz coverage

Makefile

@@ -1,8 +1,8 @@
-all : fuzzer-html fuzzer-email fuzzer-httpclient fuzzer-json fuzzer-difflib fuzzer-csv fuzzer-decode fuzzer-ast fuzzer-tarfile fuzzer-tarfile-hypothesis fuzzer-zipfile fuzzer-zipfile-hypothesis fuzzer-re fuzzer-configparser fuzzer-tomllib fuzzer-plistlib fuzzer-xml fuzzer-zoneinfo
+all : fuzzer-html fuzzer-email fuzzer-httpclient fuzzer-json fuzzer-difflib fuzzer-csv fuzzer-decode fuzzer-ast fuzzer-tarfile fuzzer-tarfile-hypothesis fuzzer-zipfile fuzzer-zipfile-hypothesis fuzzer-re fuzzer-configparser fuzzer-tomllib fuzzer-plistlib fuzzer-xml fuzzer-zoneinfo fuzzer-array fuzzer-binascii fuzzer-codecs fuzzer-collections fuzzer-compression fuzzer-crypto fuzzer-datetime fuzzer-dbm fuzzer-expat fuzzer-ioops fuzzer-json-decode fuzzer-json-encode fuzzer-locale fuzzer-mmap fuzzer-operator fuzzer-pickle fuzzer-ssl fuzzer-time fuzzer-unicodedata
 
 PYTHON_CONFIG_PATH=$(CPYTHON_INSTALL_PATH)/bin/python3-config
 CXXFLAGS += $(shell $(PYTHON_CONFIG_PATH) --cflags)
-LDFLAGS += -rdynamic $(shell $(PYTHON_CONFIG_PATH) --ldflags --embed)
+LDFLAGS += -rdynamic $(shell $(PYTHON_CONFIG_PATH) --ldflags --embed) $(CPYTHON_MODLIBS) $(CPYTHON_HACL_LIBS) -Wl,--allow-multiple-definition
 
 fuzzer-html:
 	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"html.py\"" -ldl $(LDFLAGS) -o fuzzer-html
@@ -40,3 +40,41 @@ fuzzer-xml:
 	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"xml.py\"" -ldl $(LDFLAGS) -o fuzzer-xml
 fuzzer-zoneinfo:
 	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"zoneinfo.py\"" -ldl $(LDFLAGS) -o fuzzer-zoneinfo
+fuzzer-array:
+	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"fuzz_array.py\"" -ldl $(LDFLAGS) -o fuzzer-array
+fuzzer-binascii:
+	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"fuzz_binascii.py\"" -ldl $(LDFLAGS) -o fuzzer-binascii
+fuzzer-codecs:
+	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"fuzz_codecs.py\"" -ldl $(LDFLAGS) -o fuzzer-codecs
+fuzzer-collections:
+	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"fuzz_collections.py\"" -ldl $(LDFLAGS) -o fuzzer-collections
+fuzzer-compression:
+	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"fuzz_compression.py\"" -ldl $(LDFLAGS) -o fuzzer-compression
+fuzzer-crypto:
+	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"fuzz_crypto.py\"" -ldl $(LDFLAGS) -o fuzzer-crypto
+fuzzer-datetime:
+	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"fuzz_datetime.py\"" -ldl $(LDFLAGS) -o fuzzer-datetime
+fuzzer-dbm:
+	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"fuzz_dbm.py\"" -ldl $(LDFLAGS) -o fuzzer-dbm
+fuzzer-expat:
+	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"fuzz_expat.py\"" -ldl $(LDFLAGS) -o fuzzer-expat
+fuzzer-ioops:
+	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"fuzz_ioops.py\"" -ldl $(LDFLAGS) -o fuzzer-ioops
+fuzzer-json-decode:
+	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"fuzz_json_decode.py\"" -ldl $(LDFLAGS) -o fuzzer-json-decode
+fuzzer-json-encode:
+	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"fuzz_json_encode.py\"" -ldl $(LDFLAGS) -o fuzzer-json-encode
+fuzzer-locale:
+	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"fuzz_locale.py\"" -ldl $(LDFLAGS) -o fuzzer-locale
+fuzzer-mmap:
+	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"fuzz_mmap.py\"" -ldl $(LDFLAGS) -o fuzzer-mmap
+fuzzer-operator:
+	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"fuzz_operator.py\"" -ldl $(LDFLAGS) -o fuzzer-operator
+fuzzer-pickle:
+	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"fuzz_pickle.py\"" -ldl $(LDFLAGS) -o fuzzer-pickle
+fuzzer-ssl:
+	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"fuzz_ssl.py\"" -ldl $(LDFLAGS) -o fuzzer-ssl
+fuzzer-time:
+	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"fuzz_time.py\"" -ldl $(LDFLAGS) -o fuzzer-time
+fuzzer-unicodedata:
+	clang++ $(CXXFLAGS) $(LIB_FUZZING_ENGINE) -std=c++17 fuzzer.cpp -DPYTHON_HARNESS_PATH="\"fuzz_unicodedata.py\"" -ldl $(LDFLAGS) -o fuzzer-unicodedata

fuzz_array.py

@@ -0,0 +1,100 @@
+from fuzz_dp import FuzzedDataProvider
+import array
+
+TYPECODES = list('bBhHiIlLqQfd')
+
+# Top-level operation constants for FuzzerRunOne
+OP_FROMBYTES = 0
+OP_METHODS = 1
+OP_SLICE = 2
+
+# Array method operation constants for op_array_methods
+METHOD_REVERSE = 0
+METHOD_BYTESWAP = 1
+METHOD_POP = 2
+METHOD_COUNT = 3
+METHOD_INDEX = 4
+METHOD_INSERT = 5
+METHOD_REMOVE = 6
+METHOD_TOBYTES = 7
+
+def _consume_array(fdp):
+    tc = fdp.PickValueInList(TYPECODES)
+    itemsize = array.array(tc).itemsize
+    n_items = fdp.ConsumeIntInRange(0, min(fdp.remaining_bytes() // itemsize, 200))
+    data = fdp.ConsumeBytes(n_items * itemsize)
+    a = array.array(tc)
+    a.frombytes(data)
+    return a, tc
+
+
+def op_array_frombytes(fdp):
+    a, tc = _consume_array(fdp)
+    a.tobytes()
+    a.tolist()
+
+def op_array_methods(fdp):
+    a, tc = _consume_array(fdp)
+    if len(a) == 0:
+        return
+    num_ops = fdp.ConsumeIntInRange(1, 20)
+    for _ in range(num_ops):
+        if fdp.remaining_bytes() == 0:
+            break
+        op = fdp.ConsumeIntInRange(METHOD_REVERSE, METHOD_TOBYTES)
+        if op == METHOD_REVERSE:
+            a.reverse()
+        elif op == METHOD_BYTESWAP:
+            a.byteswap()
+        elif op == METHOD_POP and len(a) > 0:
+            a.pop()
+        elif op == METHOD_COUNT and len(a) > 0:
+            val = fdp.ConsumeRandomValue()
+            a.count(val)
+        elif op == METHOD_INDEX and len(a) > 0:
+            val = fdp.ConsumeRandomValue()
+            try:
+                a.index(val)
+            except ValueError:
+                pass
+        elif op == METHOD_INSERT and len(a) > 0:
+            idx = fdp.ConsumeIntInRange(0, len(a) - 1)
+            val = fdp.ConsumeRandomValue()
+            a.insert(idx, val)
+        elif op == METHOD_REMOVE and len(a) > 0:
+            val = fdp.ConsumeRandomValue()
+            try:
+                a.remove(val)
+            except ValueError:
+                pass
+        elif op == METHOD_TOBYTES:
+            a.tobytes()
+
+def op_array_slice(fdp):
+    a, tc = _consume_array(fdp)
+    if len(a) < 2:
+        return
+    start = fdp.ConsumeIntInRange(0, len(a) - 1)
+    end = fdp.ConsumeIntInRange(start, len(a))
+    _ = a[start:end]
+    b = array.array(tc, a[start:end])
+    a[start:end] = b
+
+# Fuzzes the array module's C implementation (Modules/arraymodule.c).
+# Exercises array construction from raw bytes via frombytes(), element-level
+# operations (reverse, byteswap, pop, count, index, insert, remove), and
+# slice read/write across all 12 typecodes (b/B/h/H/i/I/l/L/q/Q/f/d).
+def FuzzerRunOne(FuzzerInput):
+    if len(FuzzerInput) < 1 or len(FuzzerInput) > 0x10000:
+        return
+    fdp = FuzzedDataProvider(FuzzerInput)
+    op = fdp.ConsumeIntInRange(OP_FROMBYTES, OP_SLICE)
+    try:
+        if op == OP_FROMBYTES:
+            op_array_frombytes(fdp)
+        elif op == OP_METHODS:
+            op_array_methods(fdp)
+        elif op == OP_SLICE:
+            op_array_slice(fdp)
+    except Exception:
+        pass

fuzz_binascii.py

@@ -0,0 +1,90 @@
+from fuzz_dp import FuzzedDataProvider
+import binascii
+
+# Top-level operation constants for FuzzerRunOne
+OP_DECODE = 0
+OP_ENCODE = 1
+OP_CHECKSUM = 2
+OP_ROUNDTRIP = 3
+
+# Decode/encode sub-operation constants
+CODEC_BASE64_STRICT = 0
+CODEC_HEX = 1
+CODEC_UU = 2
+CODEC_QP = 3
+CODEC_BASE64 = 4
+CODEC_BASE64_ALT = 5
+
+def op_decode(fdp):
+    which = fdp.ConsumeIntInRange(CODEC_BASE64_STRICT, CODEC_BASE64_ALT)
+    strict = fdp.ConsumeBool()
+    data = fdp.ConsumeBytes(fdp.remaining_bytes())
+    if which == CODEC_BASE64_STRICT:
+        if strict:
+            binascii.a2b_base64(data, strict_mode=True)
+        else:
+            binascii.a2b_base64(data)
+    elif which == CODEC_HEX:
+        binascii.a2b_hex(data)
+    elif which == CODEC_UU:
+        binascii.a2b_uu(data)
+    elif which == CODEC_QP:
+        binascii.a2b_qp(data)
+    elif which == CODEC_BASE64:
+        binascii.a2b_base64(data)
+    elif which == CODEC_BASE64_ALT:
+        binascii.a2b_base64(data)
+
+def op_encode(fdp):
+    which = fdp.ConsumeIntInRange(CODEC_BASE64_STRICT, CODEC_BASE64_ALT)
+    newline = fdp.ConsumeBool()
+    data = fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, 10000))
+    if not data:
+        return
+    if which == CODEC_BASE64_STRICT:
+        binascii.b2a_base64(data, newline=newline)
+    elif which == CODEC_HEX:
+        binascii.b2a_hex(data)
+    elif which == CODEC_UU:
+        uu_data = fdp.ConsumeBytes(fdp.ConsumeIntInRange(0, 45))
+        binascii.b2a_uu(uu_data)
+    elif which == CODEC_QP:
+        binascii.b2a_qp(data)
+    elif which == CODEC_BASE64:
+        binascii.b2a_base64(data)
+    elif which == CODEC_BASE64_ALT:
+        binascii.b2a_base64(data)
+
+def op_checksum(fdp):
+    use_crc32 = fdp.ConsumeBool()
+    data = fdp.ConsumeBytes(fdp.remaining_bytes())
+    if use_crc32:
+        binascii.crc32(data)
+    else:
+        binascii.crc_hqx(data, 0)
+
+def op_roundtrip(fdp):
+    data = fdp.ConsumeBytes(fdp.remaining_bytes())
+    hexed = binascii.hexlify(data)
+    binascii.unhexlify(hexed)
+
+# Fuzzes the binascii module's C implementation (Modules/binascii.c).
+# Exercises binary-to-ASCII and ASCII-to-binary conversions for base64,
+# hex, UU-encoding, and quoted-printable codecs. Also tests CRC32,
+# CRC-HQX checksums, and hexlify/unhexlify roundtrips.
+def FuzzerRunOne(FuzzerInput):
+    if len(FuzzerInput) < 1 or len(FuzzerInput) > 0x100000:
+        return
+    fdp = FuzzedDataProvider(FuzzerInput)
+    op = fdp.ConsumeIntInRange(OP_DECODE, OP_ROUNDTRIP)
+    try:
+        if op == OP_DECODE:
+            op_decode(fdp)
+        elif op == OP_ENCODE:
+            op_encode(fdp)
+        elif op == OP_CHECKSUM:
+            op_checksum(fdp)
+        else:
+            op_roundtrip(fdp)
+    except Exception:
+        pass

fuzz_codecs.py

@@ -0,0 +1,92 @@
+from fuzz_dp import FuzzedDataProvider
+import codecs
+import io
+
+DECODERS = [
+    "utf-7", "shift_jis", "euc-jp", "gb2312", "big5", "iso-2022-jp",
+    "euc-kr", "gb18030", "big5hkscs", "charmap", "ascii", "latin-1",
+    "cp1252", "unicode_escape", "raw_unicode_escape", "utf-16", "utf-32",
+]
+
+ENCODERS = [
+    "shift_jis", "euc-jp", "gb2312", "big5", "iso-2022-jp", "euc-kr",
+    "gb18030", "big5hkscs", "unicode_escape", "raw_unicode_escape",
+    "utf-7", "utf-8", "utf-16", "utf-16-le", "utf-16-be", "utf-32",
+    "latin-1", "ascii", "charmap",
+]
+
+INC_DEC_CODECS = ["shift_jis", "gb18030", "utf-16"]
+INC_ENC_CODECS = ["shift_jis", "utf-8"]
+
+OP_DECODE = 0
+OP_ENCODE = 1
+OP_INCREMENTAL_DECODE = 2
+OP_INCREMENTAL_ENCODE = 3
+OP_STREAM_READ = 4
+
+def op_decode(fdp):
+    codec = fdp.PickValueInList(DECODERS)
+    data = fdp.ConsumeBytes(fdp.remaining_bytes())
+    codecs.decode(data, codec, 'replace')
+
+def op_encode(fdp):
+    codec = fdp.PickValueInList(ENCODERS)
+    n = fdp.ConsumeIntInRange(1, min(fdp.remaining_bytes(), 10000)) if fdp.remaining_bytes() > 0 else 0
+    if n == 0:
+        return
+    s = fdp.ConsumeUnicode(n)
+    codecs.encode(s, codec, 'replace')
+
+def op_incremental_decode(fdp):
+    codec = fdp.PickValueInList(INC_DEC_CODECS)
+    chunk1_size = fdp.ConsumeIntInRange(0, 10000)
+    chunk1 = fdp.ConsumeBytes(chunk1_size)
+    chunk2 = fdp.ConsumeBytes(fdp.remaining_bytes())
+    decoder = codecs.getincrementaldecoder(codec)('replace')
+    decoder.decode(chunk1)
+    decoder.decode(chunk2, True)
+    decoder.getstate()
+    decoder.reset()
+
+def op_incremental_encode(fdp):
+    codec = fdp.PickValueInList(INC_ENC_CODECS)
+    n = fdp.ConsumeIntInRange(1, min(fdp.remaining_bytes(), 10000)) if fdp.remaining_bytes() > 0 else 0
+    if n == 0:
+        return
+    s = fdp.ConsumeUnicode(n)
+    split = fdp.ConsumeIntInRange(0, len(s))
+    encoder = codecs.getincrementalencoder(codec)('replace')
+    encoder.encode(s[:split])
+    encoder.reset()
+    encoder.encode(s[split:])
+    encoder.getstate()
+
+def op_stream(fdp):
+    data = fdp.ConsumeBytes(fdp.remaining_bytes())
+    bio = io.BytesIO(data)
+    reader = codecs.getreader('utf-8')(bio, 'replace')
+    reader.read()
+
+# Fuzzes CPython's codec infrastructure (Modules/cjkcodecs/, Python/codecs.c).
+# Exercises full and incremental encode/decode for CJK codecs (Shift-JIS,
+# EUC-JP, GB2312, Big5, ISO-2022-JP, EUC-KR, GB18030, Big5-HKSCS) and
+# Western/Unicode codecs (UTF-7/16/32, charmap, unicode_escape, latin-1).
+# Also tests stream-based reading via codecs.getreader().
+def FuzzerRunOne(FuzzerInput):
+    if len(FuzzerInput) < 1 or len(FuzzerInput) > 0x100000:
+        return
+    fdp = FuzzedDataProvider(FuzzerInput)
+    op = fdp.ConsumeIntInRange(OP_DECODE, OP_STREAM_READ)
+    try:
+        if op == OP_DECODE:
+            op_decode(fdp)
+        elif op == OP_ENCODE:
+            op_encode(fdp)
+        elif op == OP_INCREMENTAL_DECODE:
+            op_incremental_decode(fdp)
+        elif op == OP_INCREMENTAL_ENCODE:
+            op_incremental_encode(fdp)
+        else:
+            op_stream(fdp)
+    except Exception:
+        pass