Improve performance with nested PacketFields

scapy/fields.py

@@ -1478,6 +1478,8 @@ class StrField(_StrField[bytes]):
 class StrFieldUtf16(StrField):
     def any2i(self, pkt, x):
         # type: (Optional[Packet], Optional[str]) -> bytes
+        if isinstance(x, bytes):
+            return x
         if isinstance(x, str):
             return self.h2i(pkt, x)
         return super(StrFieldUtf16, self).any2i(pkt, x)
@@ -1619,6 +1621,70 @@ def getfield(self,
         return s[len_pkt:], i
 
 
+class _ListValueMeta(type):
+    """
+    Wraps modifying methods for ``list`` base type.
+
+    Inspired from https://stackoverflow.com/questions/8858525/track-changes-to-lists-and-dictionaries-in-python#8859168.
+    """
+    def __new__(
+            mcs,
+            name,  # type: str
+            bases,  # Tuple[type, ...]
+            attrs,  # type: Dict[str, Any]
+    ):  # type: (...) -> type
+        # List names of `list` methods modifying the list.
+        for method_name in [
+            "append",
+            "clear",
+            "extend",
+            "insert",
+            "pop",
+            "remove",
+            "reverse",  # Memo: Reverse *IN PLACE*.
+            "sort",  # Memo: Stable sort *IN PLACE*.
+            "__delitem__",
+            "__iadd__",
+            "__imul__",
+            "__setitem__",
+        ]:
+            # Wrap the method so that `Packet.clear_cache()` be automatically called.
+            attrs[method_name] = _ListValueMeta._wrap_method(getattr(list, method_name))
+        return type.__new__(mcs, name, bases, attrs)
+
+    @staticmethod
+    def _wrap_method(meth):  # type: (Callable[[Any, ...], Any]) -> Callable[[Any, ...], Any]
+        def wrapped(
+                self,  # type: ListValue
+                *args,  # type: Any
+                **kwargs,  # type: Any
+        ):  # type: (...) -> Any
+            # Automatically call `Packet.clear_cache()` when the `ListValue` is modified.
+            self.pkt.clear_cache(upwards=True, downwards=False)
+
+            # Call the wrapped method, and return its result.
+            return meth(self, *args, **kwargs)
+        return wrapped
+
+
+class ListValue(list, metaclass=_ListValueMeta):
+    """
+    Overrides the base ``list`` type for list fields bound with packets.
+
+    Ensures ``Packet.clear_cache()`` is called when the list is modified.
+    """
+    def __init__(
+            self,
+            pkt,  # type: Packet,
+            *args  # type: Any
+    ):  # type: (...) -> None
+        # Call the `list.__init__()` super constructor.
+        super().__init__(*args)
+
+        #: Packet bound with this list field.
+        self.pkt = pkt
+
+
 class PacketListField(_PacketField[List[BasePacket]]):
     """PacketListField represents a list containing a series of Packet instances
     that might occur right in the middle of another Packet field.
@@ -2973,7 +3039,7 @@ def __next__(self):
 
 
 class FlagValue(object):
-    __slots__ = ["value", "names", "multi"]
+    __slots__ = ["pkt", "value", "names", "multi"]
 
     def _fixvalue(self, value):
         # type: (Any) -> int
@@ -2990,6 +3056,12 @@ def _fixvalue(self, value):
 
     def __init__(self, value, names):
         # type: (Union[List[str], int, str], Union[List[str], str]) -> None
+
+        #: Packet bound with this flag value.
+        #:
+        #: Ensures ``Packet.clear_cache()`` is called when the flags are modified.
+        self.pkt = None  # type: Optional[Packet]
+
         self.multi = isinstance(names, list)
         self.names = names
         self.value = self._fixvalue(value)
@@ -3125,6 +3197,10 @@ def __setattr__(self, attr, value):
                 self.value |= (2 ** self.names.index(attr))
             else:
                 self.value &= ~(2 ** self.names.index(attr))
+
+            # Automatically call `Packet.clear_cache()` when the flags are modified.
+            if self.pkt is not None:
+                self.pkt.clear_cache(upwards=True, downwards=True)
         else:
             return super(FlagValue, self).__setattr__(attr, value)
 

scapy/layers/bluetooth4LE.py

@@ -214,6 +214,9 @@ class BTLE(Packet):
         X3BytesField("crc", None)
     ]
 
+    # Avoid caching with this layer for data reordering to work properly.
+    no_cache = True
+
     @staticmethod
     def compute_crc(pdu, init=0x555555):
         def swapbits(a):

scapy/layers/dcerpc.py

@@ -1292,6 +1292,9 @@ def _e(ndrendian):
 class _NDRPacket(Packet):
     __slots__ = ["ndr64", "ndrendian", "deferred_pointers", "request_packet"]
 
+    # Can't trust the cache due to deferred pointers.
+    no_cache = True
+
     def __init__(self, *args, **kwargs):
         self.ndr64 = kwargs.pop("ndr64", False)
         self.ndrendian = kwargs.pop("ndrendian", "little")
@@ -1313,12 +1316,6 @@ def do_dissect(self, s):
             )
         return super(_NDRPacket, self).do_dissect(s)
 
-    def post_dissect(self, s):
-        if self.deferred_pointers:
-            # Can't trust the cache if there were deferred pointers
-            self.raw_packet_cache = None
-        return s
-
     def do_build(self):
         _up = self.parent or self.underlayer
         for f in self.fields.values():

scapy/layers/dhcp.py

@@ -52,6 +52,7 @@
     RandInt,
     RandNum,
     RandNumExpo,
+    VolatileValue,
 )
 
 from scapy.arch import get_if_hwaddr
@@ -401,6 +402,15 @@ class DHCPOptionsField(StrField):
     islist = 1
 
     def i2repr(self, pkt, x):
+        # `x` may happen to be a `RandDHCPOptions` instance.
+        # Example:
+        # ```python
+        # pkt = fuzz(DHCP())
+        # pkt.show()
+        # ```
+        if isinstance(x, VolatileValue):
+            return repr(x)
+
         s = []
         for v in x:
             if isinstance(v, tuple) and len(v) >= 2:
@@ -471,6 +481,10 @@ def m2i(self, pkt, x):
         return opt
 
     def i2m(self, pkt, x):
+        # Ensure the `RandDHCPOptions` instance has been resolved before (see `Packet.self_build()`).
+        #
+        assert not isinstance(x, VolatileValue), f"Bad value {x!r}, should have been resolved before"
+
         if isinstance(x, str):
             return x
         s = b""

scapy/layers/ntlm.py

@@ -152,6 +152,10 @@ def h2i(self, pkt, x):
         # type: (Optional[Packet], bytes) -> List[Tuple[str, str]]
         return self._on_payload(pkt, x, "h2i")
 
+    def any2i(self, pkt, x):
+        # type: (Optional[Packet], bytes) -> List[Tuple[str, str]]
+        return self._on_payload(pkt, x, "any2i")
+
     def i2repr(self, pkt, x):
         # type: (Optional[Packet], bytes) -> str
         return repr(self._on_payload(pkt, x, "i2repr"))