Replace yappcap with scapy for packet capture.

Yappcap will never build again because of ancient Cython so it's been
replaced with "scapy" which is popular and well maintained.

* The extra.txt requirements file was removed as it's no longer needed.

* The verify_pidf and verify_xpidf tests needed minor tweaks to make them
reliable because the tests ended with the sipp scenarios before the
validation could complete.

* The callee_local_direct_media and caller_local_direct_media tests indicated
that they needed yappcap and pyxb but they actually need neither so those
dependencies were removed.  The tests still have issues however so they
remain skipped.

* The codec_opus tests all fail for other reasons but they never get run
anyway. Issue opened.

Author: gtjoseph

extras.txt

@@ -26,7 +26,7 @@
 from construct.core import *
 from asterisk.pcap_proxy import *
 try:
-    from yappcap import PcapOffline
+    from scapy import interfaces, sendrecv, packet
     PCAP_AVAILABLE = True
 except:
     PCAP_AVAILABLE = False

lib/python/asterisk/pcap.py

@@ -58,7 +58,7 @@ def __init__(self, packet_type, raw_packet):
             self.ip_layer = None
             self.transport_layer = None
         else:
-            self.eth_layer = ip_stack.parse(bytes(raw_packet.data))
+            self.eth_layer = ip_stack.parse(bytes(raw_packet))
             self.ip_layer = self.eth_layer.next
             self.transport_layer = self.ip_layer.next
 
@@ -505,7 +505,7 @@ def interpret_packet(self, packet):
         """
         ret_packet = None
         if not isinstance(packet, str):
-            hex_string = packet.data[42:]
+            hex_string = packet[42:]
             ascii_string = hex_string.decode('ascii')
         else:
             ascii_string = packet

lib/python/asterisk/pcap_proxy.py

@@ -1,50 +1,15 @@
 from twisted.internet import abstract, protocol
-from yappcap import PcapLive, findalldevs, PcapTimeout
+import logging
+import scapy
+from scapy.all import *
+from scapy.config import conf
+import socket
+import os
 
+LOGGER = logging.getLogger(__name__)
 
-class PcapFile(abstract.FileDescriptor):
-    """Treat a live pcap capture as a file for Twisted to call select() on"""
-    def __init__(self, protocol, interface, xfilter=None, dumpfile=None,
-                 snaplen=65535, buffer_size=0):
-        abstract.FileDescriptor.__init__(self)
-
-        p = PcapLive(interface, autosave=dumpfile, snaplen=snaplen,
-                     buffer_size=buffer_size)
-        p.immediate = True
-        p.activate()
-        p.blocking = False
-
-        if xfilter is not None:
-            p.filter = xfilter
-
-        self.pcap = p
-        self.fd = p.selectable_fd
-        self.protocol = protocol
-        self.protocol.makeConnection(self)
-        self.startReading()
-
-    def fileno(self):
-        return self.fd
-
-    def doRead(self):
-        try:
-            pkt = next(self.pcap)
-        except PcapTimeout:
-            return 0
-
-        # we may not have a packet if something weird happens
-        if not pkt:
-            # according to the twisted docs 0 implies no write done
-            return 0
-
-        self.protocol.dataReceived(pkt)
-
-    def connectionLost(self, reason):
-        self.protocol.connectionLost(reason)
-
-
-class PcapListener(protocol.Protocol):
-    """A Twisted protocol wrapper for a pcap capture"""
+class PcapListener():
+    """A A wrapper for a scapy pcap capture"""
     def __init__(self, interface, bpf_filter=None, dumpfile=None,
                  callback=None, snaplen=None, buffer_size=None):
         """Initialize a new PcapListener
@@ -58,19 +23,41 @@ def __init__(self, interface, bpf_filter=None, dumpfile=None,
         buffer_size - The ring buffer size. If None, then 0.
 
         """
+
         if interface is None:
-            interface = [x.name for x in findalldevs() if x.loopback][0]
+            interface = self.find_first_loopback_device()
         if buffer_size is None:
-            buffer_size = 0
+            buffer_size = 8192
         if snaplen is None:
             snaplen = 65535
-        self.pf = PcapFile(self, interface, bpf_filter, dumpfile, snaplen,
-                           buffer_size)
+        self.pcap_writer = None
+
+        scapy.config.conf.use_pcap = True
         self.callback = callback
 
-    def makeConnection(self, transport):
-        self.connectionMade()
+        LOGGER.info("Starting capture.  if: %s dumpfile: %s snap: %d  bs: %d",
+                    interface, dumpfile, snaplen, buffer_size)
+
+        if dumpfile is not None:
+            self.pcap_writer = PcapWriter(dumpfile, sync=True,
+                nano=True, snaplen=snaplen, bufsz=buffer_size)
+
+        self.pf = AsyncSniffer(iface=interface,filter=bpf_filter,
+            prn=self._callback)
+
+        self.pf.start()
 
-    def dataReceived(self, data):
+    def _callback(self, packet: scapy.packet.Packet):
         if self.callback:
-            self.callback(data)
+            self.callback(packet.original)
+
+        if self.pcap_writer:
+            self.pcap_writer.write(packet)
+        return None
+
+    def find_first_loopback_device(self):
+        iflist = scapy.interfaces.get_working_ifaces()
+        for i in iflist:
+            if "LOOPBACK" in i.flags and "UP" in i.flags:
+                return i.name
+        return None

lib/python/pcap_listener.py

@@ -6,7 +6,6 @@ cffi==1.17.1
 charset-normalizer==2.0.12
 constantly==15.1.0
 construct==2.10.68
-Cython>=0.29.34,<3.0.0
 hyperlink==21.0.0
 idna==3.3
 incremental==24.7.2
@@ -27,3 +26,4 @@ typing_extensions==4.12.2
 urllib3==1.26.9
 zope.interface==7.0.1
 starpy @ git+https://github.com/asterisk/[email protected]
+scapy==2.6.1

requirements.txt

@@ -16,14 +16,8 @@ function do_pip_setup {
 	python3 -m pip install --upgrade pip
 	python3 -m pip install wheel setuptools build
 	python3 -m pip install -r ./requirements.txt
-	python3 -m pip install -r ./extras.txt || {
-		echo "**************************" >&2
-		echo "Some optional python requirements failed to install. The following tests may not run:" >&2
-		grep -lEr "python\s*:\s*[']?yappcap" tests >&2
-		echo "**************************" >&2
-	}
 	$REALTIME && python3 -m pip install -r ./requirements-realtime.txt
-	md5sum requirements.txt extras.txt requirements-realtime.txt > $1/checksums
+	md5sum requirements.txt requirements-realtime.txt > $1/checksums
 }
 
 if [[ "$VIRTUAL_ENV" != "" ]]

setupVenv.sh

@@ -22,7 +22,7 @@ properties:
             version : 'v3.0'
         - python: 'twisted'
         - python: 'starpy'
-        - python: 'yappcap'
+        - python: 'scapy'
         - asterisk: 'res_pjsip'
         - asterisk: 'res_pjsip_exten_state'
         - asterisk: 'res_pjsip_pidf_body_generator'
@@ -47,6 +47,7 @@ test-modules:
 sipp-config:
     reactor-timeout: 30
     fail-on-any: True
+    stop-after-scenarios: False
     test-iterations:
         -
             scenarios:

tests/channels/pjsip/subscriptions/presence/verify_bodies/verify_pidf/test-config.yaml

@@ -22,7 +22,7 @@ properties:
             version : 'v3.0'
         - python: 'twisted'
         - python: 'starpy'
-        - python: 'yappcap'
+        - python: 'scapy'
         - asterisk: 'res_pjsip'
         - asterisk: 'res_pjsip_exten_state'
         - asterisk: 'res_pjsip_xpidf_body_generator'
@@ -47,6 +47,7 @@ test-modules:
 sipp-config:
     reactor-timeout: 30
     fail-on-any: True
+    stop-after-scenarios: False
     test-iterations:
         -
             scenarios:

tests/channels/pjsip/subscriptions/presence/verify_bodies/verify_xpidf/test-config.yaml

@@ -52,7 +52,7 @@ def log_packet(packet, write_packet_contents):
     """Writes the contents of a SIP packet to the log.
 
     Keyword Arguments:
-    packet                  -- A yappcap.PcapPacket
+    packet                  -- A PcapPacket
     write_packet_contents   -- Whether or not to dump the contents of the
                                packet to the log.
     """

tests/channels/pjsip/subscriptions/rls/rls_test.py

@@ -105,10 +105,7 @@ properties:
     dependencies:
         - python : twisted
         - python : starpy
-        - python : yappcap
-        - python : pyxb
         - asterisk : res_pjsip
-        - custom : rawsocket
         - app : sipp
     tags:
         - pjsip