Enable debugger testing in CI (#50)

Fix hang & improves tests related to the debugger + activate Debugger tests in CI

.github/workflows/mypytest.yml

@@ -7,11 +7,11 @@ on: [push, workflow_dispatch]
 jobs:
   build:
     runs-on: windows-latest
+    timeout-minutes: 15
     strategy:
       fail-fast: false
       matrix:
         python-version: [2.7, 3.6, 3.11]
-        # python-version: [3.6] # Concentrate on working CI for python3 & we will see backport for py2 afterward
         python-architecture: [x86, x64]
         include:
             # Translate architecture to bitness for py.exe commandline
@@ -61,7 +61,7 @@ jobs:
 
       # Testing
       - name: Testing
-        run: py -${{ matrix.python-version}}-${{ matrix.python-bitness-to-test}} -m pytest --junitxml=junit/test-results.xml -k "not debugger and not known_to_fail" -v tests/
+        run: py -${{ matrix.python-version}}-${{ matrix.python-bitness-to-test}} -m pytest --junitxml=junit/test-results.xml -s -k "not known_to_fail" -v tests/
 
       - name: Publish PyTest Results
         uses: EnricoMi/publish-unit-test-result-action/composite@v1

tests/test_debugger.py

@@ -2,6 +2,7 @@ import pytest
 import textwrap
 import ctypes
 import os
+import time
 
 import windows
 import windows.debug
@@ -133,6 +134,8 @@ def test_four_hwx_breakpoint_fail(proc32_64_debug):
     proc32_64_debug.create_thread(addr, 0)
     with pytest.raises(ValueError) as e:
         d.loop()
+    d.detach()
+    proc32_64_debug.exit()
     assert "DRx" in e.value.args[0]
 
 
@@ -186,7 +189,9 @@ def test_simple_breakpoint_name_addr(proc32_64_debug, bptype):
             TSTBP.COUNTER += 1
             d.current_process.exit()
 
+    # import pdb; pdb.set_trace()
     d = windows.debug.Debugger(proc32_64_debug)
+    # Broken in Win11 for now: https://twitter.com/hakril/status/1555473886321549312
     d.add_bp(TSTBP("ntdll!LdrLoadDll"))
     d.loop()
     assert TSTBP.COUNTER == 1
@@ -372,9 +377,19 @@ def test_standard_breakpoint_self_remove(proc32_64_debug, bptype):
     data = set()
 
     def do_check():
-        proc32_64_debug.execute_python_unsafe("open(u'FILENAME1')").wait()
-        proc32_64_debug.execute_python_unsafe("open(u'FILENAME2')").wait()
-        proc32_64_debug.execute_python_unsafe("open(u'FILENAME3')").wait()
+        time.sleep(1)
+        print("[==================] LOADING PYTHON")
+        proc32_64_debug.execute_python_unsafe("1").wait()
+        print("[==================] OPEN SELF_FILENAME1")
+        proc32_64_debug.execute_python_unsafe("open(u'SELF_FILENAME1')").wait()
+        time.sleep(0.1)
+        print("[==================] OPEN SELF_FILENAME2")
+        proc32_64_debug.execute_python_unsafe("open(u'SELF_FILENAME2')").wait()
+        time.sleep(0.1)
+        print("[==================] OPEN SELF_FILENAME3")
+        proc32_64_debug.execute_python_unsafe("open(u'SELF_FILENAME3')").wait()
+        time.sleep(0.1)
+        print("[==================] KILLING TARGET")
         proc32_64_debug.exit()
 
     class TSTBP(bptype):
@@ -384,15 +399,27 @@ def test_standard_breakpoint_self_remove(proc32_64_debug, bptype):
             ctx = dbg.current_thread.context
             filename = self.extract_arguments(dbg.current_process, dbg.current_thread)["lpFileName"]
             data.add(filename)
-            if filename == u"FILENAME2":
+            print("[+++++++++++++++++] Filename: {0}".format(filename))
+            if filename == u"SELF_FILENAME2":
+                print("[+++++++++++++++++] del_bp")
                 dbg.del_bp(self)
 
     d = windows.debug.Debugger(proc32_64_debug)
     d.add_bp(TSTBP("kernelbase!CreateFileW"))
     threading.Thread(target=do_check).start()
     d.loop()
-    assert data >= set([u"FILENAME1", u"FILENAME2"])
-    assert u"FILENAME3" not in data
+    assert data >= set([u"SELF_FILENAME1", u"SELF_FILENAME2"])
+    assert u"SELF_FILENAME3" not in data
+
+class MyMetaDbgDebuger(windows.debug.Debugger):
+    def on_exception(self, exc):
+        print(exc)
+        import pdb;pdb.set_trace()
+        print(exc)
+        x = 2
+        if x == 3:
+            return gdef.DBG_EXCEPTION_NOT_HANDLED
+        return gdef.DBG_CONTINUE
 
 @pytest.mark.timeout(DEFAULT_DEBUGGER_TIMEOUT)
 @python_injection
@@ -401,10 +428,21 @@ def test_standard_breakpoint_remove(proc32_64_debug, bptype):
     data = set()
 
     def do_check():
+        time.sleep(1)
+        print("[==================] LOADING PYTHON")
+        proc32_64_debug.execute_python_unsafe("1").wait()
+        print("[==================] OPEN FILENAME1")
         proc32_64_debug.execute_python_unsafe("open(u'FILENAME1')").wait()
+        time.sleep(0.1)
+        print("[==================] OPEN FILENAME2")
         proc32_64_debug.execute_python_unsafe("open(u'FILENAME2')").wait()
+        time.sleep(0.1)
+        print("[==================] RM BP")
         d.del_bp(the_bp)
+        print("[==================] OPEN FILENAME3")
         proc32_64_debug.execute_python_unsafe("open(u'FILENAME3')").wait()
+        time.sleep(0.1)
+        print("[==================] KILLING TARGET")
         proc32_64_debug.exit()
 
     class TSTBP(bptype):
@@ -413,18 +451,22 @@ def test_standard_breakpoint_remove(proc32_64_debug, bptype):
             addr = exc.ExceptionRecord.ExceptionAddress
             ctx = dbg.current_thread.context
             filename = self.extract_arguments(dbg.current_process, dbg.current_thread)["lpFileName"]
+            print("[+++++++++++++++++] Filename: {0}".format(filename))
             data.add(filename)
 
     d = windows.debug.Debugger(proc32_64_debug)
+    # d = MyMetaDbgDebuger(proc32_64_debug)
     the_bp = TSTBP("kernelbase!CreateFileW")
     # import pdb;pdb.set_trace()
     d.add_bp(the_bp)
+    time.sleep(0.1)
     threading.Thread(target=do_check).start()
     d.loop()
     assert data >= set([u"FILENAME1", u"FILENAME2"])
     assert u"FILENAME3" not in data
 
 
+
 def get_generate_read_at_for_proc(target):
     if target.bitness == 32:
         def generate_read_at(addr):

windows/debug/debugger.py

@@ -127,9 +127,9 @@ class Debugger(object):
             else:
                 self.del_bp(bp, [target])
 
+        del self._breakpoint_to_reput[target.pid]
         for thread in [t for t in target.threads if t.tid in self.threads]:
             del self._explicit_single_step[thread.tid]
-            del self._breakpoint_to_reput[thread.tid]
             del self.threads[thread.tid]
             ctx = thread.context
             if ctx.EEFlags.TF:  # Remove TRAPFlag before detaching (or it will lead to a crash)
@@ -203,6 +203,10 @@ class Debugger(object):
             yield debug_event
 
     def _finish_debug_event(self, event, action):
+        if self.current_thread:
+            dbgprint("Finishing event for TID <{0}>".format(self.current_thread.tid), "DBG")
+        else:
+            dbgprint("Finishing event", "DBG")
         if action not in [windef.DBG_CONTINUE, windef.DBG_EXCEPTION_NOT_HANDLED]:
             raise ValueError('Unknow action : <0>'.format(action))
         winproxy.ContinueDebugEvent(event.dwProcessId, event.dwThreadId, action)
@@ -243,6 +247,10 @@ class Debugger(object):
         mod = None
         if dll in modules:
             mod = [modules[dll]]
+        elif target.is_wow_64 and dll == "ntdll" and "ntdll32" in modules:
+            # https://twitter.com/hakril/status/1555473886321549312
+            mod = [modules["ntdll32"]]
+
         if not mod:
             return None
         # TODO: optim exports are the same for whole system (32 vs 64 bits)
@@ -284,12 +292,12 @@ class Debugger(object):
             return _setup_method(bp, target)
 
     def _restore_breakpoints(self):
-        for bp in self._breakpoint_to_reput[self.current_thread.tid]:
+        for bp in self._breakpoint_to_reput[self.current_process.pid]:
             if bp.type == HARDWARE_EXEC_BP:
                 raise NotImplementedError("Why is this here ? we use RF flags to pass HXBP")
             restore = getattr(self, "_restore_breakpoint_" + bp.type)
             restore(bp, self.current_process)
-        del self._breakpoint_to_reput[self.current_thread.tid][:]
+        self._breakpoint_to_reput[self.current_process.pid].clear()
         return
 
     def _setup_breakpoint_BP(self, bp, target):
@@ -486,7 +494,9 @@ class Debugger(object):
                 target_dll = bp.addr.lower().split("!")[0]
                 # Cannot work AS-IS yet. Implement it ?
                 # if target_dll == "*" or target_dll == dll_name:
-                if target_dll == dll_name:
+                if (target_dll == dll_name or
+                    # https://twitter.com/hakril/status/1555473886321549312
+                    (self.current_process.is_wow_64 and target_dll == "ntdll" and dll_name == "ntdll32")):
                     _setup_method = getattr(self, "_setup_breakpoint_" + bp.type)
                     if bp.apply_to_target(self.current_process):
                         _setup_method(bp, self.current_process)
@@ -497,7 +507,9 @@ class Debugger(object):
         for bp in self._pending_breakpoints_new[self.current_process.pid]:
             if isinstance(bp.addr, basestring):
                 target_dll = bp.addr.split("!")[0]
-                if target_dll == dll_name:
+                if (target_dll == dll_name or
+                    # https://twitter.com/hakril/status/1555473886321549312
+                    (self.current_process.is_wow_64 and target_dll == "ntdll" and dll_name == "ntdll32")):
                     _setup_method = getattr(self, "_setup_breakpoint_" + bp.type)
                     _setup_method(bp, self.current_process)
 
@@ -505,7 +517,9 @@ class Debugger(object):
             for bp in self._pending_breakpoints_new[thread.tid]:
                 if isinstance(bp.addr, basestring):
                     target_dll = bp.addr.split("!")[0]
-                    if target_dll == dll_name:
+                if (target_dll == dll_name or
+                    # https://twitter.com/hakril/status/1555473886321549312
+                    (self.current_process.is_wow_64 and target_dll == "ntdll" and dll_name == "ntdll32")):
                         _setup_method = getattr(self, "_setup_breakpoint_" + bp.type)
                         _setup_method(bp, self.thread)
 
@@ -518,7 +532,7 @@ class Debugger(object):
         #regs.pc -= 1 # Done in _handle_exception_breakpoint before dispatch
         thread.set_context(regs)
         bp = self.breakpoints[self.current_process.pid][addr]
-        self._breakpoint_to_reput[thread.tid].append(bp) #Register pending breakpoint for next single step
+        self._breakpoint_to_reput[process.pid].add(bp) #Register pending breakpoint for next single step
 
     def _pass_memory_breakpoint(self, bp, page_protect, fault_page):
         cp = self.current_process
@@ -529,7 +543,7 @@ class Debugger(object):
         ctx.EEFlags.TF = 1
         thread.set_context(ctx)
         bp._reput_page = (fault_page, page_prot.value)
-        self._breakpoint_to_reput[thread.tid].append(bp)
+        self._breakpoint_to_reput[cp.pid].add(bp)
 
     # debug event handlers
     def _handle_unknown_debug_event(self, debug_event):
@@ -582,7 +596,7 @@ class Debugger(object):
             return self.on_exception(exception)
 
     def _handle_exception_singlestep(self, exception, excp_addr):
-        if self.current_thread.tid in self._breakpoint_to_reput and self._breakpoint_to_reput[self.current_thread.tid]:
+        if self._breakpoint_to_reput.get(self.current_process.pid):
             self._restore_breakpoints()
             if self._explicit_single_step[self.current_thread.tid]:
                 with self.DisabledMemoryBreakpoint():
@@ -696,17 +710,17 @@ class Debugger(object):
         excp_code = exception.ExceptionRecord.ExceptionCode
         excp_addr = exception.ExceptionRecord.ExceptionAddress
         if excp_code in [EXCEPTION_BREAKPOINT, STATUS_WX86_BREAKPOINT]:
-            dbgprint("Handle exception as breakpoint", "DBG")
+            dbgprint("Handle exception as breakpoint in TID {0}".format(self.current_thread.tid), "DBG")
             return self._handle_exception_breakpoint(exception, excp_addr)
         elif excp_code in [EXCEPTION_SINGLE_STEP, STATUS_WX86_SINGLE_STEP]:
-            dbgprint("Handle exception as single step", "DBG")
+            dbgprint("Handle exception as single step in TID {0}".format(self.current_thread.tid), "DBG")
             return self._handle_exception_singlestep(exception, excp_addr)
         elif excp_code == EXCEPTION_ACCESS_VIOLATION:
-            dbgprint("Handle exception as access_violation", "DBG")
+            dbgprint("Handle exception as access_violation in TID {0}".format(self.current_thread.tid), "DBG")
             return self._handle_exception_access_violation(exception, excp_addr)
         else:
             with self.DisabledMemoryBreakpoint():
-                dbgprint("Handle exception as on_exception", "DBG")
+                dbgprint("Handle exception as on_exception".format(self.current_thread.tid), "DBG")
                 continue_flag = self.on_exception(exception)
             if self._killed_in_action():
                 return continue_flag
@@ -765,10 +779,10 @@ class Debugger(object):
         self.threads[self.current_thread.tid] = self.current_thread
         self._explicit_single_step[self.current_thread.tid] = False
         self._hardware_breakpoint[self.current_thread.tid] = {}
-        self._breakpoint_to_reput[self.current_thread.tid] = []
         self.processes[self.current_process.pid] = self.current_process
         self._watched_pages[self.current_process.pid] = {} #defaultdict(list)
         self.breakpoints[self.current_process.pid] = {}
+        self._breakpoint_to_reput[self.current_process.pid] = set()
         self._memory_save[self.current_process.pid] = {}
         self._module_by_process[self.current_process.pid] = {}
         self._internal_on_create_process(create_process) # Allow hook for symbol-debugger
@@ -791,7 +805,7 @@ class Debugger(object):
         del self.threads[self.current_thread.tid]
         del self._explicit_single_step[self.current_thread.tid]
         del self._hardware_breakpoint[self.current_thread.tid]
-        del self._breakpoint_to_reput[self.current_thread.tid]
+        del self._breakpoint_to_reput[self.current_process.pid]
         del self.processes[self.current_process.pid]
         del self._watched_pages[self.current_process.pid]
         del self._memory_save[self.current_process.pid]
@@ -818,7 +832,6 @@ class Debugger(object):
         # The new thread is on the thread pool: we can now update the debugger state
         self._update_debugger_state(debug_event)
         self._explicit_single_step[self.current_thread.tid] = False
-        self._breakpoint_to_reput[self.current_thread.tid] = []
         self._hardware_breakpoint[self.current_thread.tid] = {}
         self._setup_pending_breakpoints_new_thread(self.current_thread)
         with self.DisabledMemoryBreakpoint():
@@ -834,7 +847,6 @@ class Debugger(object):
         del self.threads[self.current_thread.tid]
         del self._hardware_breakpoint[self.current_thread.tid]
         del self._explicit_single_step[self.current_thread.tid]
-        del self._breakpoint_to_reput[self.current_thread.tid]
         return retvalue
 
     def _internal_on_create_process(self, create_process):
@@ -853,6 +865,8 @@ class Debugger(object):
             dll_name = dll_name[:-4]
         # Mais c'est debile..
         # Si j'ai ntdll et ntdll64: les deux vont avoir le meme nom..
+        # A changer sur Win11 ou ntdll est remontee en tant que ntdll32.dll
+        #   https://twitter.com/hakril/status/1555473886321549312
         if dll_name.endswith(".dll64"):
             dll_name = dll_name[:-6] +  "64" # Crade..
         #print("Load {0} -> {1}".format(dll, dll_name))
@@ -915,7 +929,7 @@ class Debugger(object):
         """
         if getattr(bp, "addr", None) is None:
             if addr is None or type is None:
-                raise ValueError("SUCK YOUR NONE")
+                raise ValueError("Breakpoing should have addr attribute or function should receive explicite addr parameter")
             bp = ProxyBreakpoint(bp, addr, type)
         else:
             if addr is not None or type is not None: