Working in new ctypes generation code and features

ctypes_generation/def_parser.py

@@ -7,12 +7,11 @@ class WinDef(object):
     def __init__(self, name, code):
         self.name = name
         self.code = code
-        
+
     def generate_ctypes(self):
         return """{0} = make_flag("{0}", {1})""".format(self.name, self.code)
 
 class WinDefParser(Parser):
-
     def parse_define(self):
         self.assert_token_type(SharpToken)
         define = self.assert_token_type(NameToken)
@@ -26,9 +25,26 @@ class WinDefParser(Parser):
                 v = v[:-1]
             define_value.append(v)
         return WinDef(define_name.value, " ".join(define_value))
-        
+
     def parse(self):
         res = []
         while self.peek() is not None:
             res.append(self.parse_define())
-        return res 
+        return res
+
+# A simple Fake parser for NTSTATUS
+class NtStatusParser(object):
+    def __init__(self, data):
+        self.input = data
+
+    def parse(self):
+        nt_status_defs = []
+        for line in self.input.split("\n"):
+            if not line:
+                continue
+            code, name, descr = line.split("|", 2)
+            code = int(code, 0)
+            descr = re.sub(" +", " ", descr[:-1]) # remove \n
+            descr = descr.replace('"', "'")
+            nt_status_defs.append((code, name, descr))
+        return nt_status_defs

ctypes_generation/definitions/com/template.py

@@ -0,0 +1,72 @@
+import functools
+import ctypes
+
+
+generate_IID = IID.from_raw
+
+class COMInterface(ctypes.c_void_p):
+    _functions_ = {
+    }
+
+    def __getattr__(self, name):
+        if name in self._functions_:
+            return functools.partial(self._functions_[name], self)
+        return super(COMInterface, self).__getattribute__(name)
+
+class COMImplementation(object):
+    IMPLEMENT = None
+
+    def get_index_of_method(self, method):
+        # This code is horrible but not totally my fault
+        # the PyCFuncPtrObject->index is not exposed to Python..
+        # repr is: '<COM method offset 2: WinFunctionType at 0x035DDBE8>'
+        rpr = repr(method)
+        if not rpr.startswith("<COM method offset ") or ":" not in rpr:
+            raise ValueError("Could not extract offset of {0}".format(rpr))
+        return int(rpr[len("<COM method offset "): rpr.index(":")])
+
+    def extract_methods_order(self, interface):
+        index_and_method = sorted((self.get_index_of_method(m),name, m) for name, m in interface._functions_.items())
+        return index_and_method
+
+    def verify_implem(self, interface):
+        for func_name in interface._functions_:
+            implem = getattr(self, func_name, None)
+            if implem is None:
+                raise ValueError("<{0}> implementing <{1}> has no method <{2}>".format(type(self).__name__, self.IMPLEMENT.__name__, func_name))
+            if not callable(implem):
+                raise ValueError("{0} implementing <{1}>: <{2}> is not callable".format(type(self).__name__, self.IMPLEMENT.__name__, func_name))
+        return True
+
+    def _create_vtable(self, interface):
+        implems = []
+        names = []
+        for index, name, method in self.extract_methods_order(interface):
+            func_implem = getattr(self, name)
+            #PVOID is 'this'
+            types = [method.restype, PVOID] + list(method.argtypes)
+            implems.append(ctypes.WINFUNCTYPE(*types)(func_implem))
+            names.append(name)
+        class Vtable(ctypes.Structure):
+            _fields_ = [(name, ctypes.c_void_p) for name in names]
+        return Vtable(*[ctypes.cast(x, ctypes.c_void_p) for x in implems]), implems
+
+    def __init__(self):
+        self.verify_implem(self.IMPLEMENT)
+        vtable, implems = self._create_vtable(self.IMPLEMENT)
+        self.vtable = vtable
+        self.implems = implems
+        self.vtable_pointer = ctypes.pointer(self.vtable)
+        self._as_parameter_ = ctypes.addressof(self.vtable_pointer)
+
+    def QueryInterface(self, this, piid, result):
+        if piid[0] in (IUnknown.IID, self.IMPLEMENT.IID):
+            result[0] = this
+            return 1
+        return E_NOINTERFACE
+
+    def AddRef(self, *args):
+        return 1
+
+    def Release(self, *args):
+        return 0

ctypes_generation/definitions/defines/template.py

@@ -0,0 +1,7 @@
+import platform
+from flag import make_flag
+
+bits = platform.architecture()[0]
+bitness =  int(bits[:2])
+
+NATIVE_WORD_MAX_VALUE = 0xffffffff if bitness == 32 else 0xffffffffffffffff

ctypes_generation/definitions/flag.py

@@ -0,0 +1,50 @@
+import sys
+
+if sys.version_info.major == 3:
+    long = int
+
+class Flag(long):
+    def __new__(cls, name, value):
+        return super(Flag, cls).__new__(cls, value)
+
+    def __init__(self, name, value):
+        self.name = name
+
+    def __repr__(self):
+        return "{0}({1})".format(self.name, hex(self))
+
+    __str__ = __repr__
+
+   # Fix pickling with protocol 2
+    def __getnewargs__(self, *args):
+        return self.name, long(self)
+
+class StrFlag(str):
+    def __new__(cls, name, value):
+        if isinstance(value, cls):
+            return value
+        return super(StrFlag, cls).__new__(cls, value)
+
+    def __init__(self, name, value):
+        self.name = name
+
+    def __repr__(self):
+        return "{0}({1})".format(self.name, str.__repr__(self))
+
+    # __str__ = __repr__
+
+    # Fix pickling with protocol 2
+    def __getnewargs__(self, *args):
+        return self.name, str.__str__(self)
+
+def make_flag(name, value):
+    if isinstance(value, (int, long)):
+        return Flag(name, value)
+    return StrFlag(name, value)
+
+class FlagMapper(dict):
+    def __init__(self, *values):
+        self.update({x:x for x in values})
+
+    def __missing__(self, key):
+        return key

ctypes_generation/definitions/ntstatus_template.py

@@ -0,0 +1,29 @@
+import ctypes
+from flag import Flag
+
+class NtStatusException(WindowsError):
+    ALL_STATUS = {}
+    def __init__(self , code):
+        try:
+            x = self.ALL_STATUS[code]
+        except KeyError:
+            x = (code, 'UNKNOW_ERROR', 'Error non documented in ntstatus.py')
+        self.code = x[0]
+        self.name = x[1]
+        self.descr = x[2]
+        x =  ctypes.c_long(x[0]).value, x[1], x[2]
+        return super(NtStatusException, self).__init__(*x)
+
+    def __str__(self):
+        return "{e.name}(0x{e.code:x}): {e.descr}".format(e=self)
+
+    def __repr__(self):
+        return "{0}(0x{1:08x}, {2})".format(type(self).__name__, self.code, self.name)
+
+    @classmethod
+    def register_ntstatus(cls, code, name, descr):
+        if code in cls.ALL_STATUS:
+            return # Use the first def
+        cls.ALL_STATUS[code] = (code, name, descr)
+        return Flag(name, code)
+

ctypes_generation/definitions/simple_types.txt

@@ -0,0 +1,91 @@
+VOID = DWORD // This one is a 'cheat' to handle function returning void in ctypes
+BYTE = c_ubyte
+PWSTR = LPWSTR
+PCWSTR = LPWSTR
+SIZE_T = c_size_t
+PSIZE_T = POINTER(SIZE_T)
+PVOID = c_void_p
+PPS_POST_PROCESS_INIT_ROUTINE = PVOID
+NTSTATUS = DWORD
+SECURITY_INFORMATION = DWORD
+PSECURITY_INFORMATION = POINTER(SECURITY_INFORMATION)
+PULONG = POINTER(ULONG)
+PDWORD = POINTER(DWORD)
+LPDWORD = POINTER(DWORD)
+LPTHREAD_START_ROUTINE = PVOID
+WNDENUMPROC = PVOID
+PHANDLER_ROUTINE = PVOID
+LPBYTE = POINTER(BYTE)
+ULONG_PTR = PVOID
+DWORD_PTR = ULONG_PTR
+KAFFINITY = ULONG_PTR
+KPRIORITY = LONG
+CHAR = c_char
+UCHAR = c_char
+CSHORT = c_short
+VARTYPE = c_ushort
+PBOOL = POINTER(BOOL)
+PSTR = LPSTR
+PCSTR = LPSTR
+va_list = c_char_p
+BSTR = c_wchar_p
+OLECHAR = c_wchar
+POLECHAR = c_wchar_p
+PUCHAR = POINTER(UCHAR)
+double = c_double
+FARPROC = PVOID
+PSID = PVOID
+PVECTORED_EXCEPTION_HANDLER = PVOID
+ULONGLONG = c_ulonglong
+LONGLONG = c_longlong
+ULONG64 = c_ulonglong
+UINT64 = ULONG64
+LONG64 = c_longlong
+PLARGE_INTEGER = POINTER(LARGE_INTEGER)
+DWORD64 = ULONG64
+SCODE = LONG
+CIMTYPE = LONG
+NET_IFINDEX = ULONG
+IF_INDEX = NET_IFINDEX
+IFTYPE = ULONG
+PULONG64 = POINTER(ULONG64)
+PBYTE = POINTER(BYTE)
+PUINT = POINTER(UINT)
+PHANDLE = POINTER(HANDLE)
+HCATADMIN = HANDLE
+HCATINFO = HANDLE
+HCERTCHAINENGINE = HANDLE
+LPHANDLE = POINTER(HANDLE)
+ALPC_HANDLE = HANDLE
+PALPC_HANDLE = POINTER(ALPC_HANDLE)
+PHKEY = POINTER(HKEY)
+ACCESS_MASK = DWORD
+REGSAM = ACCESS_MASK
+PBOOLEAN = POINTER(BOOLEAN)
+SECURITY_CONTEXT_TRACKING_MODE = BOOLEAN
+HCRYPTPROV_OR_NCRYPT_KEY_HANDLE = PULONG
+HCRYPTPROV_LEGACY = PULONG
+HCRYPTKEY = PULONG
+HCRYPTPROV = PULONG
+HCRYPTHASH = PULONG
+ALG_ID = UINT
+DISPID = LONG
+MEMBERID = DISPID
+PSECURITY_DESCRIPTOR = PVOID
+LPPROC_THREAD_ATTRIBUTE_LIST = PVOID
+LPUNKNOWN = POINTER(PVOID)
+LPFILETIME = POINTER(FILETIME)
+LPPOINT = POINTER(POINT)
+LPRECT = POINTER(RECT)
+SPC_UUID = BYTE * 16
+PIO_APC_ROUTINE = PVOID
+DEVICE_TYPE = DWORD
+PWINDBG_EXTENSION_APIS32 = PVOID
+PWINDBG_EXTENSION_APIS64 = PVOID
+
+// Will be changed at import time
+LPCONTEXT = PVOID
+HCERTSTORE = PVOID
+HCRYPTMSG = PVOID
+PALPC_PORT_ATTRIBUTES = PVOID
+PPORT_MESSAGE = PVOID

ctypes_generation/definitions/structures/display_struct.txt

@@ -1,11 +1,3 @@
-typedef struct tagRECT
-{
-    LONG    left;
-    LONG    top;
-    LONG    right;
-    LONG    bottom;
-} RECT, *PRECT, *NPRECT, *LPRECT;
-
 typedef struct tagRGBTRIPLE {
         BYTE    rgbtBlue;
         BYTE    rgbtGreen;
@@ -71,4 +63,10 @@ typedef struct tagBITMAPINFO {
 typedef struct tagBITMAPCOREINFO {
     BITMAPCOREHEADER    bmciHeader;
     RGBTRIPLE           bmciColors[1];
-} BITMAPCOREINFO, *LPBITMAPCOREINFO, *PBITMAPCOREINFO;
+} BITMAPCOREINFO, *LPBITMAPCOREINFO, *PBITMAPCOREINFO;
+
+/*
+typedef struct _FAKEYOLOSTRUCT {
+        ACL    Reserved;
+} _FAKEYOLOSTRUCT;
+*/

ctypes_generation/definitions/structures/template.py

@@ -0,0 +1,37 @@
+from ctypes import *
+from ctypes.wintypes import *
+
+from flag import Flag, FlagMapper
+from yolo import *
+
+class EnumValue(Flag):
+    def __new__(cls, enum_name, name, value):
+        return super(EnumValue, cls).__new__(cls, name, value)
+
+    def __init__(self, enum_name, name, value):
+        self.enum_name = enum_name
+        self.name = name
+
+    def __repr__(self):
+        return "{0}.{1}({2})".format(self.enum_name, self.name, hex(self))
+
+    # Fix pickling with protocol 2
+    def __getnewargs__(self, *args):
+        return self.enum_name, self.name, int(self)
+
+
+class EnumType(DWORD):
+    values = ()
+    mapper = {}
+
+    @property
+    def value(self):
+        raw_value = super(EnumType, self).value
+        return self.mapper.get(raw_value, raw_value)
+
+    def __repr__(self):
+        raw_value = super(EnumType, self).value
+        if raw_value in self.values:
+            value = self.value
+            return "<{0} {1}({2})>".format(type(self).__name__, value.name, hex(raw_value))
+        return "<{0}({1})>".format(type(self).__name__, hex(self.value))

ctypes_generation/definitions/structures/winstruct.txt

@@ -2447,12 +2447,6 @@ typedef struct WINTRUST_SGNR_INFO_
     HCERTSTORE      *pahStores;
 } WINTRUST_SGNR_INFO, *PWINTRUST_SGNR_INFO;
 
-
-typedef struct _FILETIME {
-    DWORD dwLowDateTime;
-    DWORD dwHighDateTime;
-} FILETIME, *PFILETIME, *LPFILETIME;
-
 typedef struct WINTRUST_CERT_INFO_
 {
     DWORD           cbStruct;
@@ -2705,24 +2699,6 @@ typedef struct _PUBLIC_OBJECT_BASIC_INFORMATION {
  } PUBLIC_OBJECT_BASIC_INFORMATION, *PPUBLIC_OBJECT_BASIC_INFORMATION;
 
 
- typedef struct _EVENTLOGRECORD {
-  DWORD Length;
-  DWORD Reserved;
-  DWORD RecordNumber;
-  DWORD TimeGenerated;
-  DWORD TimeWritten;
-  DWORD EventID;
-  WORD  EventType;
-  WORD  NumStrings;
-  WORD  EventCategory;
-  WORD  ReservedFlags;
-  DWORD ClosingRecordNumber;
-  DWORD StringOffset;
-  DWORD UserSidLength;
-  DWORD UserSidOffset;
-  DWORD DataLength;
-  DWORD DataOffset;
-} EVENTLOGRECORD, *PEVENTLOGRECORD;
 
 typedef struct tagSOLE_AUTHENTICATION_SERVICE {
     DWORD dwAuthnSvc;
@@ -3119,10 +3095,6 @@ typedef struct _CRYPT_ENCODE_PARA {
   PVOID  pfnFree;
 } CRYPT_ENCODE_PARA, *PCRYPT_ENCODE_PARA;
 
-typedef struct tagPOINT {
-  LONG x;
-  LONG y;
-} POINT, *PPOINT, *LPPOINT;
 
 
 typedef struct _ACL {

ctypes_generation/generate.py

@@ -16,7 +16,6 @@ pexists = os.path.exists
 dedent = textwrap.dedent
 
 
-
 TYPE_EQUIVALENCE = [
     # BYTE is defined in ctypes.wintypes as c_byte but who wants
     # BYTE to be signed ? (from MSDN: <typedef unsigned char BYTE;>)
@@ -74,6 +73,9 @@ TYPE_EQUIVALENCE = [
     ('IF_INDEX', 'NET_IFINDEX'),
     ('IFTYPE', 'ULONG'),
     ('PULONG64', 'POINTER(ULONG64)'),
+    ('LPFILETIME', 'POINTER(FILETIME)'),
+    ('LPPOINT', 'POINTER(POINT)'),
+    ('LPRECT', 'POINTER(RECT)'),
     ('PBYTE', 'POINTER(BYTE)'),
     ('PUINT', 'POINTER(UINT)'),
     ('PHANDLE', 'POINTER(HANDLE)'),
@@ -132,10 +134,10 @@ class CtypesGenerator(object):
     PARSER = None
     IMPORT_HEADER = "{deps}"
 
-    def __init__(self, infilename, outfilename, dependances=()):
-        self.infilename = infilename
+    def __init__(self, indirname, outfilename, dependances=()):
+        self.indirname = indirname
         self.outfilename = outfilename
-        self.infile = open(self.infilename)
+        # self.infile = open(self.infilename)
         self.data = None
         self.dependances = dependances
 
@@ -146,11 +148,22 @@ class CtypesGenerator(object):
         self.analyse(self.data)
         self.check_dependances()
 
+    # def parse(self):
+        # if self.data is None:
+            # print("Parsing <{0}>".format(self.infilename))
+            # self.data = self.PARSER(self.infile.read()).parse()
+        # return self.data
+
     def parse(self):
-        if self.data is None:
-            print("Parsing <{0}>".format(self.infilename))
-            self.data = self.PARSER(self.infile.read()).parse()
-        return self.data
+        if self.data is not None:
+            return self.data
+        data = []
+        for filename in glob.glob(self.indirname):
+            print("Parsing <{0}>".format(filename))
+            # data.append(self.PARSER(open(filename).read()).parse())
+            data += self.PARSER(open(filename).read()).parse()
+        self.data = data
+        return data
 
     def analyse(self, data):
         raise NotImplementedError("<{0}> doest not implement <analyse>".format(type(self).__name__))
@@ -160,7 +173,8 @@ class CtypesGenerator(object):
         for dep in self.dependances:
             missing -= dep.exports
         if missing:
-            raise ValueError("Missing dependance <{0}> in <{1}>".format(missing, self.infilename))
+            # raise ValueError("Missing dependance <{0}> in <{1}>".format(missing, self.infilename))
+            print ValueError("Missing dependance <{0}>".format(missing))
 
     def generate_import(self):
         deps = "\n".join(["from {0} import *".format(os.path.basename(dep.outfilename).rsplit(".")[0]) for dep in self.dependances])
@@ -356,6 +370,7 @@ class StructGenerator(CtypesGenerator):
                 extended_struct_filename = from_here(os.path.join("extended_structs", "{0}.py".format(definition.name)))
                 with open(extended_struct_filename) as f:
                     ctypes_lines.append(f.read())
+                    # import pdb;pdb.set_trace()
                     ctypes_lines.append(definition.generate_typedef_ctypes() + "\n")
 
         ctypes_code = "\n".join(ctypes_lines)
@@ -364,6 +379,19 @@ class StructGenerator(CtypesGenerator):
         print("<{0}> generated".format(self.outfilename))
         return ctypes_code
 
+    def parse(self):
+        if self.data is not None:
+            return self.data
+        data = [[], []]
+        for filename in glob.glob(self.indirname):
+            print("Parsing <{0}>".format(filename))
+            # data.append(self.PARSER(open(filename).read()).parse())
+            new_data = self.PARSER(open(filename).read()).parse()
+            data[0].extend(new_data[0])
+            data[1].extend(new_data[1])
+        self.data = data
+        return data
+
     def append_input_file(self, filename):
         print("Adding file <{0}>".format(filename))
         self.parse()
@@ -478,6 +506,20 @@ class NtStatusGenerator(CtypesGenerator):
             return Flag(name, code)
     """)
 
+    def __init__(self, infilename, outfilename, dependances=()):
+        self.infilename = infilename
+        self.outfilename = outfilename
+        self.infile = open(self.infilename)
+        self.data = None
+        self.dependances = dependances
+
+        self.exports = set([])
+        self.imports = set([])
+
+        self.parse()
+        self.analyse(self.data)
+        self.check_dependances()
+
     def parse_ntstatus(self, content):
         nt_status_defs = []
         for line in content.split("\n"):
@@ -498,6 +540,7 @@ class NtStatusGenerator(CtypesGenerator):
             self.parse_ntstatus(self.infile.read())
         return self.data
 
+
     def analyse(self, data):
         self.add_imports("Flag")
 
@@ -668,7 +711,6 @@ class InitialCOMGenerator(CtypesGenerator):
                         byreflevel = arg.byreflevel - 1
                         method.args[pos] = arg = type(arg)(atype, byreflevel, arg.name)
                         self.real_type[arg] = initial_arg
-
                     self.add_imports(arg.type)
 
     com_interface_comment_template = """ #{0} -> {1}"""
@@ -728,6 +770,17 @@ class InitialCOMGenerator(CtypesGenerator):
         return ", ".join(str_iid)
 
 
+
+        com_interface_template = dedent("""
+    class {0}(COMInterface):
+        IID = generate_IID({2}, name="{0}", strid="{3}")
+
+        _functions_ = {{
+    {1}
+        }}
+    """)
+
+
 class COMGenerator(InitialCOMGenerator):
     IMPORT_HEADER = "{deps}"
     HEADER = ""
@@ -777,30 +830,43 @@ DEFAULT_INTERFACE_TO_IID = from_here("definitions\\interface_to_iid.txt")
 
 # A partial define without the dependance to ntstatus defintion
 # BOOTSTRAP!!
-non_generated_def = InitialDefGenerator(from_here("definitions\\windef.txt"), from_here(r"..\windows\generated_def\\windef.py"))
+non_generated_def = InitialDefGenerator(from_here("definitions\\defines\\windef.txt"), from_here(r"..\windows\generated_def\\windef.py"))
 ntstatus = NtStatusGenerator(from_here("definitions\\ntstatus.txt"), from_here(r"..\windows\generated_def\\ntstatus.py"), dependances=[non_generated_def])
 # Not a real circular def (import not at the begin of file
-defs_with_ntstatus = InitialDefGenerator(from_here("definitions\\windef.txt"), from_here(r"..\windows\generated_def\\windef.py"), dependances=[ntstatus])
+defs_with_ntstatus = InitialDefGenerator(from_here("definitions\\defines\\*.txt"), from_here(r"..\windows\generated_def\\windef.py"), dependances=[ntstatus])
 
 # YOLO HACK FOR NOW :DD
-defs_with_ntstatus.append_input_file(from_here("definitions\\wintrust_crypt_def.txt"))
-defs_with_ntstatus.append_input_file(from_here("definitions\\windef_error.txt"))
-defs_with_ntstatus.append_input_file(from_here("definitions\\custom_rpc_windef.txt"))
-defs_with_ntstatus.append_input_file(from_here("definitions\\windef_evtlog.txt"))
+# import pdb;pdb.set_trace()
 
 
-structs = StructGenerator(from_here("definitions\\winstruct.txt"), from_here(r"..\windows\generated_def\\winstructs.py"), dependances=[defs_with_ntstatus])
-structs.append_input_file(from_here("definitions\\winstruct_apisetmap.txt"))
-structs.append_input_file(from_here("definitions\\display_struct.txt"))
-structs.append_input_file(from_here("definitions\\winstruct_bits.txt"))
-structs.append_input_file(from_here("definitions\\winstruct_alpc.txt"))
-structs.append_input_file(from_here("definitions\\winstruct_evtlog.txt"))
-structs.append_input_file(from_here("definitions\\winstruct_file_info.txt"))
+# for filename in [f for f in glob.glob(from_here("definitions\\defines\\*.txt")) if not f.endswith("\\windef.txt")]:
+    # defs_with_ntstatus.append_input_file(from_here("definitions\\wintrust_crypt_def.txt"))
+    # defs_with_ntstatus.append_input_file(from_here("definitions\\windef_error.txt"))
+    # defs_with_ntstatus.append_input_file(from_here("definitions\\custom_rpc_windef.txt"))
+    # defs_with_ntstatus.append_input_file(from_here("definitions\\windef_evtlog.txt"))
+    # defs_with_ntstatus.append_input_file(filename)
 
-functions = FuncGenerator(from_here("definitions\\winfunc.txt"), from_here(r"..\windows\generated_def\\winfuncs.py"), dependances=[structs])
-functions.append_input_file(from_here("definitions\\winfunc_crypto_wintrust.txt"))
-functions.append_input_file(from_here("definitions\\winfunc_notdoc.txt"))
-functions.append_input_file(from_here("definitions\\winfunc_evtlog.txt"))
+
+structs = StructGenerator(from_here("definitions\\structures\\*.txt"), from_here(r"..\windows\generated_def\\winstructs.py"), dependances=[defs_with_ntstatus])
+
+# for filename in [f for f in glob.glob(from_here("definitions\\structures\\*.txt")) if not f.endswith("\\winstruct.txt")]:
+    # structs.append_input_file(filename)
+
+# structs.append_input_file(from_here("definitions\\winstruct_apisetmap.txt"))
+# structs.append_input_file(from_here("definitions\\display_struct.txt"))
+# structs.append_input_file(from_here("definitions\\winstruct_bits.txt"))
+# structs.append_input_file(from_here("definitions\\winstruct_alpc.txt"))
+# structs.append_input_file(from_here("definitions\\winstruct_evtlog.txt"))
+# structs.append_input_file(from_here("definitions\\winstruct_file_info.txt"))
+
+functions = FuncGenerator(from_here("definitions\\functions\\*.txt"), from_here(r"..\windows\generated_def\\winfuncs.py"), dependances=[structs])
+
+# for filename in [f for f in glob.glob(from_here("definitions\\functions\\*.txt")) if not f.endswith("\\winfunc.txt")]:
+    # functions.append_input_file(filename)
+
+# functions.append_input_file(from_here("definitions\\winfunc_crypto_wintrust.txt"))
+# functions.append_input_file(from_here("definitions\\winfunc_notdoc.txt"))
+# functions.append_input_file(from_here("definitions\\winfunc_evtlog.txt"))
 
 com = InitialCOMGenerator(from_here("definitions\\com\\*.txt"), DEFAULT_INTERFACE_TO_IID, from_here(r"..\windows\generated_def\\interfaces.py"), dependances=[structs, defs_with_ntstatus])
 
@@ -848,4 +914,5 @@ if __name__ == "__main__":
     print("Generating documentation")
     ntstatus.generate_doc(from_here(r"..\docs\source\ntstatus_generated.rst"))
     defs_with_ntstatus.generate_doc(from_here(r"..\docs\source\windef_generated.rst"))
-    structs.generate_doc(from_here(r"..\docs\source\winstructs_generated.rst"))
+    structs.generate_doc(from_here(r"..\docs\source\winstructs_generated.rst"))
+

ctypes_generation/newgen.py

@@ -0,0 +1,418 @@
+import sys
+import os
+import os.path
+import re
+import glob
+import textwrap
+import StringIO
+
+import shutil
+
+import dummy_wintypes
+import struct_parser
+import func_parser
+import def_parser
+import com_parser
+
+pjoin = os.path.join
+pexists = os.path.exists
+dedent = textwrap.dedent
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+print(SCRIPT_DIR)
+from_here = lambda path: pjoin(SCRIPT_DIR, path)
+
+
+
+class ParsedFile(object):
+    def __init__(self, filename):
+        self.filename = filename
+        self.data = self.PARSER(open(filename).read()).parse()
+        self.exports = set()
+        self.imports = set()
+        self.compute_imports_exports(self.data)
+
+    def add_exports(self, *names):
+        self.exports.update(names)
+
+    def add_imports(self, *names):
+        self.imports.update(names)
+
+    def compute_imports_exports(self):
+        raise NotImplementedError("compute_imports_exports")
+
+    def __repr__(self):
+        return '<{clsname} "{0}">'.format(self.filename, clsname=type(self).__name__)
+
+class StructureParsedFile(ParsedFile):
+    PARSER = struct_parser.WinStructParser
+
+    def compute_imports_exports(self, data):
+        structs, enums = data
+        for enum in enums:
+            self.add_exports(enum.name)
+            self.add_exports(*enum.typedef)
+        for struct in structs:
+            self.add_exports(struct.name)
+            self.add_exports(*struct.typedef)
+            for field_type, field_name, nb_rep in struct.fields:
+                if field_type.name not in self.exports:
+                    self.add_imports(field_type.name)
+                try:
+                    int(nb_rep)
+                except:
+                    self.add_imports(nb_rep)
+
+class SimpleTypeParsedFile(ParsedFile):
+    PARSER = struct_parser.SimpleTypesParser
+
+    def compute_imports_exports(self, data):
+        for simple_type in data:
+            self.add_exports(simple_type.lvalue) # No dependancy check on rvalue for now
+
+
+class DefinitionParsedFile(ParsedFile):
+    PARSER = def_parser.WinDefParser
+
+    def compute_imports_exports(self, data):
+        for windef in data:
+            self.add_exports(windef.name) # No dependancy check on rvalue for now
+
+class NtStatusParsedFile(ParsedFile):
+    PARSER = def_parser.NtStatusParser
+
+    def compute_imports_exports(self, data):
+        for ntstatus in data:
+            self.add_exports(ntstatus[1])
+
+class FunctionParsedFile(ParsedFile):
+    PARSER = func_parser.WinFuncParser
+
+    def compute_imports_exports(self, data):
+        for func in data:
+            if isinstance(func.return_type, tuple) and func.return_type[0] == "PTR":
+                self.add_imports(func.return_type[1])
+            else:
+                self.add_imports(func.return_type)
+            for param_type, _ in func.params:
+                if param_type.startswith("POINTER(") and param_type.endswith(")"):
+                    param_type = param_type[len("POINTER("): -1]
+                self.add_imports(param_type)
+            self.add_exports(func.name)
+
+
+class COMParsedFile(ParsedFile):
+    PARSER = com_parser.WinComParser
+
+    def compute_imports_exports(self, cominterface):
+        self.add_exports(cominterface.name)
+        if cominterface.typedefptr:
+            self.add_exports(cominterface.typedefptr)
+
+
+class ParsedFileGraph(object):
+    def __init__(self, nodes, depnodes): # depnodes: nodes that we dont have to handle but want can take export from
+        self.nodes = nodes
+        self.depnodes = depnodes
+        self.exports_database = {}
+        self.depandances_database = {node: set() for node in nodes}
+        self.build_export_database(self.nodes)
+        self.build_depandance_database()
+
+    def build_dependancy_graph(self):
+        todo = set(self.nodes)
+        start = self.find_starting_node()
+        print("Starting node is {0}".format(start))
+        todo.remove(start)
+        flatten = [start]
+        depdone = set(flatten) | set(self.depnodes)
+        while todo:
+            for node in todo:
+                if self.depandances_database[node].issubset(depdone):
+                    break
+            else:
+                raise ValueError("POUET")
+
+            flatten.append(node)
+            depdone.add(node)
+            todo.remove(node)
+            print("Next is <{0}>".format(node))
+        return flatten
+
+
+    def build_depandance_database(self):
+        for node in self.nodes:
+            for import_ in node.imports:
+                try:
+                    self.depandances_database[node].add(self.exports_database[import_])
+                except KeyError as e:
+                    raise ValueError("Missing dependancy <{0}> of {1}".format(import_, node))
+
+
+    def build_export_database(self, nodes):
+        for node in self.nodes + self.depnodes:
+            for export in node.exports:
+                if export in self.exports_database:
+                    raise ValueError("{0} IN {1} but  already exported by {2}".format(export, self.exports_database[export], node))
+                self.exports_database[export] = node
+
+    def find_starting_node(self):
+        for node in self.nodes:
+            if self.depandances_database[node].issubset(set(self.depnodes)):
+                return node
+        raise ValueError("Could not find a starting NODE without dependancy")
+
+
+class BasicTypeNodes(object):
+    @property
+    def exports(self):
+        # Let allow ourself to redefine the bugged BYTE define & MAX_PATH which is NOT A TYPE !
+        return set(dummy_wintypes.names) - set(["BYTE", "MAX_PATH"])
+
+class FakeExporter(object):
+    def __init__(self, exports):
+        self.exports = exports
+
+class ParsedDirectory(object):
+    def __init__(self, filetype, directory):
+        self.nodes = [filetype(f) for f in glob.glob(directory)]
+
+
+### Generation Class ###
+
+class CtypesGenerator(object):
+    def __init__(self, parsed_files, template):
+        self.files = parsed_files # Already in generation order
+        self.template = template # MAKE BETTER
+        self.result = StringIO.StringIO()
+
+    def emit(self, str):
+        self.result.write(str)
+
+    def emitline(self, str):
+        self.emit(str)
+        self.emit("\n")
+
+    def before_emit_template(self):
+        pass
+
+    def after_emit_template(self):
+        pass
+
+    def copy_template(self):
+        with open(self.template) as f:
+            self.emit(f.read())
+
+    def generate(self):
+        self.before_emit_template()
+        self.copy_template()
+        self.after_emit_template()
+
+        for file in self.files:
+            self.generate_for_file(file)
+
+    def generate_for_file(self, file):
+        pass
+
+NTSTATUS_MODULE = "ntstatus"
+
+class DefineCtypesGenerator(CtypesGenerator):
+    def after_emit_template(self):
+        self.emitline("from {0} import *".format(NTSTATUS_MODULE))
+
+    def generate_for_file(self, file):
+        for define in file.data:
+            self.emitline(define.generate_ctypes())
+
+class NtStatusCtypesGenerator(CtypesGenerator):
+    def generate_for_file(self, file):
+        for value, name, descr in file.data:
+            value = "{:#x}".format(value)
+            line = '{1} = NtStatusException.register_ntstatus({0}, "{1}", "{2}")'.format(value, name, descr)
+            self.emitline(line)
+
+class COMCtypesGenerator(CtypesGenerator):
+    IGNORED_INTERFACE = set(["ITypeInfo"])
+    def __init__(self, *args, **kwargs):
+        super(COMCtypesGenerator, self).__init__(*args, **kwargs)
+        self.iids_def = {}
+        self.generated_interfaces_names = set(self.IGNORED_INTERFACE)
+        for file in self.files:
+            self.generated_interfaces_names.update(file.exports)
+
+
+    def parse_iid_file(self, filename):
+        data = open(filename).read()
+        for line in data.split("\n"):
+            name, iid = line.split("|")
+            self.iids_def[name] = self.parse_iid(iid), iid
+
+    def before_emit_template(self):
+        self.emitline("from POUET import *")
+        self.emitline("")
+
+    def generate_for_file(self, file):
+        define = []
+        cominterface = file.data
+        return self.generate_com_interface(cominterface)
+
+    def generate_com_interface(self, cominterface):
+        name = cominterface.name
+        if cominterface.iid is not None:
+            iid_str = cominterface.iid
+            iid_python = self.parse_iid(iid_str)
+        else:
+            print("Lookup of IID for <{0}>".format(cominterface.name))
+            iid_python, iid_str = self.iids_def[cominterface.name]
+
+        cls_format_param = {"name": name, "iid_python" : iid_python, "iid_str": iid_str}
+
+        self.emitline("class {name} (COMInterface):".format(**cls_format_param))
+        self.emitline('    IID = generate_IID({iid_python}, name="{name}", strid="{iid_str}")'.format(**cls_format_param))
+        self.emitline('    _functions_ = {')
+        self.emit_com_interface_functions(cominterface)
+        self.emitline('    }')
+        self.emitline('')
+
+
+    def emit_com_interface_functions(self, cominterface):
+        indent = " " * 8
+        for method_nb, method in enumerate(cominterface.methods):
+            args_to_define = method.args[1:] # ctypes doesnt not need the This
+            name = method.name
+            params = ", ".join([arg.name +":"+ ("*"* arg.byreflevel) +arg.type for arg in args_to_define])
+            self.emitline(indent + "# {name} -> {params}".format(name=name, params=params))
+
+            str_args = []
+            for arg in args_to_define:
+                if arg.type == "void" and arg.byreflevel > 0:
+                    arg = type(arg)("PVOID", arg.byreflevel - 1, arg.name)
+                atype = arg.type
+                byreflevel = arg.byreflevel
+                if atype in self.generated_interfaces_names:
+                    byreflevel = arg.byreflevel - 1
+                    atype = "PVOID"
+
+                for i in range(byreflevel):
+                    atype = "POINTER({0})".format(atype)
+                str_args.append(atype)
+
+            # methods_string.append(self.com_interface_method_template.format(method.name, ", ".join([method.ret_type] + str_args), method_nb))
+            params = ", ".join([method.ret_type] + str_args)
+            self.emitline(indent + '"{0}": ctypes.WINFUNCTYPE({1})({2}, "{0}"),'.format(name, params, method_nb))
+        return
+
+
+    def parse_iid(self, iid_str):
+        part_iid = iid_str.split("-")
+        str_iid = []
+        str_iid.append("0x" + part_iid[0])
+        str_iid.append("0x" + part_iid[1])
+        str_iid.append("0x" + part_iid[2])
+        str_iid.append("0x" + part_iid[3][:2])
+        str_iid.append("0x" + part_iid[3][2:])
+        for i in range(6): str_iid.append("0x" + part_iid[4][i * 2:(i + 1) * 2])
+        return ", ".join(str_iid)
+
+class FunctionCtypesGenerator(CtypesGenerator):
+    def __init__(self, parsed_files):
+        self.files = parsed_files # Already in generation order
+        self.result = StringIO.StringIO()
+
+    def copy_template(self):
+        self.emitline("from ctypes import *")
+        self.emitline("from POUET import *")
+        # self.emitline("PPORT_MESSAGE = INT")
+
+    def generate_for_file(self, file):
+        for item in file.data:
+            self.emitline(item.generate_ctypes())
+
+
+
+EXTENDED_STRUCT_FILE = glob.glob(pjoin(SCRIPT_DIR, "extended_structs", "*.py"))
+EXTENDED_STRUCT = [os.path.basename(filename)[:-len(".py")] for filename in EXTENDED_STRUCT_FILE]
+
+class StructureCtypesGenerator(CtypesGenerator):
+    def generate_for_simple_type_file(self, file):
+        for simple_type in file.data:
+            self.emitline(simple_type.generate_ctypes())
+
+    def generate_for_file(self, file):
+        if isinstance(file, SimpleTypeParsedFile):
+            return self.generate_for_simple_type_file(file)
+        structs, enums = file.data
+        for definition in [d for l in (enums, structs) for d in l]:
+            self.emitline(definition.generate_ctypes())
+            if definition.name in EXTENDED_STRUCT:
+                print("Including extended definition for <{0}>".format(definition.name))
+                extended_struct_filename = from_here(os.path.join("extended_structs", "{0}.py".format(definition.name)))
+                with open(extended_struct_filename) as f:
+                    self.emitline(f.read())
+                    # RE-generate the typedef to apply them to the extended definition
+                    self.emitline(definition.generate_typedef_ctypes())
+
+
+
+stfilename = r"C:\Users\hakril\Documents\projets\PythonForWindows\ctypes_generation\definitions\simple_types.txt"
+struct_parser.SimpleTypesParser(open(stfilename).read()).parse()
+
+ss = SimpleTypeParsedFile(stfilename)
+
+ds = ParsedDirectory(DefinitionParsedFile, from_here(r"definitions\defines\*.txt"))
+fds = ParsedDirectory(FunctionParsedFile, from_here(r"definitions\functions\*.txt"))
+
+x = from_here("definitions\\structures\\*.txt")
+
+ntp = NtStatusParsedFile(from_here(r"definitions\ntstatus.txt"))
+
+sds = ParsedDirectory(StructureParsedFile, from_here(r"definitions\structures\*.txt"))
+
+scom = ParsedDirectory(COMParsedFile, from_here(r"definitions\COM\*.txt"))
+
+g = ParsedFileGraph(sds.nodes + [ss], depnodes=[BasicTypeNodes()] + ds.nodes)
+snodes = g.build_dependancy_graph()
+
+
+gg = ParsedFileGraph(fds.nodes, depnodes=[BasicTypeNodes()] + snodes)
+fnodes = gg.build_dependancy_graph()
+
+## EMIT TEST CODE
+
+
+
+edef = DefineCtypesGenerator(ds.nodes, from_here(r"definitions\defines\template.py"))
+edef.generate()
+
+
+ents = NtStatusCtypesGenerator([ntp], from_here(r"definitions\ntstatus_template.py"))
+ents.generate()
+
+snts = StructureCtypesGenerator(snodes, from_here(r"definitions\structures\template.py"))
+snts.generate()
+
+fnts = FunctionCtypesGenerator(fnodes)
+fnts.generate()
+
+cnts = COMCtypesGenerator(scom.nodes, from_here(r"definitions\com\template.py"))
+cnts.parse_iid_file(from_here("definitions\\interface_to_iid.txt"))
+cnts.generate()
+
+shutil.copy(from_here(r"definitions\flag.py"), "tmp\\")
+
+
+
+with open(r"tmp\yolo.py", "w") as f:
+    f.write(edef.result.getvalue())
+
+with open(r"tmp\{0}.py".format(NTSTATUS_MODULE), "w") as f:
+    f.write(ents.result.getvalue())
+
+with open(r"tmp\POUET.py", "w") as f:
+    f.write(snts.result.getvalue())
+
+with open(r"tmp\FUNCS.py", "w") as f:
+    f.write(fnts.result.getvalue())
+
+with open(r"tmp\COM.py", "w") as f:
+    f.write(cnts.result.getvalue())

ctypes_generation/simpleparser.py

@@ -65,7 +65,9 @@ class SharpToken(NoValueToken):
 
 class EqualToken(NoValueToken):
     value = "="
-    pass
+
+class NewLineToken(NoValueToken):
+    value = "\n"
 
 class Lexer(object):
     keywords = ["typedef", "struct", "enum", "union", "const"]
@@ -74,8 +76,9 @@ class Lexer(object):
                     "{" : OpenBracketToken, "}" : CloseBracketToken, ";" : ColonToken,
                     "," : CommaToken, "(" : OpenParenthesisToken, ")" :  CloseParenthesisToken, "#" : SharpToken, "=" : EqualToken}
 
-    def __init__(self, code):
+    def __init__(self, code, newlinetoken=False):
         self.code = code
+        self.newlinetoken = newlinetoken
 
     def split_line(self, line):
         return line.strip().split()
@@ -106,6 +109,8 @@ class Lexer(object):
                     continue
                 for tok in self.split_word(word):
                     yield tok
+            if self.newlinetoken:
+                yield NewLineToken()
 
 
 
@@ -113,7 +118,6 @@ class ParsingError(Exception):
     pass
 
 class Parser(object):
-
     def __init__(self, data):
         self.lexer = iter(Lexer(self.initial_processing(data)))
         self.peek_token = None

ctypes_generation/struct_parser.py

@@ -143,6 +143,32 @@ class WinStructParser(Parser):
                 raise ValueError("Unknow returned type {0}".format(x))
         return strucs, enums
 
+class SimpleTypeDefine(object):
+    def __init__(self, lvalue,  rvalue):
+        self.lvalue = lvalue
+        self.rvalue = rvalue
+
+    def generate_ctypes(self):
+        return "{self.lvalue} = {self.rvalue}".format(self=self)
+
+class SimpleTypesParser(Parser):
+    def __init__(self, data):
+        self.lexer = iter(Lexer(self.initial_processing(data), newlinetoken=True))
+        self.peek_token = None
+
+    def parse(self):
+        results = []
+        while self.peek() is not None:
+            lvalue = self.assert_token_type(NameToken).value
+            self.assert_token_type(EqualToken)
+            rvalue = ""
+            while type(self.peek()) is not NewLineToken:
+                rvalue += self.next_token().value
+            results.append(SimpleTypeDefine(lvalue, rvalue))
+            while type(self.peek()) is NewLineToken: # discard the NewLineToken(s)
+                self.next_token()
+        return results
+
 def dbg_lexer(data):
     for i in Lexer(data).token_generation():
         print i