ctypes generation allow COM mission interface autocopy from a directory + support anon structs/enum in typedef

ctypes_generation/com_parser.py

@@ -9,7 +9,7 @@ from simpleparser import *
 class WinComParser(Parser):
     PARAM_INFO =  ["__RPC__deref_out", "__RPC__in", "__RPC__deref_out_opt", "__RPC__out", "__RPC__in_opt",
         "__RPC__deref_opt_inout_opt", "__in", "__out", "__out_opt", "__in_opt", "__inout",
-        "__reserved", "__RPC__in_opt_string", "__RPC__inout_opt", "__RPC__in_string", "__deref_out_opt"]
+        "__reserved", "__RPC__in_opt_string", "__RPC__inout_opt", "__RPC__in_string", "__deref_out_opt", "__RPC__inout"]
     PARAM_INFO_WITH_VALUE = ["__RPC__in_ecount", "__RPC__out_ecount_part", "__RPC__in_ecount_full",
             "__RPC__in_range", "__RPC__out_ecount_full", "__out_ecount_opt", "__out_ecount", "__in_ecount_opt",
             "__in_ecount", "__out_bcount_opt", "__out_bcount", "__in_bcount", "__in_bcount_opt", "__RPC__out_ecount_full_string"]

ctypes_generation/generate.py

@@ -14,6 +14,8 @@ import func_parser
 import def_parser
 import com_parser
 
+from simpleparser import ParsingError
+
 pjoin = os.path.join
 pexists = os.path.exists
 dedent = textwrap.dedent
@@ -28,7 +30,13 @@ to_dest = lambda path: pjoin(DEST_DIR, path)
 class ParsedFile(object):
     def __init__(self, filename):
         self.filename = filename
-        self.data = self.PARSER(open(filename).read()).parse()
+        try:
+            self.data = self.PARSER(open(filename).read()).parse()
+        except ParsingError as e:
+            print(" !! Error while parsing file <{0}> !!".format(filename))
+            print(e)
+            raise
+
         self.exports = set()
         self.imports = set()
         self.compute_imports_exports(self.data)
@@ -131,12 +139,13 @@ class COMParsedFile(ParsedFile):
 
 
 class ParsedFileGraph(object):
-    def __init__(self, nodes, depnodes): # depnodes: nodes that we dont have to handle but want can take export from
+    def __init__(self, nodes, depnodes, missing_handler=None): # depnodes: nodes that we dont have to handle but want can take export from
         self.nodes = nodes
         self.depnodes = depnodes
         self.exports_database = {}
         self.depandances_database = {node: set() for node in nodes}
         self.build_export_database(self.nodes)
+        self.missing_handler = missing_handler
         self.build_depandance_database()
 
     def build_dependancy_graph(self):
@@ -166,7 +175,13 @@ class ParsedFileGraph(object):
                 try:
                     self.depandances_database[node].add(self.exports_database[import_])
                 except KeyError as e:
-                    raise ValueError("Missing dependancy <{0}> of {1}".format(import_, node))
+                    self.on_missing_dependancy(import_, node)
+                    # raise ValueError("Missing dependancy <{0}> of {1}".format(import_, node))
+
+    def on_missing_dependancy(self, import_, node):
+        if self.missing_handler is not None:
+            return self.missing_handler(import_, node)
+        raise ValueError("Missing dependancy <{0}> of {1}".format(import_, node))
 
 
     def build_export_database(self, nodes):
@@ -194,8 +209,23 @@ class FakeExporter(object):
         self.exports = exports
 
 class ParsedDirectory(object):
-    def __init__(self, filetype, directory):
-        self.nodes = [filetype(f) for f in glob.glob(directory)]
+    def __init__(self, filetype, src, recurse=False):
+        if not recurse:
+            if os.path.isdir(src):
+                srcglob = pjoin(src, "*.txt")
+            else:
+                srcglob = src
+            files = glob.glob(srcglob)
+        else:
+            # Recurse search of .txt files
+            files = [os.path.join(path, filename)
+                        for (path, _, files) in os.walk(src)
+                            for filename in files
+                                if filename.endswith(".txt")]
+
+
+
+        self.nodes = [filetype(f) for f in files]
 
 
 ### Generation Class ###
@@ -528,7 +558,6 @@ class MetaFileGenerator(NoTemplatedGenerator):
                 self.emitline("import {0} as {0}_module".format(modname))
                 self.emitline("{0}_walker = generate_walker({0}, {1}_module)".format(name, modname))
 
-
 class ModuleGenerator(object):
     def __init__(self, name, filetype, ctypesgenerator, docgenerator, src):
         self.name = name
@@ -543,27 +572,44 @@ class ModuleGenerator(object):
     def add_module_dependancy(self, module):
         self.dependances_modules.add(module)
 
-    def parse_directory(self, globdir):
-        self.nodes += ParsedDirectory(self.filetype, globdir).nodes
 
     def get_template_filename(self):
         return pjoin(self.src, "template.py")
 
-    def parse_source_directory(self):
-        if os.path.isdir(self.src):
-            srcglob = pjoin(self.src, "*.txt")
-        else:
-            srcglob = self.src
-        self.parse_directory(srcglob)
+    def parse_source_directory(self, recurse=False):
+        self.nodes += ParsedDirectory(self.filetype, self.src, recurse=recurse).nodes
 
     def resolve_dependancies(self, depnodes=[]):
         g = ParsedFileGraph(self.nodes, depnodes=depnodes)
         return g.build_dependancy_graph()
 
     def check_dependancies_without_flattening(self, depnodes):
-        g = ParsedFileGraph(self.nodes, depnodes=depnodes) # init check for missing dependance
+        self.missing_interfaces = []
+        g = ParsedFileGraph(self.nodes, depnodes=depnodes, missing_handler=self.missing_com_interface) # init check for missing dependance
+        if self.missing_interfaces:
+            missing_names = [x[0] for x in self.missing_interfaces]
+            if not args.autocopy:
+                raise ValueError("Missing COM dependancy Names : {0}".format(missing_names))
+
+            print("Missing COM interfaces are: {0}".format(missing_names))
+            autocopied = []
+            for name, node in self.missing_interfaces:
+                filename = "{0}\\{1}.txt".format(args.autocopy, name)
+                if os.path.exists(filename):
+                    autocopied.append(name)
+                    print("Auto-copy <{0}>".format(filename))
+                    targetdir = os.path.dirname(node.filename)
+                    print(filename, targetdir)
+                    shutil.copy(filename, targetdir)
+            if autocopied:
+                raise ValueError("Auto-copyied Names : {0}".format(autocopied))
+            raise ValueError("Missing COM dependancy Names : {0}".format(missing_names))
         return g.nodes
 
+    def missing_com_interface(self, import_, node):
+        print("Missing name <{0}> in file <{1}>".format(import_, node.filename))
+        self.missing_interfaces.append((import_, node))
+
     def generate(self):
         self.parse_source_directory()
         # Flatten the graph
@@ -607,6 +653,14 @@ class ModuleGenerator(object):
         return res
 
 
+import argparse
+
+parser = argparse.ArgumentParser(prog=__file__)
+parser.add_argument('--autocopy', help="[PRIVATE OPTION] A directory used to find missing COM interface")
+args = parser.parse_args()
+
+
+
 # Copy Flag code
 shutil.copy(from_here(r"definitions\flag.py"), DEST_DIR)
 
@@ -643,7 +697,7 @@ print("== Generating COM interfaces ==")
 com_module_generator = ModuleGenerator("interfaces", COMParsedFile, COMCtypesGenerator, None, from_here(r"definitions\com"))
 # Load the interface_to_iid file needed by the 'COMCtypesGenerator'
 com_module_generator.after_ctypes_generator_init = lambda cgen: cgen.parse_iid_file(from_here("definitions\\interface_to_iid.txt"))
-com_module_generator.parse_source_directory()
+com_module_generator.parse_source_directory(recurse=True)
 com_module_generator.add_module_dependancy(structure_module_generator)
 com_module_generator.resolve_dependancies = com_module_generator.check_dependancies_without_flattening # No real flattening as we have circular dep in Interfaces VTBL
 com_module_generator.resolve_dep_and_generate([BasicTypeNodes()])

ctypes_generation/struct_parser.py

@@ -52,10 +52,17 @@ class WinStructParser(Parser):
             sep = self.next_token()
         self.assert_token_type(ColonToken, sep)
 
-    def parse_enum(self):
+    def parse_enum(self, is_typedef):
         """Handle enum typedef with no value assignement and 1 typedef after"""
-        enum_name = self.assert_token_type(NameToken).value
-        res_enum = WinEnum(enum_name)
+        if not type(self.peek()) == OpenBracketToken:
+            # Not an ANON enum
+            enum_name = self.assert_token_type(NameToken).value
+            res_enum = WinEnum(enum_name)
+        else:
+            if not is_typedef:
+                raise ValueError("Anonymous union not in a typedef")
+            res_enum = WinEnum(None)
+
         self.assert_token_type(OpenBracketToken)
         count = itertools.count()
         assigned_value = False
@@ -84,7 +91,6 @@ class WinStructParser(Parser):
         return res_enum
 
 
-
     def parse_winstruct(self):
         is_typedef = False
         peeked = self.peek()
@@ -94,17 +100,24 @@ class WinStructParser(Parser):
 
         def_type = self.assert_token_type(KeywordToken)
         if def_type.value == "enum":
-            return self.parse_enum()
+            return self.parse_enum(is_typedef)
         if def_type.value == "struct":
             WinDefType = WinStruct
         elif def_type.value == "union":
             WinDefType = WinUnion
         else:
             raise ParsingError("Expecting union or struct got <{0}> instead".format(def_type.value))
-        struct_name = self.assert_token_type(NameToken)
+        if not type(self.peek()) == OpenBracketToken:
+            # Not an anonymous structure def
+            struct_name = self.assert_token_type(NameToken).value
+        else:
+            # Anonymous structure def: check if we are ina  typedef
+            if not is_typedef:
+                raise ValueError("Anonymous structure/union not in a typedef")
+            struct_name = None #
         self.assert_token_type(OpenBracketToken)
 
-        result = WinDefType(struct_name.value, self.pack)
+        result = WinDefType(struct_name, self.pack)
 
         while type(self.peek()) != CloseBracketToken:
             tok_type, tok_name, nb_rep = self.parse_def()

ctypes_generation/winstruct.py

@@ -39,11 +39,15 @@ class WinStruct(object):
         self.fields.append(field)
 
     def add_typedef(self, name):
+        if self.name is None:
+            self.name = name
         if name in self.typedef:
             raise ValueError("nop")
         self.typedef[name] = self
 
     def add_ptr_typedef(self, name):
+        if self.name is None:
+            raise ValueError("Anonymous struct first typedef ({0}) should not be a PTR type".format(name))
         if name in self.typedef:
             raise ValueError("nop")
         self.typedef[name] = Ptr(self)
@@ -103,6 +107,8 @@ class WinStruct(object):
         typedef_ctypes = []
         for typedef_name, value in self.typedef.items():
             str_value = self.name
+            if typedef_name == str_value: # Do not generate "X= X" line (anonymous structs gen this)
+                continue
             if type(value) == Ptr:
                 str_value = "POINTER({0})".format(self.name)
             typedef_ctypes += ["{0} = {1}".format(typedef_name, str_value)]
@@ -130,11 +136,16 @@ class WinEnum(object):
         self.fields.append((number, name))
 
     def add_typedef(self, name):
+        if self.name is None:
+            # Setup our name to our first typedef
+            self.name = name
         if name in self.typedef:
             raise ValueError("nop")
         self.typedef[name] = self
 
     def add_ptr_typedef(self, name):
+        if self.name is None:
+            raise ValueError("Anonymous enum first typedef ({0}) should not be a PTR type".format(name))
         if name in self.typedef:
             raise ValueError("nop")
         self.typedef[name] = Ptr(self)
@@ -152,6 +163,8 @@ class WinEnum(object):
 
         for typedef_name, value in self.typedef.items():
             str_value = self.name
+            if typedef_name == str_value: # Do not generate "X= X" line (anonymous enum gen this)
+                continue
             if type(value) == Ptr:
                 str_value = "POINTER({0})".format(self.name)
             lines += ["{0} = {1}".format(typedef_name, str_value)]