From 8cc361b1f9c272c910c061307c3d994aeaed83ac Mon Sep 17 00:00:00 2001 From: hakril Date: Mon, 16 Apr 2018 19:54:09 +0200 Subject: [PATCH] ctypes generation allow COM mission interface autocopy from a directory + support anon structs/enum in typedef --- ctypes_generation/com_parser.py | 2 +- ctypes_generation/generate.py | 86 ++++++++++++++++++++++++------ ctypes_generation/struct_parser.py | 27 +++++++--- ctypes_generation/winstruct.py | 13 +++++ 4 files changed, 104 insertions(+), 24 deletions(-) diff --git a/ctypes_generation/com_parser.py b/ctypes_generation/com_parser.py index 50be992..e28da2d 100644 --- a/ctypes_generation/com_parser.py +++ b/ctypes_generation/com_parser.py @@ -9,7 +9,7 @@ from simpleparser import * class WinComParser(Parser): PARAM_INFO = ["__RPC__deref_out", "__RPC__in", "__RPC__deref_out_opt", "__RPC__out", "__RPC__in_opt", "__RPC__deref_opt_inout_opt", "__in", "__out", "__out_opt", "__in_opt", "__inout", - "__reserved", "__RPC__in_opt_string", "__RPC__inout_opt", "__RPC__in_string", "__deref_out_opt"] + "__reserved", "__RPC__in_opt_string", "__RPC__inout_opt", "__RPC__in_string", "__deref_out_opt", "__RPC__inout"] PARAM_INFO_WITH_VALUE = ["__RPC__in_ecount", "__RPC__out_ecount_part", "__RPC__in_ecount_full", "__RPC__in_range", "__RPC__out_ecount_full", "__out_ecount_opt", "__out_ecount", "__in_ecount_opt", "__in_ecount", "__out_bcount_opt", "__out_bcount", "__in_bcount", "__in_bcount_opt", "__RPC__out_ecount_full_string"] diff --git a/ctypes_generation/generate.py b/ctypes_generation/generate.py index e80782d..f718a03 100644 --- a/ctypes_generation/generate.py +++ b/ctypes_generation/generate.py @@ -14,6 +14,8 @@ import func_parser import def_parser import com_parser +from simpleparser import ParsingError + pjoin = os.path.join pexists = os.path.exists dedent = textwrap.dedent @@ -28,7 +30,13 @@ to_dest = lambda path: pjoin(DEST_DIR, path) class ParsedFile(object): def __init__(self, filename): self.filename = filename - self.data = self.PARSER(open(filename).read()).parse() + try: + self.data = self.PARSER(open(filename).read()).parse() + except ParsingError as e: + print(" !! Error while parsing file <{0}> !!".format(filename)) + print(e) + raise + self.exports = set() self.imports = set() self.compute_imports_exports(self.data) @@ -131,12 +139,13 @@ class COMParsedFile(ParsedFile): class ParsedFileGraph(object): - def __init__(self, nodes, depnodes): # depnodes: nodes that we dont have to handle but want can take export from + def __init__(self, nodes, depnodes, missing_handler=None): # depnodes: nodes that we dont have to handle but want can take export from self.nodes = nodes self.depnodes = depnodes self.exports_database = {} self.depandances_database = {node: set() for node in nodes} self.build_export_database(self.nodes) + self.missing_handler = missing_handler self.build_depandance_database() def build_dependancy_graph(self): @@ -166,7 +175,13 @@ class ParsedFileGraph(object): try: self.depandances_database[node].add(self.exports_database[import_]) except KeyError as e: - raise ValueError("Missing dependancy <{0}> of {1}".format(import_, node)) + self.on_missing_dependancy(import_, node) + # raise ValueError("Missing dependancy <{0}> of {1}".format(import_, node)) + + def on_missing_dependancy(self, import_, node): + if self.missing_handler is not None: + return self.missing_handler(import_, node) + raise ValueError("Missing dependancy <{0}> of {1}".format(import_, node)) def build_export_database(self, nodes): @@ -194,8 +209,23 @@ class FakeExporter(object): self.exports = exports class ParsedDirectory(object): - def __init__(self, filetype, directory): - self.nodes = [filetype(f) for f in glob.glob(directory)] + def __init__(self, filetype, src, recurse=False): + if not recurse: + if os.path.isdir(src): + srcglob = pjoin(src, "*.txt") + else: + srcglob = src + files = glob.glob(srcglob) + else: + # Recurse search of .txt files + files = [os.path.join(path, filename) + for (path, _, files) in os.walk(src) + for filename in files + if filename.endswith(".txt")] + + + + self.nodes = [filetype(f) for f in files] ### Generation Class ### @@ -528,7 +558,6 @@ class MetaFileGenerator(NoTemplatedGenerator): self.emitline("import {0} as {0}_module".format(modname)) self.emitline("{0}_walker = generate_walker({0}, {1}_module)".format(name, modname)) - class ModuleGenerator(object): def __init__(self, name, filetype, ctypesgenerator, docgenerator, src): self.name = name @@ -543,27 +572,44 @@ class ModuleGenerator(object): def add_module_dependancy(self, module): self.dependances_modules.add(module) - def parse_directory(self, globdir): - self.nodes += ParsedDirectory(self.filetype, globdir).nodes def get_template_filename(self): return pjoin(self.src, "template.py") - def parse_source_directory(self): - if os.path.isdir(self.src): - srcglob = pjoin(self.src, "*.txt") - else: - srcglob = self.src - self.parse_directory(srcglob) + def parse_source_directory(self, recurse=False): + self.nodes += ParsedDirectory(self.filetype, self.src, recurse=recurse).nodes def resolve_dependancies(self, depnodes=[]): g = ParsedFileGraph(self.nodes, depnodes=depnodes) return g.build_dependancy_graph() def check_dependancies_without_flattening(self, depnodes): - g = ParsedFileGraph(self.nodes, depnodes=depnodes) # init check for missing dependance + self.missing_interfaces = [] + g = ParsedFileGraph(self.nodes, depnodes=depnodes, missing_handler=self.missing_com_interface) # init check for missing dependance + if self.missing_interfaces: + missing_names = [x[0] for x in self.missing_interfaces] + if not args.autocopy: + raise ValueError("Missing COM dependancy Names : {0}".format(missing_names)) + + print("Missing COM interfaces are: {0}".format(missing_names)) + autocopied = [] + for name, node in self.missing_interfaces: + filename = "{0}\\{1}.txt".format(args.autocopy, name) + if os.path.exists(filename): + autocopied.append(name) + print("Auto-copy <{0}>".format(filename)) + targetdir = os.path.dirname(node.filename) + print(filename, targetdir) + shutil.copy(filename, targetdir) + if autocopied: + raise ValueError("Auto-copyied Names : {0}".format(autocopied)) + raise ValueError("Missing COM dependancy Names : {0}".format(missing_names)) return g.nodes + def missing_com_interface(self, import_, node): + print("Missing name <{0}> in file <{1}>".format(import_, node.filename)) + self.missing_interfaces.append((import_, node)) + def generate(self): self.parse_source_directory() # Flatten the graph @@ -607,6 +653,14 @@ class ModuleGenerator(object): return res +import argparse + +parser = argparse.ArgumentParser(prog=__file__) +parser.add_argument('--autocopy', help="[PRIVATE OPTION] A directory used to find missing COM interface") +args = parser.parse_args() + + + # Copy Flag code shutil.copy(from_here(r"definitions\flag.py"), DEST_DIR) @@ -643,7 +697,7 @@ print("== Generating COM interfaces ==") com_module_generator = ModuleGenerator("interfaces", COMParsedFile, COMCtypesGenerator, None, from_here(r"definitions\com")) # Load the interface_to_iid file needed by the 'COMCtypesGenerator' com_module_generator.after_ctypes_generator_init = lambda cgen: cgen.parse_iid_file(from_here("definitions\\interface_to_iid.txt")) -com_module_generator.parse_source_directory() +com_module_generator.parse_source_directory(recurse=True) com_module_generator.add_module_dependancy(structure_module_generator) com_module_generator.resolve_dependancies = com_module_generator.check_dependancies_without_flattening # No real flattening as we have circular dep in Interfaces VTBL com_module_generator.resolve_dep_and_generate([BasicTypeNodes()]) diff --git a/ctypes_generation/struct_parser.py b/ctypes_generation/struct_parser.py index 719533a..4f7d589 100644 --- a/ctypes_generation/struct_parser.py +++ b/ctypes_generation/struct_parser.py @@ -52,10 +52,17 @@ class WinStructParser(Parser): sep = self.next_token() self.assert_token_type(ColonToken, sep) - def parse_enum(self): + def parse_enum(self, is_typedef): """Handle enum typedef with no value assignement and 1 typedef after""" - enum_name = self.assert_token_type(NameToken).value - res_enum = WinEnum(enum_name) + if not type(self.peek()) == OpenBracketToken: + # Not an ANON enum + enum_name = self.assert_token_type(NameToken).value + res_enum = WinEnum(enum_name) + else: + if not is_typedef: + raise ValueError("Anonymous union not in a typedef") + res_enum = WinEnum(None) + self.assert_token_type(OpenBracketToken) count = itertools.count() assigned_value = False @@ -84,7 +91,6 @@ class WinStructParser(Parser): return res_enum - def parse_winstruct(self): is_typedef = False peeked = self.peek() @@ -94,17 +100,24 @@ class WinStructParser(Parser): def_type = self.assert_token_type(KeywordToken) if def_type.value == "enum": - return self.parse_enum() + return self.parse_enum(is_typedef) if def_type.value == "struct": WinDefType = WinStruct elif def_type.value == "union": WinDefType = WinUnion else: raise ParsingError("Expecting union or struct got <{0}> instead".format(def_type.value)) - struct_name = self.assert_token_type(NameToken) + if not type(self.peek()) == OpenBracketToken: + # Not an anonymous structure def + struct_name = self.assert_token_type(NameToken).value + else: + # Anonymous structure def: check if we are ina typedef + if not is_typedef: + raise ValueError("Anonymous structure/union not in a typedef") + struct_name = None # self.assert_token_type(OpenBracketToken) - result = WinDefType(struct_name.value, self.pack) + result = WinDefType(struct_name, self.pack) while type(self.peek()) != CloseBracketToken: tok_type, tok_name, nb_rep = self.parse_def() diff --git a/ctypes_generation/winstruct.py b/ctypes_generation/winstruct.py index f91aa30..cac8bf7 100644 --- a/ctypes_generation/winstruct.py +++ b/ctypes_generation/winstruct.py @@ -39,11 +39,15 @@ class WinStruct(object): self.fields.append(field) def add_typedef(self, name): + if self.name is None: + self.name = name if name in self.typedef: raise ValueError("nop") self.typedef[name] = self def add_ptr_typedef(self, name): + if self.name is None: + raise ValueError("Anonymous struct first typedef ({0}) should not be a PTR type".format(name)) if name in self.typedef: raise ValueError("nop") self.typedef[name] = Ptr(self) @@ -103,6 +107,8 @@ class WinStruct(object): typedef_ctypes = [] for typedef_name, value in self.typedef.items(): str_value = self.name + if typedef_name == str_value: # Do not generate "X= X" line (anonymous structs gen this) + continue if type(value) == Ptr: str_value = "POINTER({0})".format(self.name) typedef_ctypes += ["{0} = {1}".format(typedef_name, str_value)] @@ -130,11 +136,16 @@ class WinEnum(object): self.fields.append((number, name)) def add_typedef(self, name): + if self.name is None: + # Setup our name to our first typedef + self.name = name if name in self.typedef: raise ValueError("nop") self.typedef[name] = self def add_ptr_typedef(self, name): + if self.name is None: + raise ValueError("Anonymous enum first typedef ({0}) should not be a PTR type".format(name)) if name in self.typedef: raise ValueError("nop") self.typedef[name] = Ptr(self) @@ -152,6 +163,8 @@ class WinEnum(object): for typedef_name, value in self.typedef.items(): str_value = self.name + if typedef_name == str_value: # Do not generate "X= X" line (anonymous enum gen this) + continue if type(value) == Ptr: str_value = "POINTER({0})".format(self.name) lines += ["{0} = {1}".format(typedef_name, str_value)]