""" Generate the index.html/.md for the examples subdirectory, based on structured docstrings on each Python example """ from __future__ import print_function import ast import re import os import sys prog_name, _ = os.path.splitext(os.path.basename(sys.argv[0])) prog_dir = os.path.dirname(sys.argv[0]) from argparse import ArgumentParser parser = ArgumentParser() subparsers = parser.add_subparsers(dest="command") subparsers.required = True w_subparser = subparsers.add_parser("write") s_subparser = subparsers.add_parser("show") parser.add_argument("-v", "--verbose", default=False, action="store_true") w_subparser.add_argument("-e", "--examples-dir", required=True) w_subparser.add_argument("-t", "--template", required=True) w_subparser.add_argument("-o", "--output", required=True) s_subparser.add_argument("-k", "--keywords", default=False, action="store_true") s_subparser.add_argument("-s", "--search", type=str) s_subparser.add_argument("-e", "--examples-dir", required=True) args = parser.parse_args() #------------------------------------------------------------------------------- CATEGORY_INFO = [ ( "ui", "User interface", "Creating & manipulating user-interface widgets, prompting the " + "user with forms, enriching existing widgets, or creating your " + "own UI through Python Qt bindings." ), ( "disassembler", "Disassembly", "Various ways to query, or modify the disassembly listing, " + "alter the way analysis is performed, or be notified of changes " + "made to the IDB." ), ( "decompiler", "Decompilation", "Querying the decompiler, manipulating the decompilation trees " + "(either at the microcode level, or the C-tree), and examples " + "showing how to intervene in the decompilation output." ), ( "debugger", "Debuggers", "Driving debugging sessions, be notified of debugging events." ), ( "types", "Working with types", "These samples utilize our Type APIs, which allow you to manage " + "the types and perform various operations on them, like creating " + "the structures or enums and adding their members programmatically." ), ( "misc", "Miscellaneous", "Miscellaneous examples that don't quite fall into another category, " + "but don't really justify one of their own." ), ] #------------------------------------------------------------------------------- LEVEL_INFO = [ ( "beginner", "Beginner", ), ( "intermediate", "Intermediate", ), ( "advanced", "Advanced", ), ] # -------------------------------------------------------------------------- def verb(msg): if args.verbose: print(msg) # -------------------------------------------------------------------------- class ProcessException(Exception): pass # -------------------------------------------------------------------------- def read_config(): global config config_path = os.path.join(prog_dir, prog_name + ".cfg") with open(config_path, "r", encoding="UTF-8", errors="surrogateescape") as f: config = ast.literal_eval(f.read()) config["exclude"] = set(config["exclude"]) for _, key_res in config["auto-keywords"]: for i in range(len(key_res)): key_res[i] = re.compile(key_res[i] + "$") # anchored for match() #------------------------------------------------------------------------------- class Examples(object): def __init__(self): self.re_indent = re.compile(r"^ *") self.re_tag = re.compile(r"^ *([a-zA-Z_][a-zA-Z_0-9]*):") self.re_wspace = re.compile(r"[ \t\r\n]*") self.re_userlink = re.compile(r"@{(.*?)}") self.index = None self.shortcut_finder = ShortcutFinder() def process(self): if args.command == "write": # parses the templates, and bails out on template error # before going into analyzing the examples self.index = ExamplesIndex() self.examples = [] for relpath, path in self._files_with_extension(".py", args.examples_dir): verb("Processing \"%s\"" % path) with open(path, "r", encoding="UTF-8", errors="surrogateescape") as f: self._load_ast(relpath, ast.parse(f.read(), path)) self._post_read_processing() if args.command == "show": self._show() elif args.command == "write": self.index.produce(self.examples, self._collect_all_keywords()) def _post_read_processing(self): self._collect_example_names() # requisite for other post-read actions self._collect_imported_uses() # trim down the final collection self.examples = [e for e in self.examples \ if e.name not in config["exclude"]] self.examples = sorted(self.examples, key=lambda x: x.name) self._collect_example_names() # now invalid, redo self._parse_description_links() self._check_see_also_links() def _show(self): if args.keywords: print("\nKeywords used:") for keyword in self._collect_all_keywords(): print(" ", keyword) if args.search: print("\nExamples using \"%s\":" % args.search) for example in self.examples: if args.search in example.keywords: print(" ", example.path) def _collect_all_keywords(self): all_keywords = set() for example in self.examples: all_keywords.update(example.keywords) return sorted(all_keywords, key=lambda x: x.lower()) def _load_ast(self, relpath, tree): name = os.path.splitext(os.path.basename(relpath))[0] category = relpath.split(os.sep)[0] # fix for Windows: paths may contain "\" instead of "/"; # these ultimately come from os.walk() and the command line # (_files_with_extension(), plus args.examples_dir -> rootdir) relpath = relpath.replace("\\", "/") uses, imports = self._uses_and_imports(tree) docstring = ast.get_docstring(tree, False) tags = Tags() tags.add(Tags.PATH, relpath) tags.add(Tags.NAME, name) tags.add(Tags.CATEGORY, category) shortcuts = sorted(self.shortcut_finder.find(tree), \ key=lambda x: x.lower()) tags.add(Tags.SHORTCUTS, shortcuts) # the next two are kept as sets; # they will be sorted lists after _collect_imported_uses() tags.add(Tags.USES, uses) tags.add(Tags.IMPORTS, imports) self._parse_structured_comment(docstring, tags) self._add_autokeywords(tags) self.examples.append(Example(tags.get())) def _add_autokeywords(self, tags): def add_keyword(m, key_out): groups_dict = {str(n): m.group(n) for n in range(1, m.re.groups+1)} new_keyword = key_out % groups_dict keywords = tags.tag_so_far(Tags.KEYWORDS) keywords.append(new_keyword) global config autokeys = config["auto-keywords"] for use in tags.tag_so_far(Tags.USES): for key_out, key_res in autokeys: # check for ALL auto-keywords generated by this "use" for key_re in key_res: # check for ANY pattern that matches this auto-keyword m = key_re.match(use) if m: add_keyword(m, key_out) break keywords = tags.tag_so_far(Tags.KEYWORDS) tags.add(Tags.KEYWORDS, sorted(keywords, key=lambda x: x.lower())) def _parse_structured_comment(self, docstring, tags): if not docstring: return first_line = True group_tag = None group_lines = [] def end_group(): if group_tag: self._remove_common_indentation(group_lines) if group_tag == Tags.DESCRIPTION: item = "\n".join(group_lines).strip("\n") elif group_tag in (Tags.KEYWORDS, Tags.SEE_ALSO): item = [] for x in ",".join(group_lines).split(","): x = x.strip() if x: item.append(x) else: item = " ".join(group_lines).strip() if group_tag in (Tags.SUMMARY, Tags.DESCRIPTION): # capitalize first m = self.re_wspace.match(item) if m and m.end() < len(item): item = item[:m.end()] + item[m.end()].upper() \ + item[m.end()+1:] verb(" Read from docstring: tag \"%s\"" % group_tag) tags.add(group_tag, item) # Blocks of lines are separated by indentation: # a block MUST begin with a tag (followed by ":") at indent 0. # # An attempt is made to preserve the original indentation # of each block of lines. for line in docstring.split("\n"): indent, line = self._indented_line(line.rstrip(), 0) if indent == 0 and line: end_group() first_line = True group_tag = None group_lines = [] if first_line: if not line: # ignore blank lines before the first tag continue first_line = False m = self.re_tag.match(line) if not m: tags.error("No tag in docstring: \"\"\"\n%s\n\"\"\"" % docstring) group_tag = m.group(1) tags.valid(group_tag) tags.can_occur_in_docstring(group_tag) indent, line = self._indented_line(line, m.end()) if group_tag: group_lines.append( (indent, line) ) end_group() def _indented_line(self, line, fromPos): line = line[fromPos:] m = self.re_indent.match(line) indent = fromPos + len(m.group()) line = line[m.end():] return indent, line def _remove_common_indentation(self, lines): if not lines: return min_indent = min(t[0] for t in lines if t[1]) for i in range(len(lines)): if lines[i][1]: lines[i] = ((lines[i][0] - min_indent) * " ") + lines[i][1] else: lines[i] = "" def _uses_and_imports(self, tree): names = set() def callback_uses(name): verb(" Uses \"%s\"" % name) names.add(name) imports = set() def callback_imports(name): verb(" Imports \"%s\"" % name) imports.add(name) visitor = TreeVisitor(callback_uses, callback_imports, ["idc", "idautils"]) visitor.generic_visit(tree) return names, imports def _parse_description_links(self): # (all examples have been read already) for example in self.examples: text = example.description out = "" while True: m = self.re_userlink.search(text) if not m: break target_name = m.group(1) if target_name not in self.examples_set: raise ProcessException( "Link in \"%s\": \"%s\" is not an example name" \ % (example.path, target_name)) try: subs = config["link-format"] % {'1': target_name} except Exception as ex: raise ProcessException( "Link format, %s: %s" % (type(ex).__name__, ex)) out = text[:m.start()] + subs text = text[m.end():] example.description = out + text def _check_see_also_links(self): # do their targets exist? # (all examples have been read already) for example in self.examples: for see_also in example.see_also: if see_also not in self.examples_set: raise ProcessException( "%s\n See-also link to unexistent \"%s\"" % (example.path, see_also)) def _collect_imported_uses(self): # if an example imports another, # make the parent "use" the API calls that the child "uses" # (this has to be done after all examples have been read) examples_dict = { e.name: e for e in self.examples } # 1. restrict the imports to examples only imported_set = set() for example in self.examples: example.imports &= self.examples_set imported_set |= example.imports # 2. topological sort of the imported examples graph = {} for example in self.examples: for imported in example.imports: # this example imports other examples if example.name in graph: graph[example.name].append(imported) else: graph[example.name] = [imported] roots = set(graph) - imported_set # 3. add children uses to parents for name in roots: self._add_imported_uses(name, examples_dict, graph, set()) # 4. now we can make them sorted lists instead of sets # (we leave "uses" of uppercase before lowercase; # the use of constants looks better this way) for example in self.examples: example.uses = sorted(example.uses) example.imports = sorted(example.imports, key=lambda x: x.lower()) def _add_imported_uses(self, name, examples_dict, graph, cycle_detect): # cycles shouldn't happen if we're reading code that actually works, # but we better protect ourselves against bad input cycle_detect.add(name) example = examples_dict[name] for child in graph[name]: if child in graph: if child in cycle_detect: raise ProcessException("Cycle in import: %s -> %s" % (name, child)) self._add_imported_uses(child, examples_dict, graph, cycle_detect) example.uses |= examples_dict[child].uses cycle_detect.remove(name) def _collect_example_names(self): # (after all examples have been read) self.examples_set = set(e.name for e in self.examples) def _files_with_extension(self, ext, rootdir): for path, _, files in os.walk(rootdir): for filename in files: if os.path.splitext(filename)[1] == ext: relpath = os.path.relpath(path, rootdir) yield os.path.join(relpath, filename), \ os.path.join(path, filename) #------------------------------------------------------------------------------- class Tags(object): NAME = "name" PATH = "path" SUMMARY = "summary" DESCRIPTION = "description" CATEGORY = "category" KEYWORDS = "keywords" SHORTCUTS = "shortcuts" USES = "uses" IMPORTS = "imports" SEE_ALSO = "see_also" AUTHOR = "author" LEVEL = "level" # tags that can be used in the template (with the {{example.XXX}} syntax) ALL_TAGS = set([NAME, PATH, SUMMARY, DESCRIPTION, CATEGORY, \ KEYWORDS, USES, IMPORTS, SEE_ALSO, AUTHOR, \ LEVEL]) # tags that can be used in the examples' docstrings IN_DOCSTRING = set([SUMMARY, DESCRIPTION, CATEGORY, \ KEYWORDS, USES, SEE_ALSO, AUTHOR, \ LEVEL]) def __init__(self): self.items = {} for tag in self.ALL_TAGS: if tag in (self.KEYWORDS, self.SEE_ALSO): self.items[tag] = [] elif tag in (self.USES, self.IMPORTS): self.items[tag] = set() else: self.items[tag] = "" def error(self, message): raise ProcessException("%s:\n %s" \ % (self.items[self.PATH], message)) def valid(self, tag): if tag not in self.ALL_TAGS: self.error("Unrecognized tag: \"%s\"" % tag) def can_occur_in_docstring(self, tag): if tag not in self.IN_DOCSTRING: self.error("Not allowed in docstring: \"%s\"" % tag) def add(self, tag, item): self.items[tag] = item def tag_so_far(self, tag): return self.items[tag] def get(self): return self.items #------------------------------------------------------------------------------- class Example(object): def __init__(self, content): self.content = content _, self.file_name = os.path.split(self.path) def __getattr__(self, key): if key in self.content: return self.content[key] raise AttributeError("Example: \"%s\"" % key) #------------------------------------------------------------------------------- class TreeVisitor(ast.NodeVisitor): def __init__(self, callback_uses, callback_imports, interesting_names, *args): super(TreeVisitor, self).__init__(*args) self.callback_uses = callback_uses self.callback_imports = callback_imports self.interesting_names = set(interesting_names) def visit_Import(self, node): for alias in node.names: self.callback_imports(alias.name) def visit_ImportFrom(self, node): self.callback_imports(node.module) if self._is_interesting(node.module): self._add_interesting_names(node.names) def _add_interesting_names(self, aliases): for alias in aliases: if isinstance(alias, ast.alias): name = alias.asname if alias.asname else alias.name self.interesting_names.add(name) def _is_interesting(self, name): return name.startswith("ida_") or name in self.interesting_names def visit_Name(self, node): self._callback_if_interesting(node) def visit_Attribute(self, node): self._callback_if_interesting(node) def _callback_if_interesting(self, node): names = self._dotted_name(node) if not names: return if self._is_interesting(names[0]) \ and names[-1] != "__init__": # removed calls to base constructors self.callback_uses(".".join(names)) def _dotted_name(self, node): if isinstance(node, ast.Name): return [node.id] if isinstance(node, ast.Attribute): names = self._dotted_name(node.value) if names: return names + [node.attr] return None #------------------------------------------------------------------------------- class ShortcutFinder(ast.NodeVisitor): def find(self, tree): self._reset() self.generic_visit(tree) return self.results def _reset(self): self.results = [] self.stack = [] def generic_visit(self, node): # look for "action_desc_t" instances, 4th constructor parameter if isinstance(node, ast.Call): name = self._name(node.func) if name == "ida_kernwin.action_desc_t": if len(node.args) >= 4: # search the scopes for value(s) of this parameter values = self._find_values(node.args[3]) for value in values: self.results.append(value) # assigment - fill in (string) values into the current scope elif isinstance(node, ast.Assign): names = [self._name(t) for t in node.targets] scope = self.stack[-1][1] value = None # save strings if isinstance(node.value, ast.Constant): value = node.value.value # save also lists of names, in case they are class names elif isinstance(node.value, ast.List): if all(isinstance(elt, ast.Name) for elt in node.value.elts): value = [elt.id for elt in node.value.elts] for name in names: self._assign(scope, name, value) # if our immediate ancestor is a class definition, # add this symbol also to the class scope if len(self.stack) > 1 and self.stack[-2][0]: class_scope = self.stack[-2][0] if self.class_name in class_scope: for name in names: self._assign(class_scope[self.class_name], name, value) # remember class names elif isinstance(node, ast.ClassDef): class_scope = self.stack[-1][0] self.class_name = node.name class_scope[self.class_name] = {} # "for" - treat it as a list assignment elif isinstance(node, ast.For): # only syntax recognized: "for name in name" if isinstance(node.target, ast.Name) and \ isinstance(node.iter, ast.Name): name = node.target.id scope = self.stack[-1][1] values = self._find_list(node.iter.id) # save the variable as having multiple values self._assign(scope, name, values) scope[name] = values # keep a stack of scopes [classes, variables] self.stack.append([{}, {}]) super(ShortcutFinder, self).generic_visit(node) self.stack.pop() def _assign(self, scope, name, value): if value: scope[name] = value else: # symbol overwritten with something I don't understand, forget if name in scope: del scope[name] def _find_values(self, arg): if isinstance(arg, ast.Constant): return [arg.value] # single variable if isinstance(arg, ast.Name): value = self._variable_in_scope(arg.id) if value: return [value] # variable dot variable - TODO: nested classes if isinstance(arg, ast.Attribute) and \ isinstance(arg.value, ast.Name): left_name = arg.value.id right_name = arg.attr values = self._find_list(left_name) if values: # return the values for each class scope returned = [] for c_name in values: class_scope = self._find_class_scope(c_name) if right_name in class_scope: returned.append(class_scope[right_name]) return returned return [] def _variable_in_scope(self, name): for i in range(len(self.stack) - 1, -1, -1): scope = self.stack[i][1] if name in scope: return scope[name] return None def _find_list(self, name): values = self._variable_in_scope(name) if isinstance(values, list): return values return None def _find_class_scope(self, name): for i in range(len(self.stack) - 1, -1, -1): class_scope = self.stack[i][0] if name in class_scope: return class_scope[name] return {} def _name(self, node): if isinstance(node, ast.Name): return node.id if isinstance(node, ast.Attribute): return self._name(node.value) + "." + node.attr # too complex for me return "@" + node.__class__.__name__ #------------------------------------------------------------------------------- class ExamplesIndex(object): def __init__(self): self.replacer = TemplateReplacer() with open(args.template, "r", encoding="UTF-8", errors="surrogateescape") as f: self.replacer.template(f.read()) def produce(self, examples, all_keywords): with open(args.output, "w", encoding="UTF-8") as f: self.replacer.expand(examples, all_keywords, f) #------------------------------------------------------------------------------- class Categories(object): class Category(object): class Level(object): def __init__(self, id): self.id = id assert(self.id) for rank, tpl in enumerate(LEVEL_INFO): if tpl[0] == id: self.rank = rank _, label = tpl self.label = label # will bomb if not found self.examples = [] def append(self, example): self.examples.append(example) def __init__(self, id): self.id = id for rank, tpl in enumerate(CATEGORY_INFO): if tpl[0] == id: self.rank = rank _, label, description = tpl self.label = label # will bomb if not found self.description = " ".join(description.split("\n")) self.levels = [] def get_level(self, id): for l in self.levels: if l.id == id: return l l = self.Level(id) self.levels.append(l) return l def append(self, example): if not example.level: raise Exception("Missing level for %s" % example.path) self.get_level(example.level).append(example) def __init__(self): self.categories = [] def get_category(self, id): for c in self.categories: if c.id == id: return c c = self.Category(id) self.categories.append(c) return c def append(self, example): self.get_category(example.category).append(example) #------------------------------------------------------------------------------- class TemplateReplacer(object): def __init__(self): self.re_percent_line = re.compile(r" *% *") self.re_end_keyword = re.compile(r"end([a-z]+|})$") self.re_variable = re.compile(r"{{(.*?)}}") self.keywords = { # has_body, keyword ast.If: (True, "if"), ast.For: (True, "for"), ast.FunctionDef: (True, "def"), ast.Break: (False, "break"), ast.Continue: (False, "continue"), ast.Return: (False, "return") } # "else" treated separately def template(self, content): lines = content.rstrip("\n").split("\n") # convert the template lines into a Python module body = [self._sandwich(lines)] self.module = ast.Module(body=body, type_ignores=[]) self.module = ast.fix_missing_locations(self.module) # "import" the module self.module_scope = {} exec(compile(self.module, "Template", "exec"), self.module_scope) def expand(self, examples, all_keywords, f): # group by category categories = Categories() for example in examples: assert example.description and example.summary and not example.summary.endswith("."), "Path: %s" % example.path categories.append(example) # call the template's entry point self.module_scope["__mm__entry"](categories, all_keywords, f) def _sandwich(self, lines, lineno=0, open_keyword=None): in_else = False body = [] text = "" while lineno < len(lines): line = lines[lineno] lineno += 1 # process "% statement"s in the template m = self.re_percent_line.match(line) if m: line = self._clean_line(line, m.end()) # text so far if text: body.extend(self._substitute_variables(text)) text = "" # "else" is not an ast.; parsed separately ret = self._is_else(line, lineno, open_keyword) if ret: in_else = True prev_body = body body = [] continue # code block start "%{" if self._is_start_of_code_block(line, lineno): inner_body, _, lineno = self._code_block(lines, lineno) body.extend(inner_body) continue if self._is_end_keyword(line, lineno, open_keyword): if not in_else: prev_body, body = body, [] return prev_body, body, lineno has_body, keyword, statement = self._check_percent_line(line, lineno) if has_body: inner_body, \ inner_else, lineno = self._sandwich(lines, lineno, keyword) statement.body = inner_body if inner_else: statement.orelse = inner_else body.append(statement) continue text += line + "\n" if open_keyword: raise ProcessException( "Template: unterminated \"%s\"" % open_keyword) if text: body.extend(self._substitute_variables(text)) # wrap the final code in a function, so that data can be passed return self._function_def("__mm__entry", ["categories", "all_keywords", "_mm_f"], body) def _code_block(self, lines, lineno): start_lineno = lineno code = "" while lineno < len(lines): line = lines[lineno] lineno += 1 # process "% statement"s in the template m = self.re_percent_line.match(line) if m: line = self._clean_line(line, m.end()) if self._is_end_of_code_block(line, lineno): try: body = ast.parse(code, "Template").body return body, [], lineno except: line_range = "%s" % lineno if lineno == start_lineno \ else "%d-%d" % (start_lineno, lineno) raise ProcessException("Template[%s]: syntax error" % \ line_range) break # no other % allowed but %} code += line + "\n" raise ProcessException("Template: unterminated \"{\"") def _is_else(self, line, lineno, open_keyword): if line != "else:": return False # use only in statements that accept "else" if open_keyword not in ("if", "for"): raise ProcessException( "Template[%d]: invalid \"else\"%s" % \ (lineno, " in \"%s\"" % open_keyword \ if open_keyword else "")) return True def _is_start_of_code_block(self, line, lineno): return line == "{" def _is_end_of_code_block(self, line, lineno): return line == "}" def _is_end_keyword(self, line, lineno, open_keyword): m = self.re_end_keyword.match(line) if not m: return False if m.group(1) != open_keyword: raise ProcessException( " Template[%d]: expected \"end%s\"" % \ (lineno, open_keyword)) return True def _check_percent_line(self, line, lineno): # return the code for a "% statement" in the template if line[-1] == ":": line += "\n pass" try: tree = ast.parse(line, "Template") except: raise ProcessException("Template[%d]: syntax error" % lineno) if type(tree) is ast.Module and \ len(tree.body) == 1: head = type(tree.body[0]) if head in self.keywords: return self.keywords[head] + (tree.body[0],) raise ProcessException("Template[%d]: unrecognized keyword" % lineno) def _clean_line(self, line, code_pos): comment_pos = line.find("#") if comment_pos >= 0: line = line[:comment_pos] return line[code_pos:].rstrip() def _substitute_variables(self, input_text): # generate code to substitute {{variables}} when the template is run code = [] text = "" while input_text: m = self.re_variable.search(input_text) if not m: break # text in-between text += input_text[:m.start()] if text: code.append(self._output_text(text)) text = "" input_text = input_text[m.end():] code.append(self._output_eval(m.group(1))) text += input_text if text: code.append(self._output_text(text)) return code def _output_text(self, text): return ast.Expr(value=self._call(self._dot("_mm_f", "write"), [ast.Constant(text)])) def _output_eval(self, expr): # eval(expr) value = self._call(ast.Name("eval", ast.Load()), [ast.Constant(expr)]) # str(eval(expr)) valueStr = self._call(ast.Name("str", ast.Load()), [value]) # _mm_f.write(str(eval(expr))) call = self._call(self._dot("_mm_f", "write"), [valueStr]) return ast.Expr(call) def _dot(self, name1, name2): return ast.Attribute(value=ast.Name(name1, ast.Load()), attr=name2, ctx=ast.Load()) def _call(self, func, args): return ast.Call( func=func, args=args, keywords=[] ) def _function_def(self, fun_name, fun_args, fun_body): fun_args = ast.arguments(args=[ast.arg(a, None) \ for a in fun_args], posonlyargs=[], kwonlyargs=[], kw_defaults=[], defaults=[]) return ast.FunctionDef(name=fun_name, args=fun_args, body=fun_body, decorator_list=[]) #------------------------------------------------------------------------------- try: read_config() e = Examples() e.process() except ProcessException as ex: print(ex) sys.exit(1) # other exceptions - let them fail