From 61ddb4821a8407f488e40fbae8d136c66cef6083 Mon Sep 17 00:00:00 2001 From: hakril Date: Wed, 17 Jan 2018 01:17:55 +0100 Subject: [PATCH] Working in new ctypes generation code and features --- ctypes_generation/def_parser.py | 24 +- ctypes_generation/definitions/com/template.py | 72 +++ .../{ => defines}/custom_rpc_windef.txt | 0 .../definitions/defines/template.py | 7 + .../definitions/{ => defines}/windef.txt | 0 .../{ => defines}/windef_error.txt | 0 .../{ => defines}/windef_evtlog.txt | 0 .../{ => defines}/wintrust_crypt_def.txt | 0 ctypes_generation/definitions/flag.py | 50 +++ .../definitions/{ => functions}/winfunc.txt | 0 .../winfunc_crypto.txt.ignore} | 0 .../winfunc_crypto_wintrust.txt | 0 .../{ => functions}/winfunc_evtlog.txt | 0 .../{ => functions}/winfunc_notdoc.txt | 0 .../definitions/ntstatus_template.py | 29 ++ .../definitions/simple_types.txt | 91 ++++ .../{ => structures}/display_struct.txt | 16 +- .../definitions/structures/template.py | 37 ++ .../{ => structures}/winstruct.txt | 28 -- .../{ => structures}/winstruct_alpc.txt | 0 .../{ => structures}/winstruct_apisetmap.txt | 0 .../{ => structures}/winstruct_bits.txt | 0 .../{ => structures}/winstruct_evtlog.txt | 0 .../{ => structures}/winstruct_file_info.txt | 0 ctypes_generation/generate.py | 123 ++++-- ctypes_generation/newgen.py | 418 ++++++++++++++++++ ctypes_generation/simpleparser.py | 10 +- ctypes_generation/struct_parser.py | 26 ++ 28 files changed, 859 insertions(+), 72 deletions(-) create mode 100644 ctypes_generation/definitions/com/template.py rename ctypes_generation/definitions/{ => defines}/custom_rpc_windef.txt (100%) create mode 100644 ctypes_generation/definitions/defines/template.py rename ctypes_generation/definitions/{ => defines}/windef.txt (100%) rename ctypes_generation/definitions/{ => defines}/windef_error.txt (100%) rename ctypes_generation/definitions/{ => defines}/windef_evtlog.txt (100%) rename ctypes_generation/definitions/{ => defines}/wintrust_crypt_def.txt (100%) create mode 100644 ctypes_generation/definitions/flag.py rename ctypes_generation/definitions/{ => functions}/winfunc.txt (100%) rename ctypes_generation/definitions/{winfunc_crypto.txt => functions/winfunc_crypto.txt.ignore} (100%) rename ctypes_generation/definitions/{ => functions}/winfunc_crypto_wintrust.txt (100%) rename ctypes_generation/definitions/{ => functions}/winfunc_evtlog.txt (100%) rename ctypes_generation/definitions/{ => functions}/winfunc_notdoc.txt (100%) create mode 100644 ctypes_generation/definitions/ntstatus_template.py create mode 100644 ctypes_generation/definitions/simple_types.txt rename ctypes_generation/definitions/{ => structures}/display_struct.txt (90%) create mode 100644 ctypes_generation/definitions/structures/template.py rename ctypes_generation/definitions/{ => structures}/winstruct.txt (99%) rename ctypes_generation/definitions/{ => structures}/winstruct_alpc.txt (100%) rename ctypes_generation/definitions/{ => structures}/winstruct_apisetmap.txt (100%) rename ctypes_generation/definitions/{ => structures}/winstruct_bits.txt (100%) rename ctypes_generation/definitions/{ => structures}/winstruct_evtlog.txt (100%) rename ctypes_generation/definitions/{ => structures}/winstruct_file_info.txt (100%) create mode 100644 ctypes_generation/newgen.py diff --git a/ctypes_generation/def_parser.py b/ctypes_generation/def_parser.py index 7117d6c..42d4c84 100644 --- a/ctypes_generation/def_parser.py +++ b/ctypes_generation/def_parser.py @@ -7,12 +7,11 @@ class WinDef(object): def __init__(self, name, code): self.name = name self.code = code - + def generate_ctypes(self): return """{0} = make_flag("{0}", {1})""".format(self.name, self.code) class WinDefParser(Parser): - def parse_define(self): self.assert_token_type(SharpToken) define = self.assert_token_type(NameToken) @@ -26,9 +25,26 @@ class WinDefParser(Parser): v = v[:-1] define_value.append(v) return WinDef(define_name.value, " ".join(define_value)) - + def parse(self): res = [] while self.peek() is not None: res.append(self.parse_define()) - return res \ No newline at end of file + return res + +# A simple Fake parser for NTSTATUS +class NtStatusParser(object): + def __init__(self, data): + self.input = data + + def parse(self): + nt_status_defs = [] + for line in self.input.split("\n"): + if not line: + continue + code, name, descr = line.split("|", 2) + code = int(code, 0) + descr = re.sub(" +", " ", descr[:-1]) # remove \n + descr = descr.replace('"', "'") + nt_status_defs.append((code, name, descr)) + return nt_status_defs \ No newline at end of file diff --git a/ctypes_generation/definitions/com/template.py b/ctypes_generation/definitions/com/template.py new file mode 100644 index 0000000..107c54e --- /dev/null +++ b/ctypes_generation/definitions/com/template.py @@ -0,0 +1,72 @@ +import functools +import ctypes + + +generate_IID = IID.from_raw + +class COMInterface(ctypes.c_void_p): + _functions_ = { + } + + def __getattr__(self, name): + if name in self._functions_: + return functools.partial(self._functions_[name], self) + return super(COMInterface, self).__getattribute__(name) + +class COMImplementation(object): + IMPLEMENT = None + + def get_index_of_method(self, method): + # This code is horrible but not totally my fault + # the PyCFuncPtrObject->index is not exposed to Python.. + # repr is: '' + rpr = repr(method) + if not rpr.startswith(" implementing <{1}> has no method <{2}>".format(type(self).__name__, self.IMPLEMENT.__name__, func_name)) + if not callable(implem): + raise ValueError("{0} implementing <{1}>: <{2}> is not callable".format(type(self).__name__, self.IMPLEMENT.__name__, func_name)) + return True + + def _create_vtable(self, interface): + implems = [] + names = [] + for index, name, method in self.extract_methods_order(interface): + func_implem = getattr(self, name) + #PVOID is 'this' + types = [method.restype, PVOID] + list(method.argtypes) + implems.append(ctypes.WINFUNCTYPE(*types)(func_implem)) + names.append(name) + class Vtable(ctypes.Structure): + _fields_ = [(name, ctypes.c_void_p) for name in names] + return Vtable(*[ctypes.cast(x, ctypes.c_void_p) for x in implems]), implems + + def __init__(self): + self.verify_implem(self.IMPLEMENT) + vtable, implems = self._create_vtable(self.IMPLEMENT) + self.vtable = vtable + self.implems = implems + self.vtable_pointer = ctypes.pointer(self.vtable) + self._as_parameter_ = ctypes.addressof(self.vtable_pointer) + + def QueryInterface(self, this, piid, result): + if piid[0] in (IUnknown.IID, self.IMPLEMENT.IID): + result[0] = this + return 1 + return E_NOINTERFACE + + def AddRef(self, *args): + return 1 + + def Release(self, *args): + return 0 diff --git a/ctypes_generation/definitions/custom_rpc_windef.txt b/ctypes_generation/definitions/defines/custom_rpc_windef.txt similarity index 100% rename from ctypes_generation/definitions/custom_rpc_windef.txt rename to ctypes_generation/definitions/defines/custom_rpc_windef.txt diff --git a/ctypes_generation/definitions/defines/template.py b/ctypes_generation/definitions/defines/template.py new file mode 100644 index 0000000..e9d55b2 --- /dev/null +++ b/ctypes_generation/definitions/defines/template.py @@ -0,0 +1,7 @@ +import platform +from flag import make_flag + +bits = platform.architecture()[0] +bitness = int(bits[:2]) + +NATIVE_WORD_MAX_VALUE = 0xffffffff if bitness == 32 else 0xffffffffffffffff diff --git a/ctypes_generation/definitions/windef.txt b/ctypes_generation/definitions/defines/windef.txt similarity index 100% rename from ctypes_generation/definitions/windef.txt rename to ctypes_generation/definitions/defines/windef.txt diff --git a/ctypes_generation/definitions/windef_error.txt b/ctypes_generation/definitions/defines/windef_error.txt similarity index 100% rename from ctypes_generation/definitions/windef_error.txt rename to ctypes_generation/definitions/defines/windef_error.txt diff --git a/ctypes_generation/definitions/windef_evtlog.txt b/ctypes_generation/definitions/defines/windef_evtlog.txt similarity index 100% rename from ctypes_generation/definitions/windef_evtlog.txt rename to ctypes_generation/definitions/defines/windef_evtlog.txt diff --git a/ctypes_generation/definitions/wintrust_crypt_def.txt b/ctypes_generation/definitions/defines/wintrust_crypt_def.txt similarity index 100% rename from ctypes_generation/definitions/wintrust_crypt_def.txt rename to ctypes_generation/definitions/defines/wintrust_crypt_def.txt diff --git a/ctypes_generation/definitions/flag.py b/ctypes_generation/definitions/flag.py new file mode 100644 index 0000000..acdb152 --- /dev/null +++ b/ctypes_generation/definitions/flag.py @@ -0,0 +1,50 @@ +import sys + +if sys.version_info.major == 3: + long = int + +class Flag(long): + def __new__(cls, name, value): + return super(Flag, cls).__new__(cls, value) + + def __init__(self, name, value): + self.name = name + + def __repr__(self): + return "{0}({1})".format(self.name, hex(self)) + + __str__ = __repr__ + + # Fix pickling with protocol 2 + def __getnewargs__(self, *args): + return self.name, long(self) + +class StrFlag(str): + def __new__(cls, name, value): + if isinstance(value, cls): + return value + return super(StrFlag, cls).__new__(cls, value) + + def __init__(self, name, value): + self.name = name + + def __repr__(self): + return "{0}({1})".format(self.name, str.__repr__(self)) + + # __str__ = __repr__ + + # Fix pickling with protocol 2 + def __getnewargs__(self, *args): + return self.name, str.__str__(self) + +def make_flag(name, value): + if isinstance(value, (int, long)): + return Flag(name, value) + return StrFlag(name, value) + +class FlagMapper(dict): + def __init__(self, *values): + self.update({x:x for x in values}) + + def __missing__(self, key): + return key \ No newline at end of file diff --git a/ctypes_generation/definitions/winfunc.txt b/ctypes_generation/definitions/functions/winfunc.txt similarity index 100% rename from ctypes_generation/definitions/winfunc.txt rename to ctypes_generation/definitions/functions/winfunc.txt diff --git a/ctypes_generation/definitions/winfunc_crypto.txt b/ctypes_generation/definitions/functions/winfunc_crypto.txt.ignore similarity index 100% rename from ctypes_generation/definitions/winfunc_crypto.txt rename to ctypes_generation/definitions/functions/winfunc_crypto.txt.ignore diff --git a/ctypes_generation/definitions/winfunc_crypto_wintrust.txt b/ctypes_generation/definitions/functions/winfunc_crypto_wintrust.txt similarity index 100% rename from ctypes_generation/definitions/winfunc_crypto_wintrust.txt rename to ctypes_generation/definitions/functions/winfunc_crypto_wintrust.txt diff --git a/ctypes_generation/definitions/winfunc_evtlog.txt b/ctypes_generation/definitions/functions/winfunc_evtlog.txt similarity index 100% rename from ctypes_generation/definitions/winfunc_evtlog.txt rename to ctypes_generation/definitions/functions/winfunc_evtlog.txt diff --git a/ctypes_generation/definitions/winfunc_notdoc.txt b/ctypes_generation/definitions/functions/winfunc_notdoc.txt similarity index 100% rename from ctypes_generation/definitions/winfunc_notdoc.txt rename to ctypes_generation/definitions/functions/winfunc_notdoc.txt diff --git a/ctypes_generation/definitions/ntstatus_template.py b/ctypes_generation/definitions/ntstatus_template.py new file mode 100644 index 0000000..bb588e8 --- /dev/null +++ b/ctypes_generation/definitions/ntstatus_template.py @@ -0,0 +1,29 @@ +import ctypes +from flag import Flag + +class NtStatusException(WindowsError): + ALL_STATUS = {} + def __init__(self , code): + try: + x = self.ALL_STATUS[code] + except KeyError: + x = (code, 'UNKNOW_ERROR', 'Error non documented in ntstatus.py') + self.code = x[0] + self.name = x[1] + self.descr = x[2] + x = ctypes.c_long(x[0]).value, x[1], x[2] + return super(NtStatusException, self).__init__(*x) + + def __str__(self): + return "{e.name}(0x{e.code:x}): {e.descr}".format(e=self) + + def __repr__(self): + return "{0}(0x{1:08x}, {2})".format(type(self).__name__, self.code, self.name) + + @classmethod + def register_ntstatus(cls, code, name, descr): + if code in cls.ALL_STATUS: + return # Use the first def + cls.ALL_STATUS[code] = (code, name, descr) + return Flag(name, code) + diff --git a/ctypes_generation/definitions/simple_types.txt b/ctypes_generation/definitions/simple_types.txt new file mode 100644 index 0000000..838aaf6 --- /dev/null +++ b/ctypes_generation/definitions/simple_types.txt @@ -0,0 +1,91 @@ +VOID = DWORD // This one is a 'cheat' to handle function returning void in ctypes +BYTE = c_ubyte +PWSTR = LPWSTR +PCWSTR = LPWSTR +SIZE_T = c_size_t +PSIZE_T = POINTER(SIZE_T) +PVOID = c_void_p +PPS_POST_PROCESS_INIT_ROUTINE = PVOID +NTSTATUS = DWORD +SECURITY_INFORMATION = DWORD +PSECURITY_INFORMATION = POINTER(SECURITY_INFORMATION) +PULONG = POINTER(ULONG) +PDWORD = POINTER(DWORD) +LPDWORD = POINTER(DWORD) +LPTHREAD_START_ROUTINE = PVOID +WNDENUMPROC = PVOID +PHANDLER_ROUTINE = PVOID +LPBYTE = POINTER(BYTE) +ULONG_PTR = PVOID +DWORD_PTR = ULONG_PTR +KAFFINITY = ULONG_PTR +KPRIORITY = LONG +CHAR = c_char +UCHAR = c_char +CSHORT = c_short +VARTYPE = c_ushort +PBOOL = POINTER(BOOL) +PSTR = LPSTR +PCSTR = LPSTR +va_list = c_char_p +BSTR = c_wchar_p +OLECHAR = c_wchar +POLECHAR = c_wchar_p +PUCHAR = POINTER(UCHAR) +double = c_double +FARPROC = PVOID +PSID = PVOID +PVECTORED_EXCEPTION_HANDLER = PVOID +ULONGLONG = c_ulonglong +LONGLONG = c_longlong +ULONG64 = c_ulonglong +UINT64 = ULONG64 +LONG64 = c_longlong +PLARGE_INTEGER = POINTER(LARGE_INTEGER) +DWORD64 = ULONG64 +SCODE = LONG +CIMTYPE = LONG +NET_IFINDEX = ULONG +IF_INDEX = NET_IFINDEX +IFTYPE = ULONG +PULONG64 = POINTER(ULONG64) +PBYTE = POINTER(BYTE) +PUINT = POINTER(UINT) +PHANDLE = POINTER(HANDLE) +HCATADMIN = HANDLE +HCATINFO = HANDLE +HCERTCHAINENGINE = HANDLE +LPHANDLE = POINTER(HANDLE) +ALPC_HANDLE = HANDLE +PALPC_HANDLE = POINTER(ALPC_HANDLE) +PHKEY = POINTER(HKEY) +ACCESS_MASK = DWORD +REGSAM = ACCESS_MASK +PBOOLEAN = POINTER(BOOLEAN) +SECURITY_CONTEXT_TRACKING_MODE = BOOLEAN +HCRYPTPROV_OR_NCRYPT_KEY_HANDLE = PULONG +HCRYPTPROV_LEGACY = PULONG +HCRYPTKEY = PULONG +HCRYPTPROV = PULONG +HCRYPTHASH = PULONG +ALG_ID = UINT +DISPID = LONG +MEMBERID = DISPID +PSECURITY_DESCRIPTOR = PVOID +LPPROC_THREAD_ATTRIBUTE_LIST = PVOID +LPUNKNOWN = POINTER(PVOID) +LPFILETIME = POINTER(FILETIME) +LPPOINT = POINTER(POINT) +LPRECT = POINTER(RECT) +SPC_UUID = BYTE * 16 +PIO_APC_ROUTINE = PVOID +DEVICE_TYPE = DWORD +PWINDBG_EXTENSION_APIS32 = PVOID +PWINDBG_EXTENSION_APIS64 = PVOID + +// Will be changed at import time +LPCONTEXT = PVOID +HCERTSTORE = PVOID +HCRYPTMSG = PVOID +PALPC_PORT_ATTRIBUTES = PVOID +PPORT_MESSAGE = PVOID \ No newline at end of file diff --git a/ctypes_generation/definitions/display_struct.txt b/ctypes_generation/definitions/structures/display_struct.txt similarity index 90% rename from ctypes_generation/definitions/display_struct.txt rename to ctypes_generation/definitions/structures/display_struct.txt index 3f5c99f..433bb5d 100644 --- a/ctypes_generation/definitions/display_struct.txt +++ b/ctypes_generation/definitions/structures/display_struct.txt @@ -1,11 +1,3 @@ -typedef struct tagRECT -{ - LONG left; - LONG top; - LONG right; - LONG bottom; -} RECT, *PRECT, *NPRECT, *LPRECT; - typedef struct tagRGBTRIPLE { BYTE rgbtBlue; BYTE rgbtGreen; @@ -71,4 +63,10 @@ typedef struct tagBITMAPINFO { typedef struct tagBITMAPCOREINFO { BITMAPCOREHEADER bmciHeader; RGBTRIPLE bmciColors[1]; -} BITMAPCOREINFO, *LPBITMAPCOREINFO, *PBITMAPCOREINFO; \ No newline at end of file +} BITMAPCOREINFO, *LPBITMAPCOREINFO, *PBITMAPCOREINFO; + +/* +typedef struct _FAKEYOLOSTRUCT { + ACL Reserved; +} _FAKEYOLOSTRUCT; +*/ \ No newline at end of file diff --git a/ctypes_generation/definitions/structures/template.py b/ctypes_generation/definitions/structures/template.py new file mode 100644 index 0000000..2b1a464 --- /dev/null +++ b/ctypes_generation/definitions/structures/template.py @@ -0,0 +1,37 @@ +from ctypes import * +from ctypes.wintypes import * + +from flag import Flag, FlagMapper +from yolo import * + +class EnumValue(Flag): + def __new__(cls, enum_name, name, value): + return super(EnumValue, cls).__new__(cls, name, value) + + def __init__(self, enum_name, name, value): + self.enum_name = enum_name + self.name = name + + def __repr__(self): + return "{0}.{1}({2})".format(self.enum_name, self.name, hex(self)) + + # Fix pickling with protocol 2 + def __getnewargs__(self, *args): + return self.enum_name, self.name, int(self) + + +class EnumType(DWORD): + values = () + mapper = {} + + @property + def value(self): + raw_value = super(EnumType, self).value + return self.mapper.get(raw_value, raw_value) + + def __repr__(self): + raw_value = super(EnumType, self).value + if raw_value in self.values: + value = self.value + return "<{0} {1}({2})>".format(type(self).__name__, value.name, hex(raw_value)) + return "<{0}({1})>".format(type(self).__name__, hex(self.value)) diff --git a/ctypes_generation/definitions/winstruct.txt b/ctypes_generation/definitions/structures/winstruct.txt similarity index 99% rename from ctypes_generation/definitions/winstruct.txt rename to ctypes_generation/definitions/structures/winstruct.txt index 9b8824a..dd9380a 100644 --- a/ctypes_generation/definitions/winstruct.txt +++ b/ctypes_generation/definitions/structures/winstruct.txt @@ -2447,12 +2447,6 @@ typedef struct WINTRUST_SGNR_INFO_ HCERTSTORE *pahStores; } WINTRUST_SGNR_INFO, *PWINTRUST_SGNR_INFO; - -typedef struct _FILETIME { - DWORD dwLowDateTime; - DWORD dwHighDateTime; -} FILETIME, *PFILETIME, *LPFILETIME; - typedef struct WINTRUST_CERT_INFO_ { DWORD cbStruct; @@ -2705,24 +2699,6 @@ typedef struct _PUBLIC_OBJECT_BASIC_INFORMATION { } PUBLIC_OBJECT_BASIC_INFORMATION, *PPUBLIC_OBJECT_BASIC_INFORMATION; - typedef struct _EVENTLOGRECORD { - DWORD Length; - DWORD Reserved; - DWORD RecordNumber; - DWORD TimeGenerated; - DWORD TimeWritten; - DWORD EventID; - WORD EventType; - WORD NumStrings; - WORD EventCategory; - WORD ReservedFlags; - DWORD ClosingRecordNumber; - DWORD StringOffset; - DWORD UserSidLength; - DWORD UserSidOffset; - DWORD DataLength; - DWORD DataOffset; -} EVENTLOGRECORD, *PEVENTLOGRECORD; typedef struct tagSOLE_AUTHENTICATION_SERVICE { DWORD dwAuthnSvc; @@ -3119,10 +3095,6 @@ typedef struct _CRYPT_ENCODE_PARA { PVOID pfnFree; } CRYPT_ENCODE_PARA, *PCRYPT_ENCODE_PARA; -typedef struct tagPOINT { - LONG x; - LONG y; -} POINT, *PPOINT, *LPPOINT; typedef struct _ACL { diff --git a/ctypes_generation/definitions/winstruct_alpc.txt b/ctypes_generation/definitions/structures/winstruct_alpc.txt similarity index 100% rename from ctypes_generation/definitions/winstruct_alpc.txt rename to ctypes_generation/definitions/structures/winstruct_alpc.txt diff --git a/ctypes_generation/definitions/winstruct_apisetmap.txt b/ctypes_generation/definitions/structures/winstruct_apisetmap.txt similarity index 100% rename from ctypes_generation/definitions/winstruct_apisetmap.txt rename to ctypes_generation/definitions/structures/winstruct_apisetmap.txt diff --git a/ctypes_generation/definitions/winstruct_bits.txt b/ctypes_generation/definitions/structures/winstruct_bits.txt similarity index 100% rename from ctypes_generation/definitions/winstruct_bits.txt rename to ctypes_generation/definitions/structures/winstruct_bits.txt diff --git a/ctypes_generation/definitions/winstruct_evtlog.txt b/ctypes_generation/definitions/structures/winstruct_evtlog.txt similarity index 100% rename from ctypes_generation/definitions/winstruct_evtlog.txt rename to ctypes_generation/definitions/structures/winstruct_evtlog.txt diff --git a/ctypes_generation/definitions/winstruct_file_info.txt b/ctypes_generation/definitions/structures/winstruct_file_info.txt similarity index 100% rename from ctypes_generation/definitions/winstruct_file_info.txt rename to ctypes_generation/definitions/structures/winstruct_file_info.txt diff --git a/ctypes_generation/generate.py b/ctypes_generation/generate.py index f438565..8a6ddf0 100644 --- a/ctypes_generation/generate.py +++ b/ctypes_generation/generate.py @@ -16,7 +16,6 @@ pexists = os.path.exists dedent = textwrap.dedent - TYPE_EQUIVALENCE = [ # BYTE is defined in ctypes.wintypes as c_byte but who wants # BYTE to be signed ? (from MSDN: ) @@ -74,6 +73,9 @@ TYPE_EQUIVALENCE = [ ('IF_INDEX', 'NET_IFINDEX'), ('IFTYPE', 'ULONG'), ('PULONG64', 'POINTER(ULONG64)'), + ('LPFILETIME', 'POINTER(FILETIME)'), + ('LPPOINT', 'POINTER(POINT)'), + ('LPRECT', 'POINTER(RECT)'), ('PBYTE', 'POINTER(BYTE)'), ('PUINT', 'POINTER(UINT)'), ('PHANDLE', 'POINTER(HANDLE)'), @@ -132,10 +134,10 @@ class CtypesGenerator(object): PARSER = None IMPORT_HEADER = "{deps}" - def __init__(self, infilename, outfilename, dependances=()): - self.infilename = infilename + def __init__(self, indirname, outfilename, dependances=()): + self.indirname = indirname self.outfilename = outfilename - self.infile = open(self.infilename) + # self.infile = open(self.infilename) self.data = None self.dependances = dependances @@ -146,11 +148,22 @@ class CtypesGenerator(object): self.analyse(self.data) self.check_dependances() + # def parse(self): + # if self.data is None: + # print("Parsing <{0}>".format(self.infilename)) + # self.data = self.PARSER(self.infile.read()).parse() + # return self.data + def parse(self): - if self.data is None: - print("Parsing <{0}>".format(self.infilename)) - self.data = self.PARSER(self.infile.read()).parse() - return self.data + if self.data is not None: + return self.data + data = [] + for filename in glob.glob(self.indirname): + print("Parsing <{0}>".format(filename)) + # data.append(self.PARSER(open(filename).read()).parse()) + data += self.PARSER(open(filename).read()).parse() + self.data = data + return data def analyse(self, data): raise NotImplementedError("<{0}> doest not implement ".format(type(self).__name__)) @@ -160,7 +173,8 @@ class CtypesGenerator(object): for dep in self.dependances: missing -= dep.exports if missing: - raise ValueError("Missing dependance <{0}> in <{1}>".format(missing, self.infilename)) + # raise ValueError("Missing dependance <{0}> in <{1}>".format(missing, self.infilename)) + print ValueError("Missing dependance <{0}>".format(missing)) def generate_import(self): deps = "\n".join(["from {0} import *".format(os.path.basename(dep.outfilename).rsplit(".")[0]) for dep in self.dependances]) @@ -356,6 +370,7 @@ class StructGenerator(CtypesGenerator): extended_struct_filename = from_here(os.path.join("extended_structs", "{0}.py".format(definition.name))) with open(extended_struct_filename) as f: ctypes_lines.append(f.read()) + # import pdb;pdb.set_trace() ctypes_lines.append(definition.generate_typedef_ctypes() + "\n") ctypes_code = "\n".join(ctypes_lines) @@ -364,6 +379,19 @@ class StructGenerator(CtypesGenerator): print("<{0}> generated".format(self.outfilename)) return ctypes_code + def parse(self): + if self.data is not None: + return self.data + data = [[], []] + for filename in glob.glob(self.indirname): + print("Parsing <{0}>".format(filename)) + # data.append(self.PARSER(open(filename).read()).parse()) + new_data = self.PARSER(open(filename).read()).parse() + data[0].extend(new_data[0]) + data[1].extend(new_data[1]) + self.data = data + return data + def append_input_file(self, filename): print("Adding file <{0}>".format(filename)) self.parse() @@ -478,6 +506,20 @@ class NtStatusGenerator(CtypesGenerator): return Flag(name, code) """) + def __init__(self, infilename, outfilename, dependances=()): + self.infilename = infilename + self.outfilename = outfilename + self.infile = open(self.infilename) + self.data = None + self.dependances = dependances + + self.exports = set([]) + self.imports = set([]) + + self.parse() + self.analyse(self.data) + self.check_dependances() + def parse_ntstatus(self, content): nt_status_defs = [] for line in content.split("\n"): @@ -498,6 +540,7 @@ class NtStatusGenerator(CtypesGenerator): self.parse_ntstatus(self.infile.read()) return self.data + def analyse(self, data): self.add_imports("Flag") @@ -668,7 +711,6 @@ class InitialCOMGenerator(CtypesGenerator): byreflevel = arg.byreflevel - 1 method.args[pos] = arg = type(arg)(atype, byreflevel, arg.name) self.real_type[arg] = initial_arg - self.add_imports(arg.type) com_interface_comment_template = """ #{0} -> {1}""" @@ -728,6 +770,17 @@ class InitialCOMGenerator(CtypesGenerator): return ", ".join(str_iid) + + com_interface_template = dedent(""" + class {0}(COMInterface): + IID = generate_IID({2}, name="{0}", strid="{3}") + + _functions_ = {{ + {1} + }} + """) + + class COMGenerator(InitialCOMGenerator): IMPORT_HEADER = "{deps}" HEADER = "" @@ -777,30 +830,43 @@ DEFAULT_INTERFACE_TO_IID = from_here("definitions\\interface_to_iid.txt") # A partial define without the dependance to ntstatus defintion # BOOTSTRAP!! -non_generated_def = InitialDefGenerator(from_here("definitions\\windef.txt"), from_here(r"..\windows\generated_def\\windef.py")) +non_generated_def = InitialDefGenerator(from_here("definitions\\defines\\windef.txt"), from_here(r"..\windows\generated_def\\windef.py")) ntstatus = NtStatusGenerator(from_here("definitions\\ntstatus.txt"), from_here(r"..\windows\generated_def\\ntstatus.py"), dependances=[non_generated_def]) # Not a real circular def (import not at the begin of file -defs_with_ntstatus = InitialDefGenerator(from_here("definitions\\windef.txt"), from_here(r"..\windows\generated_def\\windef.py"), dependances=[ntstatus]) +defs_with_ntstatus = InitialDefGenerator(from_here("definitions\\defines\\*.txt"), from_here(r"..\windows\generated_def\\windef.py"), dependances=[ntstatus]) # YOLO HACK FOR NOW :DD -defs_with_ntstatus.append_input_file(from_here("definitions\\wintrust_crypt_def.txt")) -defs_with_ntstatus.append_input_file(from_here("definitions\\windef_error.txt")) -defs_with_ntstatus.append_input_file(from_here("definitions\\custom_rpc_windef.txt")) -defs_with_ntstatus.append_input_file(from_here("definitions\\windef_evtlog.txt")) +# import pdb;pdb.set_trace() -structs = StructGenerator(from_here("definitions\\winstruct.txt"), from_here(r"..\windows\generated_def\\winstructs.py"), dependances=[defs_with_ntstatus]) -structs.append_input_file(from_here("definitions\\winstruct_apisetmap.txt")) -structs.append_input_file(from_here("definitions\\display_struct.txt")) -structs.append_input_file(from_here("definitions\\winstruct_bits.txt")) -structs.append_input_file(from_here("definitions\\winstruct_alpc.txt")) -structs.append_input_file(from_here("definitions\\winstruct_evtlog.txt")) -structs.append_input_file(from_here("definitions\\winstruct_file_info.txt")) +# for filename in [f for f in glob.glob(from_here("definitions\\defines\\*.txt")) if not f.endswith("\\windef.txt")]: + # defs_with_ntstatus.append_input_file(from_here("definitions\\wintrust_crypt_def.txt")) + # defs_with_ntstatus.append_input_file(from_here("definitions\\windef_error.txt")) + # defs_with_ntstatus.append_input_file(from_here("definitions\\custom_rpc_windef.txt")) + # defs_with_ntstatus.append_input_file(from_here("definitions\\windef_evtlog.txt")) + # defs_with_ntstatus.append_input_file(filename) -functions = FuncGenerator(from_here("definitions\\winfunc.txt"), from_here(r"..\windows\generated_def\\winfuncs.py"), dependances=[structs]) -functions.append_input_file(from_here("definitions\\winfunc_crypto_wintrust.txt")) -functions.append_input_file(from_here("definitions\\winfunc_notdoc.txt")) -functions.append_input_file(from_here("definitions\\winfunc_evtlog.txt")) + +structs = StructGenerator(from_here("definitions\\structures\\*.txt"), from_here(r"..\windows\generated_def\\winstructs.py"), dependances=[defs_with_ntstatus]) + +# for filename in [f for f in glob.glob(from_here("definitions\\structures\\*.txt")) if not f.endswith("\\winstruct.txt")]: + # structs.append_input_file(filename) + +# structs.append_input_file(from_here("definitions\\winstruct_apisetmap.txt")) +# structs.append_input_file(from_here("definitions\\display_struct.txt")) +# structs.append_input_file(from_here("definitions\\winstruct_bits.txt")) +# structs.append_input_file(from_here("definitions\\winstruct_alpc.txt")) +# structs.append_input_file(from_here("definitions\\winstruct_evtlog.txt")) +# structs.append_input_file(from_here("definitions\\winstruct_file_info.txt")) + +functions = FuncGenerator(from_here("definitions\\functions\\*.txt"), from_here(r"..\windows\generated_def\\winfuncs.py"), dependances=[structs]) + +# for filename in [f for f in glob.glob(from_here("definitions\\functions\\*.txt")) if not f.endswith("\\winfunc.txt")]: + # functions.append_input_file(filename) + +# functions.append_input_file(from_here("definitions\\winfunc_crypto_wintrust.txt")) +# functions.append_input_file(from_here("definitions\\winfunc_notdoc.txt")) +# functions.append_input_file(from_here("definitions\\winfunc_evtlog.txt")) com = InitialCOMGenerator(from_here("definitions\\com\\*.txt"), DEFAULT_INTERFACE_TO_IID, from_here(r"..\windows\generated_def\\interfaces.py"), dependances=[structs, defs_with_ntstatus]) @@ -848,4 +914,5 @@ if __name__ == "__main__": print("Generating documentation") ntstatus.generate_doc(from_here(r"..\docs\source\ntstatus_generated.rst")) defs_with_ntstatus.generate_doc(from_here(r"..\docs\source\windef_generated.rst")) - structs.generate_doc(from_here(r"..\docs\source\winstructs_generated.rst")) \ No newline at end of file + structs.generate_doc(from_here(r"..\docs\source\winstructs_generated.rst")) + diff --git a/ctypes_generation/newgen.py b/ctypes_generation/newgen.py new file mode 100644 index 0000000..d69cb56 --- /dev/null +++ b/ctypes_generation/newgen.py @@ -0,0 +1,418 @@ +import sys +import os +import os.path +import re +import glob +import textwrap +import StringIO + +import shutil + +import dummy_wintypes +import struct_parser +import func_parser +import def_parser +import com_parser + +pjoin = os.path.join +pexists = os.path.exists +dedent = textwrap.dedent + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +print(SCRIPT_DIR) +from_here = lambda path: pjoin(SCRIPT_DIR, path) + + + +class ParsedFile(object): + def __init__(self, filename): + self.filename = filename + self.data = self.PARSER(open(filename).read()).parse() + self.exports = set() + self.imports = set() + self.compute_imports_exports(self.data) + + def add_exports(self, *names): + self.exports.update(names) + + def add_imports(self, *names): + self.imports.update(names) + + def compute_imports_exports(self): + raise NotImplementedError("compute_imports_exports") + + def __repr__(self): + return '<{clsname} "{0}">'.format(self.filename, clsname=type(self).__name__) + +class StructureParsedFile(ParsedFile): + PARSER = struct_parser.WinStructParser + + def compute_imports_exports(self, data): + structs, enums = data + for enum in enums: + self.add_exports(enum.name) + self.add_exports(*enum.typedef) + for struct in structs: + self.add_exports(struct.name) + self.add_exports(*struct.typedef) + for field_type, field_name, nb_rep in struct.fields: + if field_type.name not in self.exports: + self.add_imports(field_type.name) + try: + int(nb_rep) + except: + self.add_imports(nb_rep) + +class SimpleTypeParsedFile(ParsedFile): + PARSER = struct_parser.SimpleTypesParser + + def compute_imports_exports(self, data): + for simple_type in data: + self.add_exports(simple_type.lvalue) # No dependancy check on rvalue for now + + +class DefinitionParsedFile(ParsedFile): + PARSER = def_parser.WinDefParser + + def compute_imports_exports(self, data): + for windef in data: + self.add_exports(windef.name) # No dependancy check on rvalue for now + +class NtStatusParsedFile(ParsedFile): + PARSER = def_parser.NtStatusParser + + def compute_imports_exports(self, data): + for ntstatus in data: + self.add_exports(ntstatus[1]) + +class FunctionParsedFile(ParsedFile): + PARSER = func_parser.WinFuncParser + + def compute_imports_exports(self, data): + for func in data: + if isinstance(func.return_type, tuple) and func.return_type[0] == "PTR": + self.add_imports(func.return_type[1]) + else: + self.add_imports(func.return_type) + for param_type, _ in func.params: + if param_type.startswith("POINTER(") and param_type.endswith(")"): + param_type = param_type[len("POINTER("): -1] + self.add_imports(param_type) + self.add_exports(func.name) + + +class COMParsedFile(ParsedFile): + PARSER = com_parser.WinComParser + + def compute_imports_exports(self, cominterface): + self.add_exports(cominterface.name) + if cominterface.typedefptr: + self.add_exports(cominterface.typedefptr) + + +class ParsedFileGraph(object): + def __init__(self, nodes, depnodes): # depnodes: nodes that we dont have to handle but want can take export from + self.nodes = nodes + self.depnodes = depnodes + self.exports_database = {} + self.depandances_database = {node: set() for node in nodes} + self.build_export_database(self.nodes) + self.build_depandance_database() + + def build_dependancy_graph(self): + todo = set(self.nodes) + start = self.find_starting_node() + print("Starting node is {0}".format(start)) + todo.remove(start) + flatten = [start] + depdone = set(flatten) | set(self.depnodes) + while todo: + for node in todo: + if self.depandances_database[node].issubset(depdone): + break + else: + raise ValueError("POUET") + + flatten.append(node) + depdone.add(node) + todo.remove(node) + print("Next is <{0}>".format(node)) + return flatten + + + def build_depandance_database(self): + for node in self.nodes: + for import_ in node.imports: + try: + self.depandances_database[node].add(self.exports_database[import_]) + except KeyError as e: + raise ValueError("Missing dependancy <{0}> of {1}".format(import_, node)) + + + def build_export_database(self, nodes): + for node in self.nodes + self.depnodes: + for export in node.exports: + if export in self.exports_database: + raise ValueError("{0} IN {1} but already exported by {2}".format(export, self.exports_database[export], node)) + self.exports_database[export] = node + + def find_starting_node(self): + for node in self.nodes: + if self.depandances_database[node].issubset(set(self.depnodes)): + return node + raise ValueError("Could not find a starting NODE without dependancy") + + +class BasicTypeNodes(object): + @property + def exports(self): + # Let allow ourself to redefine the bugged BYTE define & MAX_PATH which is NOT A TYPE ! + return set(dummy_wintypes.names) - set(["BYTE", "MAX_PATH"]) + +class FakeExporter(object): + def __init__(self, exports): + self.exports = exports + +class ParsedDirectory(object): + def __init__(self, filetype, directory): + self.nodes = [filetype(f) for f in glob.glob(directory)] + + +### Generation Class ### + +class CtypesGenerator(object): + def __init__(self, parsed_files, template): + self.files = parsed_files # Already in generation order + self.template = template # MAKE BETTER + self.result = StringIO.StringIO() + + def emit(self, str): + self.result.write(str) + + def emitline(self, str): + self.emit(str) + self.emit("\n") + + def before_emit_template(self): + pass + + def after_emit_template(self): + pass + + def copy_template(self): + with open(self.template) as f: + self.emit(f.read()) + + def generate(self): + self.before_emit_template() + self.copy_template() + self.after_emit_template() + + for file in self.files: + self.generate_for_file(file) + + def generate_for_file(self, file): + pass + +NTSTATUS_MODULE = "ntstatus" + +class DefineCtypesGenerator(CtypesGenerator): + def after_emit_template(self): + self.emitline("from {0} import *".format(NTSTATUS_MODULE)) + + def generate_for_file(self, file): + for define in file.data: + self.emitline(define.generate_ctypes()) + +class NtStatusCtypesGenerator(CtypesGenerator): + def generate_for_file(self, file): + for value, name, descr in file.data: + value = "{:#x}".format(value) + line = '{1} = NtStatusException.register_ntstatus({0}, "{1}", "{2}")'.format(value, name, descr) + self.emitline(line) + +class COMCtypesGenerator(CtypesGenerator): + IGNORED_INTERFACE = set(["ITypeInfo"]) + def __init__(self, *args, **kwargs): + super(COMCtypesGenerator, self).__init__(*args, **kwargs) + self.iids_def = {} + self.generated_interfaces_names = set(self.IGNORED_INTERFACE) + for file in self.files: + self.generated_interfaces_names.update(file.exports) + + + def parse_iid_file(self, filename): + data = open(filename).read() + for line in data.split("\n"): + name, iid = line.split("|") + self.iids_def[name] = self.parse_iid(iid), iid + + def before_emit_template(self): + self.emitline("from POUET import *") + self.emitline("") + + def generate_for_file(self, file): + define = [] + cominterface = file.data + return self.generate_com_interface(cominterface) + + def generate_com_interface(self, cominterface): + name = cominterface.name + if cominterface.iid is not None: + iid_str = cominterface.iid + iid_python = self.parse_iid(iid_str) + else: + print("Lookup of IID for <{0}>".format(cominterface.name)) + iid_python, iid_str = self.iids_def[cominterface.name] + + cls_format_param = {"name": name, "iid_python" : iid_python, "iid_str": iid_str} + + self.emitline("class {name} (COMInterface):".format(**cls_format_param)) + self.emitline(' IID = generate_IID({iid_python}, name="{name}", strid="{iid_str}")'.format(**cls_format_param)) + self.emitline(' _functions_ = {') + self.emit_com_interface_functions(cominterface) + self.emitline(' }') + self.emitline('') + + + def emit_com_interface_functions(self, cominterface): + indent = " " * 8 + for method_nb, method in enumerate(cominterface.methods): + args_to_define = method.args[1:] # ctypes doesnt not need the This + name = method.name + params = ", ".join([arg.name +":"+ ("*"* arg.byreflevel) +arg.type for arg in args_to_define]) + self.emitline(indent + "# {name} -> {params}".format(name=name, params=params)) + + str_args = [] + for arg in args_to_define: + if arg.type == "void" and arg.byreflevel > 0: + arg = type(arg)("PVOID", arg.byreflevel - 1, arg.name) + atype = arg.type + byreflevel = arg.byreflevel + if atype in self.generated_interfaces_names: + byreflevel = arg.byreflevel - 1 + atype = "PVOID" + + for i in range(byreflevel): + atype = "POINTER({0})".format(atype) + str_args.append(atype) + + # methods_string.append(self.com_interface_method_template.format(method.name, ", ".join([method.ret_type] + str_args), method_nb)) + params = ", ".join([method.ret_type] + str_args) + self.emitline(indent + '"{0}": ctypes.WINFUNCTYPE({1})({2}, "{0}"),'.format(name, params, method_nb)) + return + + + def parse_iid(self, iid_str): + part_iid = iid_str.split("-") + str_iid = [] + str_iid.append("0x" + part_iid[0]) + str_iid.append("0x" + part_iid[1]) + str_iid.append("0x" + part_iid[2]) + str_iid.append("0x" + part_iid[3][:2]) + str_iid.append("0x" + part_iid[3][2:]) + for i in range(6): str_iid.append("0x" + part_iid[4][i * 2:(i + 1) * 2]) + return ", ".join(str_iid) + +class FunctionCtypesGenerator(CtypesGenerator): + def __init__(self, parsed_files): + self.files = parsed_files # Already in generation order + self.result = StringIO.StringIO() + + def copy_template(self): + self.emitline("from ctypes import *") + self.emitline("from POUET import *") + # self.emitline("PPORT_MESSAGE = INT") + + def generate_for_file(self, file): + for item in file.data: + self.emitline(item.generate_ctypes()) + + + +EXTENDED_STRUCT_FILE = glob.glob(pjoin(SCRIPT_DIR, "extended_structs", "*.py")) +EXTENDED_STRUCT = [os.path.basename(filename)[:-len(".py")] for filename in EXTENDED_STRUCT_FILE] + +class StructureCtypesGenerator(CtypesGenerator): + def generate_for_simple_type_file(self, file): + for simple_type in file.data: + self.emitline(simple_type.generate_ctypes()) + + def generate_for_file(self, file): + if isinstance(file, SimpleTypeParsedFile): + return self.generate_for_simple_type_file(file) + structs, enums = file.data + for definition in [d for l in (enums, structs) for d in l]: + self.emitline(definition.generate_ctypes()) + if definition.name in EXTENDED_STRUCT: + print("Including extended definition for <{0}>".format(definition.name)) + extended_struct_filename = from_here(os.path.join("extended_structs", "{0}.py".format(definition.name))) + with open(extended_struct_filename) as f: + self.emitline(f.read()) + # RE-generate the typedef to apply them to the extended definition + self.emitline(definition.generate_typedef_ctypes()) + + + +stfilename = r"C:\Users\hakril\Documents\projets\PythonForWindows\ctypes_generation\definitions\simple_types.txt" +struct_parser.SimpleTypesParser(open(stfilename).read()).parse() + +ss = SimpleTypeParsedFile(stfilename) + +ds = ParsedDirectory(DefinitionParsedFile, from_here(r"definitions\defines\*.txt")) +fds = ParsedDirectory(FunctionParsedFile, from_here(r"definitions\functions\*.txt")) + +x = from_here("definitions\\structures\\*.txt") + +ntp = NtStatusParsedFile(from_here(r"definitions\ntstatus.txt")) + +sds = ParsedDirectory(StructureParsedFile, from_here(r"definitions\structures\*.txt")) + +scom = ParsedDirectory(COMParsedFile, from_here(r"definitions\COM\*.txt")) + +g = ParsedFileGraph(sds.nodes + [ss], depnodes=[BasicTypeNodes()] + ds.nodes) +snodes = g.build_dependancy_graph() + + +gg = ParsedFileGraph(fds.nodes, depnodes=[BasicTypeNodes()] + snodes) +fnodes = gg.build_dependancy_graph() + +## EMIT TEST CODE + + + +edef = DefineCtypesGenerator(ds.nodes, from_here(r"definitions\defines\template.py")) +edef.generate() + + +ents = NtStatusCtypesGenerator([ntp], from_here(r"definitions\ntstatus_template.py")) +ents.generate() + +snts = StructureCtypesGenerator(snodes, from_here(r"definitions\structures\template.py")) +snts.generate() + +fnts = FunctionCtypesGenerator(fnodes) +fnts.generate() + +cnts = COMCtypesGenerator(scom.nodes, from_here(r"definitions\com\template.py")) +cnts.parse_iid_file(from_here("definitions\\interface_to_iid.txt")) +cnts.generate() + +shutil.copy(from_here(r"definitions\flag.py"), "tmp\\") + + + +with open(r"tmp\yolo.py", "w") as f: + f.write(edef.result.getvalue()) + +with open(r"tmp\{0}.py".format(NTSTATUS_MODULE), "w") as f: + f.write(ents.result.getvalue()) + +with open(r"tmp\POUET.py", "w") as f: + f.write(snts.result.getvalue()) + +with open(r"tmp\FUNCS.py", "w") as f: + f.write(fnts.result.getvalue()) + +with open(r"tmp\COM.py", "w") as f: + f.write(cnts.result.getvalue()) \ No newline at end of file diff --git a/ctypes_generation/simpleparser.py b/ctypes_generation/simpleparser.py index cb19740..7139032 100644 --- a/ctypes_generation/simpleparser.py +++ b/ctypes_generation/simpleparser.py @@ -65,7 +65,9 @@ class SharpToken(NoValueToken): class EqualToken(NoValueToken): value = "=" - pass + +class NewLineToken(NoValueToken): + value = "\n" class Lexer(object): keywords = ["typedef", "struct", "enum", "union", "const"] @@ -74,8 +76,9 @@ class Lexer(object): "{" : OpenBracketToken, "}" : CloseBracketToken, ";" : ColonToken, "," : CommaToken, "(" : OpenParenthesisToken, ")" : CloseParenthesisToken, "#" : SharpToken, "=" : EqualToken} - def __init__(self, code): + def __init__(self, code, newlinetoken=False): self.code = code + self.newlinetoken = newlinetoken def split_line(self, line): return line.strip().split() @@ -106,6 +109,8 @@ class Lexer(object): continue for tok in self.split_word(word): yield tok + if self.newlinetoken: + yield NewLineToken() @@ -113,7 +118,6 @@ class ParsingError(Exception): pass class Parser(object): - def __init__(self, data): self.lexer = iter(Lexer(self.initial_processing(data))) self.peek_token = None diff --git a/ctypes_generation/struct_parser.py b/ctypes_generation/struct_parser.py index eb7a197..719533a 100644 --- a/ctypes_generation/struct_parser.py +++ b/ctypes_generation/struct_parser.py @@ -143,6 +143,32 @@ class WinStructParser(Parser): raise ValueError("Unknow returned type {0}".format(x)) return strucs, enums +class SimpleTypeDefine(object): + def __init__(self, lvalue, rvalue): + self.lvalue = lvalue + self.rvalue = rvalue + + def generate_ctypes(self): + return "{self.lvalue} = {self.rvalue}".format(self=self) + +class SimpleTypesParser(Parser): + def __init__(self, data): + self.lexer = iter(Lexer(self.initial_processing(data), newlinetoken=True)) + self.peek_token = None + + def parse(self): + results = [] + while self.peek() is not None: + lvalue = self.assert_token_type(NameToken).value + self.assert_token_type(EqualToken) + rvalue = "" + while type(self.peek()) is not NewLineToken: + rvalue += self.next_token().value + results.append(SimpleTypeDefine(lvalue, rvalue)) + while type(self.peek()) is NewLineToken: # discard the NewLineToken(s) + self.next_token() + return results + def dbg_lexer(data): for i in Lexer(data).token_generation(): print i