diff --git a/extapi.hpp b/extapi.hpp index c9a7c4c..811b8f8 100644 --- a/extapi.hpp +++ b/extapi.hpp @@ -6,9 +6,11 @@ #ifdef Py_LIMITED_API typedef void *Py_tracefunc; struct PyCompilerFlags; +# if PY_MAJOR_VERSION < 3 || PY_MINOR_VERSION < 9 struct PyFrameObject; +# endif #else -#include +# include #endif typedef void PyEval_SetTrace_t(Py_tracefunc, PyObject *); diff --git a/idapyswitch.cpp b/idapyswitch.cpp index 24bd8d7..672760a 100644 --- a/idapyswitch.cpp +++ b/idapyswitch.cpp @@ -111,7 +111,7 @@ AS_PRINTF(1, 2) int out_verb(const char *format, ...) } //------------------------------------------------------------------------- -AS_PRINTF(2, 3) void error(int exit_code, const char *format, ...) +NORETURN AS_PRINTF(2, 3) void error(int exit_code, const char *format, ...) { va_list va; va_start(va, format); @@ -214,6 +214,9 @@ typedef qvector pylib_entry_vec_t; struct pylib_entries_t { pylib_entry_vec_t entries; +#ifdef __MAC__ + qstrvec_t path_history; +#endif pylib_entry_t *get_entry_for_version(const pylib_version_t &version) { @@ -273,23 +276,6 @@ struct pyver_tool_t { do_find_python_libs(result); - // Remove Python 3.9 entries - { - auto p = result->entries.begin(); - while ( p != result->entries.end() ) - { - const pylib_entry_t &e = *p; - if ( e.version.major == 3 && e.version.minor >= 9 ) - { - out("Ignoring unusable Python 3.9.x \"%s\"\n", - !e.paths.empty() ? e.paths[0].c_str() : "?"); - p = result->entries.erase(p); - } - else - ++p; - } - } - #ifdef __UNIX__ set_preferred_pylib_version(result); #endif @@ -340,13 +326,8 @@ bool pyver_tool_t::do_pick_sip( const pylib_entry_t &entry, qstring *errbuf) const { - if ( entry.version.minor >= 9 ) - { - errbuf->sprnt("IDAPython 7.5 is incompatible with Python 3.9.x"); - return false; - } - - const char *src_sip_subdir = entry.version.minor >= 8 ? "python_3.8" + const char *src_sip_subdir = entry.version.minor >= 9 ? "python_3.9" + : entry.version.minor >= 8 ? "python_3.8" : "python_3.4"; char src_sip_path[QMAXPATH]; qmakepath(src_sip_path, sizeof(src_sip_path), idadir(""), diff --git a/idapyswitch_mac.cpp b/idapyswitch_mac.cpp index c05043e..8ecd01e 100644 --- a/idapyswitch_mac.cpp +++ b/idapyswitch_mac.cpp @@ -122,22 +122,39 @@ static int extract_pylib_bin(pylib_entries_t *result, const char *version_dir) { pylib_entries_t *result; pylib_finder_t(pylib_entries_t *_result) : result(_result) {} - virtual int visit_file(const char *bin) override + virtual int visit_file(const char *_binpath) override { + // macOS is absurdly dependent on symlinks. remove them to limit noise. + char buf[PATH_MAX]; + const char *binpath = realpath(_binpath, buf); + if ( binpath == nullptr ) + { + out_verb("Skipping %s: realpath() failed: %s\n", _binpath, winerr(errno)); + return 0; + } + if ( result->path_history.find(binpath) != result->path_history.end() ) + { + out_verb("Skipping %s: duplicate of %s\n", _binpath, binpath); + return 0; + } + + result->path_history.push_back(binpath); + qstring errbuf; pylib_version_t dummy; pylib_entry_t entry(dummy); - if ( get_pylib_entry_for_macho(&entry, bin, &errbuf) ) + if ( !get_pylib_entry_for_macho(&entry, binpath, &errbuf) ) { - qstring verbuf; - out_verb("Found: \"%s\" (version: %s)\n", bin, entry.version.str(&verbuf)); - result->entries.add_unique(entry); - return 1; + out_verb("Skipping %s: %s\n", binpath, errbuf.c_str()); + return 0; } - out_verb("Skipping %s: %s\n", bin, errbuf.c_str()); - return 0; + qstring verbuf; + out_verb("Found: \"%s\" (version: %s)\n", binpath, entry.version.str(&verbuf)); + result->entries.add_unique(entry); + + return 1; } }; @@ -170,17 +187,35 @@ static void extract_pylib_versions(pylib_entries_t *result, const char *framewor //------------------------------------------------------------------------- void pyver_tool_t::do_find_python_libs(pylib_entries_t *result) const { - static const char *framework_dirs[] = + // find all instances of Python.framework on the system + static const char *system_fwks[] = { "/Library/Frameworks", "/System/Library/Frameworks", "/Library/Developer/CommandLineTools/Library/Frameworks", "/Applications/Xcode.app/Contents/Developer/Library/Frameworks", - "/usr/local/opt/python@3/Frameworks", - "/usr/local/opt/python@2/Frameworks", "/opt/local/Library/Frameworks", }; + qstrvec_t framework_dirs; + for ( size_t i = 0; i < qnumber(system_fwks); i++ ) + framework_dirs.push_back(system_fwks[i]); + + struct ida_local homebrew_handler_t : public file_visitor_t + { + qstrvec_t *framework_dirs; + homebrew_handler_t(qstrvec_t *_framework_dirs) : framework_dirs(_framework_dirs) {} + virtual int visit_file(const char *path) override + { + framework_dirs->push_back(qstring(path) + "/Frameworks"); + return 0; + } + }; + + // homebrew keeps python installations in /usr/local/opt/python@X.X/Frameworks + homebrew_handler_t hh(&framework_dirs); + visit_files(hh, "/usr/local/opt", "python*", FA_DIREC); + struct ida_local python_framework_finder_t : public file_visitor_t { pylib_entries_t *result; @@ -192,9 +227,10 @@ void pyver_tool_t::do_find_python_libs(pylib_entries_t *result) const } }; + // check for a PythonX.framework in each framework dir python_framework_finder_t pff(result); - for ( size_t i = 0; i < qnumber(framework_dirs); ++i ) - visit_files(pff, framework_dirs[i], "Python*.framework", FA_DIREC); + for ( size_t i = 0, n = framework_dirs.size(); i < n; i++ ) + visit_files(pff, framework_dirs[i].c_str(), "Python*.framework", FA_DIREC); } //------------------------------------------------------------------------- diff --git a/idapython.cpp b/idapython.cpp index f98439d..1a6b808 100644 --- a/idapython.cpp +++ b/idapython.cpp @@ -441,9 +441,13 @@ static void handle_python_error( } //------------------------------------------------------------------------- -static const char *bomify(qstring *out) +static const char *insert_coding_cookie(qstring *out) { - out->insert(0, UTF8_BOM, UTF8_BOM_SZ); + // This is necessary for pre-3.9 parsers, to parse the + // input text as proper UTF-8. Python 3.9 switches to the PEG + // parser that, by default (and in particular since we don't + // pass PyCompilerFlags), will always assume UTF-8. + out->insert(0, "# -*- coding: UTF-8 -*-\n"); return out->c_str(); } @@ -1305,7 +1309,7 @@ bool idapython_plugin_t::_extlang_compile_expr( bool isfunc = false; qstring qstr(expr); - PyObject *code = Py_CompileString(bomify(&qstr), "", Py_eval_input); + PyObject *code = Py_CompileString(insert_coding_cookie(&qstr), "", Py_eval_input); if ( code == NULL ) { // try compiling as a list of statements @@ -1313,7 +1317,7 @@ bool idapython_plugin_t::_extlang_compile_expr( handle_python_error(errbuf); qstring func; wrap_in_function(&func, expr, name); - bomify(&func); + insert_coding_cookie(&func); code = Py_CompileString(func.c_str(), "", Py_file_input); if ( code == NULL ) { @@ -1872,7 +1876,7 @@ bool idapython_plugin_t::_cli_execute_line(const char *line) // Compile as an expression qstring qstr(line); - newref_t py_code(Py_CompileString(bomify(&qstr), "", Py_eval_input)); + newref_t py_code(Py_CompileString(insert_coding_cookie(&qstr), "", Py_eval_input)); if ( py_code == NULL || PyErr_Occurred() ) { // Not an expression? diff --git a/makefile b/makefile index 4701403..f039fa5 100644 --- a/makefile +++ b/makefile @@ -396,7 +396,7 @@ ifeq ($(OUT_OF_TREE_BUILD),) $(Q)$(CP) $? $@ DEST_SIP += $(DEST_SIP34_PYDLL) $(DEST_SIP34_PYI) - # sip for Python >= 3.8 + # sip for Python [3.8, 3.9) DEST_SIP38_DIR:=$(DEST_PYQT_DIR)/python_3.8 $(DEST_SIP38_DIR): -$(Q)if [ ! -d "$(DEST_SIP38_DIR)" ] ; then mkdir -p 2>/dev/null $(DEST_SIP38_DIR) ; fi @@ -407,6 +407,19 @@ ifeq ($(OUT_OF_TREE_BUILD),) $(DEST_SIP38_PYI): $(wildcard $(SIP38_TREE)/lib/python*/PyQt5/$(SIP_PYI_FNAME)) | $(DEST_SIP38_DIR) $(Q)$(CP) $? $@ DEST_SIP += $(DEST_SIP38_PYDLL) $(DEST_SIP38_PYI) + + # sip for Python [3.9, ... + DEST_SIP39_DIR:=$(DEST_PYQT_DIR)/python_3.9 + $(DEST_SIP39_DIR): + -$(Q)if [ ! -d "$(DEST_SIP39_DIR)" ] ; then mkdir -p 2>/dev/null $(DEST_SIP39_DIR) ; fi + DEST_SIP39_PYDLL:=$(DEST_SIP39_DIR)/$(SIP_PYDLL_FNAME) + DEST_SIP39_PYI:=$(DEST_SIP39_DIR)/$(SIP_PYI_FNAME) + $(DEST_SIP39_PYDLL): $(wildcard $(SIP39_TREE)/lib/python*/PyQt5/$(SIP_PYDLL_FNAME)) | $(DEST_SIP39_DIR) + $(Q)$(CP) $? $@ + $(DEST_SIP39_PYI): $(wildcard $(SIP39_TREE)/lib/python*/PyQt5/$(SIP_PYI_FNAME)) | $(DEST_SIP39_DIR) + $(Q)$(CP) $? $@ + DEST_SIP += $(DEST_SIP39_PYDLL) $(DEST_SIP39_PYI) + else # sip for Python 2.7 DEST_SIP27_DIR:=$(DEST_PYQT_DIR) @@ -421,10 +434,14 @@ ifeq ($(OUT_OF_TREE_BUILD),) # And pick the right sip.so now (Python3 only; for Python2, we already put it in the right place) ifeq ($(PYTHON_VERSION_MAJOR),3) - ifeq ($(shell test $(PYTHON_VERSION_MINOR) -gt 7; echo $$?),0) # ugh - DEST_INSTALL_SIP_PYDLL:=$(DEST_SIP38_PYDLL) + ifeq ($(shell test $(PYTHON_VERSION_MINOR) -gt 8; echo $$?),0) # ugh + DEST_INSTALL_SIP_PYDLL:=$(DEST_SIP39_PYDLL) else - DEST_INSTALL_SIP_PYDLL:=$(DEST_SIP34_PYDLL) + ifeq ($(shell test $(PYTHON_VERSION_MINOR) -gt 7; echo $$?),0) # ugh + DEST_INSTALL_SIP_PYDLL:=$(DEST_SIP38_PYDLL) + else + DEST_INSTALL_SIP_PYDLL:=$(DEST_SIP34_PYDLL) + endif endif $(DEST_PYQT_DIR)/$(SIP_PYDLL_FNAME): $(DEST_INSTALL_SIP_PYDLL) $(Q)$(CP) $? $@ diff --git a/out_of_tree/parsed_notifications.zip b/out_of_tree/parsed_notifications.zip index 592689e..89c0e02 100644 Binary files a/out_of_tree/parsed_notifications.zip and b/out_of_tree/parsed_notifications.zip differ diff --git a/python/idautils.py b/python/idautils.py index 0c84eda..4f456e3 100644 --- a/python/idautils.py +++ b/python/idautils.py @@ -553,6 +553,7 @@ class Strings(object): t.minlen = minlen t.only_7bit = only_7bit t.display_only_existing_strings = display_only_existing_strings + t.ignore_heads = ignore_instructions self.refresh() diff --git a/release_api_contents2.txt b/release_api_contents2.txt index 116b046..54ad1c5 100644 --- a/release_api_contents2.txt +++ b/release_api_contents2.txt @@ -2252,6 +2252,8 @@ 'custom_data_type_ids_fids_array___getitem__', 'custom_data_type_ids_fids_array___len__', 'custom_data_type_ids_fids_array___setitem__', + 'custom_data_type_ids_fids_array__get_bytes', + 'custom_data_type_ids_fids_array__set_bytes', 'custom_data_type_ids_fids_array_data_get', 'custom_data_type_ids_t___getFids', 'custom_data_type_ids_t_dtid_get', @@ -3330,6 +3332,8 @@ 'fnum_array___getitem__', 'fnum_array___len__', 'fnum_array___setitem__', + 'fnum_array__get_bytes', + 'fnum_array__set_bytes', 'fnum_array_data_get', 'fnumber_t___eq__', 'fnumber_t___ge__', @@ -7640,6 +7644,8 @@ 'operands_array___getitem__', 'operands_array___len__', 'operands_array___setitem__', + 'operands_array__get_bytes', + 'operands_array__set_bytes', 'operands_array_data_get', 'operator_info_t_fixtype_get', 'operator_info_t_fixtype_set', @@ -9489,6 +9495,8 @@ 'strpath_ids_array___getitem__', 'strpath_ids_array___len__', 'strpath_ids_array___setitem__', + 'strpath_ids_array__get_bytes', + 'strpath_ids_array__set_bytes', 'strpath_ids_array_data_get', 'strpath_t___getIds', 'strpath_t_delta_get', diff --git a/release_api_contents3.txt b/release_api_contents3.txt index c6c27e2..695866b 100644 --- a/release_api_contents3.txt +++ b/release_api_contents3.txt @@ -2252,6 +2252,8 @@ 'custom_data_type_ids_fids_array___getitem__', 'custom_data_type_ids_fids_array___len__', 'custom_data_type_ids_fids_array___setitem__', + 'custom_data_type_ids_fids_array__get_bytes', + 'custom_data_type_ids_fids_array__set_bytes', 'custom_data_type_ids_fids_array_data_get', 'custom_data_type_ids_t___getFids', 'custom_data_type_ids_t_dtid_get', @@ -3330,6 +3332,8 @@ 'fnum_array___getitem__', 'fnum_array___len__', 'fnum_array___setitem__', + 'fnum_array__get_bytes', + 'fnum_array__set_bytes', 'fnum_array_data_get', 'fnumber_t___eq__', 'fnumber_t___ge__', @@ -7644,6 +7648,8 @@ 'operands_array___getitem__', 'operands_array___len__', 'operands_array___setitem__', + 'operands_array__get_bytes', + 'operands_array__set_bytes', 'operands_array_data_get', 'operator_info_t_fixtype_get', 'operator_info_t_fixtype_set', @@ -9493,6 +9499,8 @@ 'strpath_ids_array___getitem__', 'strpath_ids_array___len__', 'strpath_ids_array___setitem__', + 'strpath_ids_array__get_bytes', + 'strpath_ids_array__set_bytes', 'strpath_ids_array_data_get', 'strpath_t___getIds', 'strpath_t_delta_get', diff --git a/release_pydoc_injections2.txt b/release_pydoc_injections2.txt index 9b0c355..854d5ae 100644 --- a/release_pydoc_injections2.txt +++ b/release_pydoc_injections2.txt @@ -23848,6 +23848,13 @@ class fnum_array(__builtin__.object) | i: size_t | v: unsigned short const & | + | _get_bytes(self, *args) + | _get_bytes(self) -> bytevec_t + | + | _set_bytes(self, *args) + | _set_bytes(self, bts) + | bts: bytevec_t const & + | | ---------------------------------------------------------------------- | Data descriptors defined here: | @@ -23857,6 +23864,9 @@ class fnum_array(__builtin__.object) | __weakref__ | list of weak references to the object (if defined) | + | bytes + | _get_bytes(self) -> bytevec_t + | | data | fnum_array_data_get(self) -> unsigned short (&)[6] | @@ -61835,6 +61845,13 @@ class custom_data_type_ids_fids_array(__builtin__.object) | i: size_t | v: short const & | + | _get_bytes(self, *args) + | _get_bytes(self) -> bytevec_t + | + | _set_bytes(self, *args) + | _set_bytes(self, bts) + | bts: bytevec_t const & + | | ---------------------------------------------------------------------- | Data descriptors defined here: | @@ -61844,6 +61861,9 @@ class custom_data_type_ids_fids_array(__builtin__.object) | __weakref__ | list of weak references to the object (if defined) | + | bytes + | _get_bytes(self) -> bytevec_t + | | data | custom_data_type_ids_fids_array_data_get(self) -> short (&)[8] | @@ -63095,6 +63115,13 @@ class strpath_ids_array(__builtin__.object) | i: size_t | v: unsigned-ea-like-numeric-type const &↗ | + | _get_bytes(self, *args) + | _get_bytes(self) -> bytevec_t + | + | _set_bytes(self, *args) + | _set_bytes(self, bts) + | bts: bytevec_t const & + | | ---------------------------------------------------------------------- | Data descriptors defined here: | @@ -63104,6 +63131,9 @@ class strpath_ids_array(__builtin__.object) | __weakref__ | list of weak references to the object (if defined) | + | bytes + | _get_bytes(self) -> bytevec_t + | | data | strpath_ids_array_data_get(self) -> unsigned-ea-like-numeric-type (&)[32]↗ | @@ -82109,6 +82139,13 @@ class operands_array(__builtin__.object) | i: size_t | v: op_t const & | + | _get_bytes(self, *args) + | _get_bytes(self) -> bytevec_t + | + | _set_bytes(self, *args) + | _set_bytes(self, bts) + | bts: bytevec_t const & + | | ---------------------------------------------------------------------- | Data descriptors defined here: | @@ -82118,6 +82155,9 @@ class operands_array(__builtin__.object) | __weakref__ | list of weak references to the object (if defined) | + | bytes + | _get_bytes(self) -> bytevec_t + | | data | operands_array_data_get(self) -> op_t (&)[8] | diff --git a/release_pydoc_injections3.txt b/release_pydoc_injections3.txt index 49b1bd4..f4453e9 100644 --- a/release_pydoc_injections3.txt +++ b/release_pydoc_injections3.txt @@ -23897,6 +23897,13 @@ class fnum_array(builtins.object) | __swig_destroy__ = delete_fnum_array(...) | delete_fnum_array(self) | + | _get_bytes(self, *args) -> 'bytevec_t' + | _get_bytes(self) -> bytevec_t + | + | _set_bytes(self, *args) -> 'void' + | _set_bytes(self, bts) + | bts: bytevec_t const & + | | ---------------------------------------------------------------------- | Data descriptors defined here: | @@ -23906,6 +23913,9 @@ class fnum_array(builtins.object) | __weakref__ | list of weak references to the object (if defined) | + | bytes + | _get_bytes(self) -> bytevec_t + | | data | fnum_array_data_get(self) -> unsigned short (&)[6] | @@ -62518,6 +62528,13 @@ class custom_data_type_ids_fids_array(builtins.object) | __swig_destroy__ = delete_custom_data_type_ids_fids_array(...) | delete_custom_data_type_ids_fids_array(self) | + | _get_bytes(self, *args) -> 'bytevec_t' + | _get_bytes(self) -> bytevec_t + | + | _set_bytes(self, *args) -> 'void' + | _set_bytes(self, bts) + | bts: bytevec_t const & + | | ---------------------------------------------------------------------- | Data descriptors defined here: | @@ -62527,6 +62544,9 @@ class custom_data_type_ids_fids_array(builtins.object) | __weakref__ | list of weak references to the object (if defined) | + | bytes + | _get_bytes(self) -> bytevec_t + | | data | custom_data_type_ids_fids_array_data_get(self) -> short (&)[8] | @@ -63760,6 +63780,13 @@ class strpath_ids_array(builtins.object) | __swig_destroy__ = delete_strpath_ids_array(...) | delete_strpath_ids_array(self) | + | _get_bytes(self, *args) -> 'bytevec_t' + | _get_bytes(self) -> bytevec_t + | + | _set_bytes(self, *args) -> 'void' + | _set_bytes(self, bts) + | bts: bytevec_t const & + | | ---------------------------------------------------------------------- | Data descriptors defined here: | @@ -63769,6 +63796,9 @@ class strpath_ids_array(builtins.object) | __weakref__ | list of weak references to the object (if defined) | + | bytes + | _get_bytes(self) -> bytevec_t + | | data | strpath_ids_array_data_get(self) -> unsigned-ea-like-numeric-type (&)[32]↗ | @@ -82707,6 +82737,13 @@ class operands_array(builtins.object) | __swig_destroy__ = delete_operands_array(...) | delete_operands_array(self) | + | _get_bytes(self, *args) -> 'bytevec_t' + | _get_bytes(self) -> bytevec_t + | + | _set_bytes(self, *args) -> 'void' + | _set_bytes(self, bts) + | bts: bytevec_t const & + | | ---------------------------------------------------------------------- | Data descriptors defined here: | @@ -82716,6 +82753,9 @@ class operands_array(builtins.object) | __weakref__ | list of weak references to the object (if defined) | + | bytes + | _get_bytes(self) -> bytevec_t + | | data | operands_array_data_get(self) -> op_t (&)[8] | diff --git a/swig/hexrays.i b/swig/hexrays.i index 448f777..35010e6 100644 --- a/swig/hexrays.i +++ b/swig/hexrays.i @@ -55,6 +55,16 @@ SWIGINTERN void __raise_vdf(const vd_failure_t &e) } %enddef +%define %method_sets_type_and_gains_ownership_of_regular_object_argument(TYPE, METHOD) +%feature("pythonprepend") TYPE::METHOD %{ + o = args[0] + self._ensure_cond(self.t == mop_z, "self.t == mop_z") +%} +%feature("pythonappend") TYPE::METHOD %{ + self._acquire_ownership(o, True) +%} +%enddef + %typemap(directorin) (const char *format, ...) { // %typemap(directorin) (const char *format, ...) @@ -150,6 +160,10 @@ SWIGINTERN void __raise_vdf(const vd_failure_t &e) %define_hexrays_lifecycle_object(mop_t); %ignore mop_t::_make_strlit(qstring *); %template(mopvec_t) qvector; +%method_sets_type_and_gains_ownership_of_regular_object_argument(mop_t, _make_cases) +%method_sets_type_and_gains_ownership_of_regular_object_argument(mop_t, _make_callinfo) +%method_sets_type_and_gains_ownership_of_regular_object_argument(mop_t, _make_pair) +%method_sets_type_and_gains_ownership_of_regular_object_argument(mop_t, _make_insn) %template(mcallargs_t) qvector;