Backported fixes

extapi.hpp

@@ -6,9 +6,11 @@
 #ifdef Py_LIMITED_API
 typedef void *Py_tracefunc;
 struct PyCompilerFlags;
+#  if PY_MAJOR_VERSION < 3 || PY_MINOR_VERSION < 9
 struct PyFrameObject;
+#  endif
 #else
-#include <frameobject.h>
+#  include <frameobject.h>
 #endif
 
 typedef void PyEval_SetTrace_t(Py_tracefunc, PyObject *);

idapyswitch.cpp

@@ -111,7 +111,7 @@ AS_PRINTF(1, 2) int out_verb(const char *format, ...)
 }
 
 //-------------------------------------------------------------------------
-AS_PRINTF(2, 3) void error(int exit_code, const char *format, ...)
+NORETURN AS_PRINTF(2, 3) void error(int exit_code, const char *format, ...)
 {
   va_list va;
   va_start(va, format);
@@ -214,6 +214,9 @@ typedef qvector<pylib_entry_t> pylib_entry_vec_t;
 struct pylib_entries_t
 {
   pylib_entry_vec_t entries;
+#ifdef __MAC__
+  qstrvec_t path_history;
+#endif
 
   pylib_entry_t *get_entry_for_version(const pylib_version_t &version)
   {
@@ -273,23 +276,6 @@ struct pyver_tool_t
   {
     do_find_python_libs(result);
 
-    // Remove Python 3.9 entries
-    {
-      auto p = result->entries.begin();
-      while ( p != result->entries.end() )
-      {
-        const pylib_entry_t &e = *p;
-        if ( e.version.major == 3 && e.version.minor >= 9 )
-        {
-          out("Ignoring unusable Python 3.9.x \"%s\"\n",
-              !e.paths.empty() ? e.paths[0].c_str() : "?");
-          p = result->entries.erase(p);
-        }
-        else
-          ++p;
-      }
-    }
-
 #ifdef __UNIX__
     set_preferred_pylib_version(result);
 #endif
@@ -340,13 +326,8 @@ bool pyver_tool_t::do_pick_sip(
         const pylib_entry_t &entry,
         qstring *errbuf) const
 {
-  if ( entry.version.minor >= 9 )
-  {
-    errbuf->sprnt("IDAPython 7.5 is incompatible with Python 3.9.x");
-    return false;
-  }
-
-  const char *src_sip_subdir = entry.version.minor >= 8 ? "python_3.8"
+  const char *src_sip_subdir = entry.version.minor >= 9 ? "python_3.9"
+                             : entry.version.minor >= 8 ? "python_3.8"
                              : "python_3.4";
   char src_sip_path[QMAXPATH];
   qmakepath(src_sip_path, sizeof(src_sip_path), idadir(""),

idapyswitch_mac.cpp

@@ -122,22 +122,39 @@ static int extract_pylib_bin(pylib_entries_t *result, const char *version_dir)
   {
     pylib_entries_t *result;
     pylib_finder_t(pylib_entries_t *_result) : result(_result) {}
-    virtual int visit_file(const char *bin) override
+    virtual int visit_file(const char *_binpath) override
     {
+      // macOS is absurdly dependent on symlinks. remove them to limit noise.
+      char buf[PATH_MAX];
+      const char *binpath = realpath(_binpath, buf);
+      if ( binpath == nullptr )
+      {
+        out_verb("Skipping %s: realpath() failed: %s\n", _binpath, winerr(errno));
+        return 0;
+      }
+      if ( result->path_history.find(binpath) != result->path_history.end() )
+      {
+        out_verb("Skipping %s: duplicate of %s\n", _binpath, binpath);
+        return 0;
+      }
+
+      result->path_history.push_back(binpath);
+
       qstring errbuf;
       pylib_version_t dummy;
       pylib_entry_t entry(dummy);
 
-      if ( get_pylib_entry_for_macho(&entry, bin, &errbuf) )
+      if ( !get_pylib_entry_for_macho(&entry, binpath, &errbuf) )
       {
-        qstring verbuf;
-        out_verb("Found: \"%s\" (version: %s)\n", bin, entry.version.str(&verbuf));
-        result->entries.add_unique(entry);
-        return 1;
+        out_verb("Skipping %s: %s\n", binpath, errbuf.c_str());
+        return 0;
       }
 
-      out_verb("Skipping %s: %s\n", bin, errbuf.c_str());
-      return 0;
+      qstring verbuf;
+      out_verb("Found: \"%s\" (version: %s)\n", binpath, entry.version.str(&verbuf));
+      result->entries.add_unique(entry);
+
+      return 1;
     }
   };
 
@@ -170,17 +187,35 @@ static void extract_pylib_versions(pylib_entries_t *result, const char *framewor
 //-------------------------------------------------------------------------
 void pyver_tool_t::do_find_python_libs(pylib_entries_t *result) const
 {
-  static const char *framework_dirs[] =
+  // find all instances of Python.framework on the system
+  static const char *system_fwks[] =
   {
     "/Library/Frameworks",
     "/System/Library/Frameworks",
     "/Library/Developer/CommandLineTools/Library/Frameworks",
     "/Applications/Xcode.app/Contents/Developer/Library/Frameworks",
-    "/usr/local/opt/python@3/Frameworks",
-    "/usr/local/opt/python@2/Frameworks",
     "/opt/local/Library/Frameworks",
   };
 
+  qstrvec_t framework_dirs;
+  for ( size_t i = 0; i < qnumber(system_fwks); i++ )
+    framework_dirs.push_back(system_fwks[i]);
+
+  struct ida_local homebrew_handler_t : public file_visitor_t
+  {
+    qstrvec_t *framework_dirs;
+    homebrew_handler_t(qstrvec_t *_framework_dirs) : framework_dirs(_framework_dirs) {}
+    virtual int visit_file(const char *path) override
+    {
+      framework_dirs->push_back(qstring(path) + "/Frameworks");
+      return 0;
+    }
+  };
+
+  // homebrew keeps python installations in /usr/local/opt/python@X.X/Frameworks
+  homebrew_handler_t hh(&framework_dirs);
+  visit_files(hh, "/usr/local/opt", "python*", FA_DIREC);
+
   struct ida_local python_framework_finder_t : public file_visitor_t
   {
     pylib_entries_t *result;
@@ -192,9 +227,10 @@ void pyver_tool_t::do_find_python_libs(pylib_entries_t *result) const
     }
   };
 
+  // check for a PythonX.framework in each framework dir
   python_framework_finder_t pff(result);
-  for ( size_t i = 0; i < qnumber(framework_dirs); ++i )
-    visit_files(pff, framework_dirs[i], "Python*.framework", FA_DIREC);
+  for ( size_t i = 0, n = framework_dirs.size(); i < n; i++ )
+    visit_files(pff, framework_dirs[i].c_str(), "Python*.framework", FA_DIREC);
 }
 
 //-------------------------------------------------------------------------

idapython.cpp

@@ -441,9 +441,13 @@ static void handle_python_error(
 }
 
 //-------------------------------------------------------------------------
-static const char *bomify(qstring *out)
+static const char *insert_coding_cookie(qstring *out)
 {
-  out->insert(0, UTF8_BOM, UTF8_BOM_SZ);
+  // This is necessary for pre-3.9 parsers, to parse the
+  // input text as proper UTF-8. Python 3.9 switches to the PEG
+  // parser that, by default (and in particular since we don't
+  // pass PyCompilerFlags), will always assume UTF-8.
+  out->insert(0, "# -*- coding: UTF-8 -*-\n");
   return out->c_str();
 }
 
@@ -1305,7 +1309,7 @@ bool idapython_plugin_t::_extlang_compile_expr(
   bool isfunc = false;
 
   qstring qstr(expr);
-  PyObject *code = Py_CompileString(bomify(&qstr), "<string>", Py_eval_input);
+  PyObject *code = Py_CompileString(insert_coding_cookie(&qstr), "<string>", Py_eval_input);
   if ( code == NULL )
   {
     // try compiling as a list of statements
@@ -1313,7 +1317,7 @@ bool idapython_plugin_t::_extlang_compile_expr(
     handle_python_error(errbuf);
     qstring func;
     wrap_in_function(&func, expr, name);
-    bomify(&func);
+    insert_coding_cookie(&func);
     code = Py_CompileString(func.c_str(), "<string>", Py_file_input);
     if ( code == NULL )
     {
@@ -1872,7 +1876,7 @@ bool idapython_plugin_t::_cli_execute_line(const char *line)
 
     // Compile as an expression
     qstring qstr(line);
-    newref_t py_code(Py_CompileString(bomify(&qstr), "<string>", Py_eval_input));
+    newref_t py_code(Py_CompileString(insert_coding_cookie(&qstr), "<string>", Py_eval_input));
     if ( py_code == NULL || PyErr_Occurred() )
     {
       // Not an expression?

makefile

@@ -396,7 +396,7 @@ ifeq ($(OUT_OF_TREE_BUILD),)
 	$(Q)$(CP) $? $@
     DEST_SIP += $(DEST_SIP34_PYDLL) $(DEST_SIP34_PYI)
 
-    # sip for Python >= 3.8
+    # sip for Python [3.8, 3.9)
     DEST_SIP38_DIR:=$(DEST_PYQT_DIR)/python_3.8
     $(DEST_SIP38_DIR):
 	-$(Q)if [ ! -d "$(DEST_SIP38_DIR)" ] ; then mkdir -p 2>/dev/null $(DEST_SIP38_DIR) ; fi
@@ -407,6 +407,19 @@ ifeq ($(OUT_OF_TREE_BUILD),)
     $(DEST_SIP38_PYI): $(wildcard $(SIP38_TREE)/lib/python*/PyQt5/$(SIP_PYI_FNAME)) | $(DEST_SIP38_DIR)
 	$(Q)$(CP) $? $@
     DEST_SIP += $(DEST_SIP38_PYDLL) $(DEST_SIP38_PYI)
+
+    # sip for Python [3.9, ...
+    DEST_SIP39_DIR:=$(DEST_PYQT_DIR)/python_3.9
+    $(DEST_SIP39_DIR):
+	-$(Q)if [ ! -d "$(DEST_SIP39_DIR)" ] ; then mkdir -p 2>/dev/null $(DEST_SIP39_DIR) ; fi
+    DEST_SIP39_PYDLL:=$(DEST_SIP39_DIR)/$(SIP_PYDLL_FNAME)
+    DEST_SIP39_PYI:=$(DEST_SIP39_DIR)/$(SIP_PYI_FNAME)
+    $(DEST_SIP39_PYDLL): $(wildcard $(SIP39_TREE)/lib/python*/PyQt5/$(SIP_PYDLL_FNAME)) | $(DEST_SIP39_DIR)
+	$(Q)$(CP) $? $@
+    $(DEST_SIP39_PYI): $(wildcard $(SIP39_TREE)/lib/python*/PyQt5/$(SIP_PYI_FNAME)) | $(DEST_SIP39_DIR)
+	$(Q)$(CP) $? $@
+    DEST_SIP += $(DEST_SIP39_PYDLL) $(DEST_SIP39_PYI)
+
   else
     # sip for Python 2.7
     DEST_SIP27_DIR:=$(DEST_PYQT_DIR)
@@ -421,10 +434,14 @@ ifeq ($(OUT_OF_TREE_BUILD),)
 
   # And pick the right sip.so now (Python3 only; for Python2, we already put it in the right place)
   ifeq ($(PYTHON_VERSION_MAJOR),3)
-    ifeq ($(shell test $(PYTHON_VERSION_MINOR) -gt 7; echo $$?),0) # ugh
-      DEST_INSTALL_SIP_PYDLL:=$(DEST_SIP38_PYDLL)
+    ifeq ($(shell test $(PYTHON_VERSION_MINOR) -gt 8; echo $$?),0) # ugh
+      DEST_INSTALL_SIP_PYDLL:=$(DEST_SIP39_PYDLL)
     else
-      DEST_INSTALL_SIP_PYDLL:=$(DEST_SIP34_PYDLL)
+      ifeq ($(shell test $(PYTHON_VERSION_MINOR) -gt 7; echo $$?),0) # ugh
+        DEST_INSTALL_SIP_PYDLL:=$(DEST_SIP38_PYDLL)
+      else
+        DEST_INSTALL_SIP_PYDLL:=$(DEST_SIP34_PYDLL)
+      endif
     endif
     $(DEST_PYQT_DIR)/$(SIP_PYDLL_FNAME): $(DEST_INSTALL_SIP_PYDLL)
 	$(Q)$(CP) $? $@

python/idautils.py

@@ -553,6 +553,7 @@ class Strings(object):
         t.minlen = minlen
         t.only_7bit = only_7bit
         t.display_only_existing_strings = display_only_existing_strings
+        t.ignore_heads = ignore_instructions
         self.refresh()
 
 

release_api_contents2.txt

@@ -2252,6 +2252,8 @@
                'custom_data_type_ids_fids_array___getitem__',
                'custom_data_type_ids_fids_array___len__',
                'custom_data_type_ids_fids_array___setitem__',
+               'custom_data_type_ids_fids_array__get_bytes',
+               'custom_data_type_ids_fids_array__set_bytes',
                'custom_data_type_ids_fids_array_data_get',
                'custom_data_type_ids_t___getFids',
                'custom_data_type_ids_t_dtid_get',
@@ -3330,6 +3332,8 @@
                'fnum_array___getitem__',
                'fnum_array___len__',
                'fnum_array___setitem__',
+               'fnum_array__get_bytes',
+               'fnum_array__set_bytes',
                'fnum_array_data_get',
                'fnumber_t___eq__',
                'fnumber_t___ge__',
@@ -7640,6 +7644,8 @@
                'operands_array___getitem__',
                'operands_array___len__',
                'operands_array___setitem__',
+               'operands_array__get_bytes',
+               'operands_array__set_bytes',
                'operands_array_data_get',
                'operator_info_t_fixtype_get',
                'operator_info_t_fixtype_set',
@@ -9489,6 +9495,8 @@
                'strpath_ids_array___getitem__',
                'strpath_ids_array___len__',
                'strpath_ids_array___setitem__',
+               'strpath_ids_array__get_bytes',
+               'strpath_ids_array__set_bytes',
                'strpath_ids_array_data_get',
                'strpath_t___getIds',
                'strpath_t_delta_get',

release_api_contents3.txt

@@ -2252,6 +2252,8 @@
                'custom_data_type_ids_fids_array___getitem__',
                'custom_data_type_ids_fids_array___len__',
                'custom_data_type_ids_fids_array___setitem__',
+               'custom_data_type_ids_fids_array__get_bytes',
+               'custom_data_type_ids_fids_array__set_bytes',
                'custom_data_type_ids_fids_array_data_get',
                'custom_data_type_ids_t___getFids',
                'custom_data_type_ids_t_dtid_get',
@@ -3330,6 +3332,8 @@
                'fnum_array___getitem__',
                'fnum_array___len__',
                'fnum_array___setitem__',
+               'fnum_array__get_bytes',
+               'fnum_array__set_bytes',
                'fnum_array_data_get',
                'fnumber_t___eq__',
                'fnumber_t___ge__',
@@ -7644,6 +7648,8 @@
                'operands_array___getitem__',
                'operands_array___len__',
                'operands_array___setitem__',
+               'operands_array__get_bytes',
+               'operands_array__set_bytes',
                'operands_array_data_get',
                'operator_info_t_fixtype_get',
                'operator_info_t_fixtype_set',
@@ -9493,6 +9499,8 @@
                'strpath_ids_array___getitem__',
                'strpath_ids_array___len__',
                'strpath_ids_array___setitem__',
+               'strpath_ids_array__get_bytes',
+               'strpath_ids_array__set_bytes',
                'strpath_ids_array_data_get',
                'strpath_t___getIds',
                'strpath_t_delta_get',

release_pydoc_injections2.txt

@@ -23848,6 +23848,13 @@ class fnum_array(__builtin__.object)
  |          i: size_t
  |          v: unsigned short const &
  |  
+ |  _get_bytes(self, *args)
+ |      _get_bytes(self) -> bytevec_t
+ |  
+ |  _set_bytes(self, *args)
+ |      _set_bytes(self, bts)
+ |          bts: bytevec_t const &
+ |  
  |  ----------------------------------------------------------------------
  |  Data descriptors defined here:
  |  
@@ -23857,6 +23864,9 @@ class fnum_array(__builtin__.object)
  |  __weakref__
  |      list of weak references to the object (if defined)
  |  
+ |  bytes
+ |      _get_bytes(self) -> bytevec_t
+ |  
  |  data
  |      fnum_array_data_get(self) -> unsigned short (&)[6]
  |  
@@ -61835,6 +61845,13 @@ class custom_data_type_ids_fids_array(__builtin__.object)
  |          i: size_t
  |          v: short const &
  |  
+ |  _get_bytes(self, *args)
+ |      _get_bytes(self) -> bytevec_t
+ |  
+ |  _set_bytes(self, *args)
+ |      _set_bytes(self, bts)
+ |          bts: bytevec_t const &
+ |  
  |  ----------------------------------------------------------------------
  |  Data descriptors defined here:
  |  
@@ -61844,6 +61861,9 @@ class custom_data_type_ids_fids_array(__builtin__.object)
  |  __weakref__
  |      list of weak references to the object (if defined)
  |  
+ |  bytes
+ |      _get_bytes(self) -> bytevec_t
+ |  
  |  data
  |      custom_data_type_ids_fids_array_data_get(self) -> short (&)[8]
  |  
@@ -63095,6 +63115,13 @@ class strpath_ids_array(__builtin__.object)
  |          i: size_t
  |          v: unsigned-ea-like-numeric-type const &↗
  |  
+ |  _get_bytes(self, *args)
+ |      _get_bytes(self) -> bytevec_t
+ |  
+ |  _set_bytes(self, *args)
+ |      _set_bytes(self, bts)
+ |          bts: bytevec_t const &
+ |  
  |  ----------------------------------------------------------------------
  |  Data descriptors defined here:
  |  
@@ -63104,6 +63131,9 @@ class strpath_ids_array(__builtin__.object)
  |  __weakref__
  |      list of weak references to the object (if defined)
  |  
+ |  bytes
+ |      _get_bytes(self) -> bytevec_t
+ |  
  |  data
  |      strpath_ids_array_data_get(self) -> unsigned-ea-like-numeric-type (&)[32]↗
  |  
@@ -82109,6 +82139,13 @@ class operands_array(__builtin__.object)
  |          i: size_t
  |          v: op_t const &
  |  
+ |  _get_bytes(self, *args)
+ |      _get_bytes(self) -> bytevec_t
+ |  
+ |  _set_bytes(self, *args)
+ |      _set_bytes(self, bts)
+ |          bts: bytevec_t const &
+ |  
  |  ----------------------------------------------------------------------
  |  Data descriptors defined here:
  |  
@@ -82118,6 +82155,9 @@ class operands_array(__builtin__.object)
  |  __weakref__
  |      list of weak references to the object (if defined)
  |  
+ |  bytes
+ |      _get_bytes(self) -> bytevec_t
+ |  
  |  data
  |      operands_array_data_get(self) -> op_t (&)[8]
  |  

release_pydoc_injections3.txt

@@ -23897,6 +23897,13 @@ class fnum_array(builtins.object)
  |  __swig_destroy__ = delete_fnum_array(...)
  |      delete_fnum_array(self)
  |  
+ |  _get_bytes(self, *args) -> 'bytevec_t'
+ |      _get_bytes(self) -> bytevec_t
+ |  
+ |  _set_bytes(self, *args) -> 'void'
+ |      _set_bytes(self, bts)
+ |          bts: bytevec_t const &
+ |  
  |  ----------------------------------------------------------------------
  |  Data descriptors defined here:
  |  
@@ -23906,6 +23913,9 @@ class fnum_array(builtins.object)
  |  __weakref__
  |      list of weak references to the object (if defined)
  |  
+ |  bytes
+ |      _get_bytes(self) -> bytevec_t
+ |  
  |  data
  |      fnum_array_data_get(self) -> unsigned short (&)[6]
  |  
@@ -62518,6 +62528,13 @@ class custom_data_type_ids_fids_array(builtins.object)
  |  __swig_destroy__ = delete_custom_data_type_ids_fids_array(...)
  |      delete_custom_data_type_ids_fids_array(self)
  |  
+ |  _get_bytes(self, *args) -> 'bytevec_t'
+ |      _get_bytes(self) -> bytevec_t
+ |  
+ |  _set_bytes(self, *args) -> 'void'
+ |      _set_bytes(self, bts)
+ |          bts: bytevec_t const &
+ |  
  |  ----------------------------------------------------------------------
  |  Data descriptors defined here:
  |  
@@ -62527,6 +62544,9 @@ class custom_data_type_ids_fids_array(builtins.object)
  |  __weakref__
  |      list of weak references to the object (if defined)
  |  
+ |  bytes
+ |      _get_bytes(self) -> bytevec_t
+ |  
  |  data
  |      custom_data_type_ids_fids_array_data_get(self) -> short (&)[8]
  |  
@@ -63760,6 +63780,13 @@ class strpath_ids_array(builtins.object)
  |  __swig_destroy__ = delete_strpath_ids_array(...)
  |      delete_strpath_ids_array(self)
  |  
+ |  _get_bytes(self, *args) -> 'bytevec_t'
+ |      _get_bytes(self) -> bytevec_t
+ |  
+ |  _set_bytes(self, *args) -> 'void'
+ |      _set_bytes(self, bts)
+ |          bts: bytevec_t const &
+ |  
  |  ----------------------------------------------------------------------
  |  Data descriptors defined here:
  |  
@@ -63769,6 +63796,9 @@ class strpath_ids_array(builtins.object)
  |  __weakref__
  |      list of weak references to the object (if defined)
  |  
+ |  bytes
+ |      _get_bytes(self) -> bytevec_t
+ |  
  |  data
  |      strpath_ids_array_data_get(self) -> unsigned-ea-like-numeric-type (&)[32]↗
  |  
@@ -82707,6 +82737,13 @@ class operands_array(builtins.object)
  |  __swig_destroy__ = delete_operands_array(...)
  |      delete_operands_array(self)
  |  
+ |  _get_bytes(self, *args) -> 'bytevec_t'
+ |      _get_bytes(self) -> bytevec_t
+ |  
+ |  _set_bytes(self, *args) -> 'void'
+ |      _set_bytes(self, bts)
+ |          bts: bytevec_t const &
+ |  
  |  ----------------------------------------------------------------------
  |  Data descriptors defined here:
  |  
@@ -82716,6 +82753,9 @@ class operands_array(builtins.object)
  |  __weakref__
  |      list of weak references to the object (if defined)
  |  
+ |  bytes
+ |      _get_bytes(self) -> bytevec_t
+ |  
  |  data
  |      operands_array_data_get(self) -> op_t (&)[8]
  |  

swig/hexrays.i

@@ -55,6 +55,16 @@ SWIGINTERN void __raise_vdf(const vd_failure_t &e)
 }
 %enddef
 
+%define %method_sets_type_and_gains_ownership_of_regular_object_argument(TYPE, METHOD)
+%feature("pythonprepend") TYPE::METHOD %{
+    o = args[0]
+    self._ensure_cond(self.t == mop_z, "self.t == mop_z")
+%}
+%feature("pythonappend") TYPE::METHOD %{
+    self._acquire_ownership(o, True)
+%}
+%enddef
+
 %typemap(directorin) (const char *format, ...)
 {
   // %typemap(directorin) (const char *format, ...)
@@ -150,6 +160,10 @@ SWIGINTERN void __raise_vdf(const vd_failure_t &e)
 %define_hexrays_lifecycle_object(mop_t);
 %ignore mop_t::_make_strlit(qstring *);
 %template(mopvec_t) qvector<mop_t>;
+%method_sets_type_and_gains_ownership_of_regular_object_argument(mop_t, _make_cases)
+%method_sets_type_and_gains_ownership_of_regular_object_argument(mop_t, _make_callinfo)
+%method_sets_type_and_gains_ownership_of_regular_object_argument(mop_t, _make_pair)
+%method_sets_type_and_gains_ownership_of_regular_object_argument(mop_t, _make_insn)
 
 %template(mcallargs_t) qvector<mcallarg_t>;