[mypyc] Fix incremental compilation with separate flag (1/3)

mypyc/codegen/emit.py

@@ -369,6 +369,23 @@ def c_error_value(self, rtype: RType) -> str:
     def native_function_name(self, fn: FuncDecl) -> str:
         return f"{NATIVE_PREFIX}{fn.cname(self.names)}"
 
+    def native_function_call(self, fn: FuncDecl) -> str:
+        """Return the C expression for a call to `fn`'s native (CPyDef_) entry.
+
+        For cross-group references under `separate=True`, this prepends the
+        exports-table indirection (e.g. `exports_other.CPyDef_foo`). Same as
+        `native_function_name()` for in-group calls.
+        """
+        return f"{self.get_group_prefix(fn)}{NATIVE_PREFIX}{fn.cname(self.names)}"
+
+    def wrapper_function_call(self, fn: FuncDecl) -> str:
+        """Return the C expression for a call to `fn`'s Python-wrapper (CPyPy_) entry.
+
+        Like `native_function_call`, but for the PyObject-level wrapper that
+        boxes/unboxes arguments. Used from slot generators (tp_init, etc.).
+        """
+        return f"{self.get_group_prefix(fn)}{PREFIX}{fn.cname(self.names)}"
+
     def tuple_c_declaration(self, rtuple: RTuple) -> list[str]:
         result = [
             f"#ifndef MYPYC_DECLARED_{rtuple.struct_name}",

mypyc/codegen/emitclass.py

@@ -228,9 +228,11 @@ def generate_class_reuse(
     context = c_emitter.context
     name = cl.name_prefix(c_emitter.names) + "_free_instance"
     struct_name = cl.struct_name(c_emitter.names)
-    context.declarations[name] = HeaderDeclaration(
-        f"CPyThreadLocal {struct_name} *{name};", needs_export=True
-    )
+    # Not exported: the free-instance slot is only read/written by the class's
+    # own setup/dealloc code, which lives in the defining group. Exporting it
+    # also trips a C diagnostic under `Py_GIL_DISABLED`, where `CPyThreadLocal`
+    # expands to `__thread` and can't legally appear inside the exports struct.
+    context.declarations[name] = HeaderDeclaration(f"CPyThreadLocal {struct_name} *{name};")
 
 
 def generate_class(cl: ClassIR, module: str, emitter: Emitter) -> None:
@@ -705,11 +707,15 @@ def emit_null_check() -> None:
         emitter.emit_line(f"PyObject *self = {setup_name}({type_arg});")
         emit_null_check()
         return
-    prefix = emitter.get_group_prefix(new_fn.decl) + NATIVE_PREFIX if native_prefix else PREFIX
+    call = (
+        emitter.native_function_call(new_fn.decl)
+        if native_prefix
+        else emitter.wrapper_function_call(new_fn.decl)
+    )
     all_args = type_arg
     if new_args != "":
         all_args += ", " + new_args
-    emitter.emit_line(f"PyObject *self = {prefix}{new_fn.cname(emitter.names)}({all_args});")
+    emitter.emit_line(f"PyObject *self = {call}({all_args});")
     emit_null_check()
 
     # skip __init__ if __new__ returns some other type
@@ -743,17 +749,13 @@ def generate_constructor_for_class(
 
     args = ", ".join(["self"] + fn_args)
     if init_fn is not None:
-        prefix = PREFIX if use_wrapper else NATIVE_PREFIX
-        cast = "!= NULL ? 0 : -1" if use_wrapper else ""
-        emitter.emit_line(
-            "char res = {}{}{}({}){};".format(
-                emitter.get_group_prefix(init_fn.decl),
-                prefix,
-                init_fn.cname(emitter.names),
-                args,
-                cast,
-            )
+        call = (
+            emitter.wrapper_function_call(init_fn.decl)
+            if use_wrapper
+            else emitter.native_function_call(init_fn.decl)
         )
+        cast = "!= NULL ? 0 : -1" if use_wrapper else ""
+        emitter.emit_line(f"char res = {call}({args}){cast};")
         emitter.emit_line("if (res == 2) {")
         emitter.emit_line("Py_DECREF(self);")
         emitter.emit_line("return NULL;")
@@ -786,9 +788,8 @@ def generate_init_for_class(cl: ClassIR, init_fn: FuncIR, emitter: Emitter) -> s
     emitter.emit_line("{")
     if cl.allow_interpreted_subclasses or cl.builtin_base or cl.has_method("__new__"):
         emitter.emit_line(
-            "return {}{}(self, args, kwds) != NULL ? 0 : -1;".format(
-                PREFIX, init_fn.cname(emitter.names)
-            )
+            f"return {emitter.wrapper_function_call(init_fn.decl)}"
+            "(self, args, kwds) != NULL ? 0 : -1;"
         )
     else:
         emitter.emit_line("return 0;")
@@ -834,7 +835,7 @@ def generate_new_for_class(
         # can enforce that instances are always properly initialized. This
         # is needed to support always defined attributes.
         emitter.emit_line(
-            f"PyObject *ret = {PREFIX}{init_fn.cname(emitter.names)}(self, args, kwds);"
+            f"PyObject *ret = {emitter.wrapper_function_call(init_fn.decl)}(self, args, kwds);"
         )
         emitter.emit_lines("if (ret == NULL) {", "    Py_DECREF(self);", "    return NULL;", "}")
         emitter.emit_line("Py_DECREF(ret);")

mypyc/codegen/emitfunc.py

@@ -88,8 +88,12 @@
 
 
 def native_function_type(fn: FuncIR, emitter: Emitter) -> str:
-    args = ", ".join(emitter.ctype(arg.type) for arg in fn.args) or "void"
-    ret = emitter.ctype(fn.ret_type)
+    return native_function_type_from_decl(fn.decl, emitter)
+
+
+def native_function_type_from_decl(decl: FuncDecl, emitter: Emitter) -> str:
+    args = ", ".join(emitter.ctype(arg.type) for arg in decl.sig.args) or "void"
+    ret = emitter.ctype(decl.sig.ret_type)
     return f"{ret} (*)({args})"
 
 
@@ -579,8 +583,11 @@ def emit_method_call(self, dest: str, op_obj: Value, name: str, op_args: list[Va
         rtype = op_obj.type
         assert isinstance(rtype, RInstance), rtype
         class_ir = rtype.class_ir
-        method = rtype.class_ir.get_method(name)
-        assert method is not None
+        # Use method_decl (not get_method) because under separate compilation the
+        # FuncIR body may live in a different group — only its declaration is
+        # visible here, and a decl is all we need to emit a direct C call
+        # (the symbol resolves through that group's exports table).
+        method_decl = rtype.class_ir.method_decl(name)
 
         # Can we call the method directly, bypassing vtable?
         is_direct = class_ir.is_method_final(name)
@@ -589,16 +596,15 @@ def emit_method_call(self, dest: str, op_obj: Value, name: str, op_args: list[Va
         # turned into the class for class methods
         obj_args = (
             []
-            if method.decl.kind == FUNC_STATICMETHOD
-            else [f"(PyObject *)Py_TYPE({obj})"] if method.decl.kind == FUNC_CLASSMETHOD else [obj]
+            if method_decl.kind == FUNC_STATICMETHOD
+            else [f"(PyObject *)Py_TYPE({obj})"] if method_decl.kind == FUNC_CLASSMETHOD else [obj]
         )
         args = ", ".join(obj_args + [self.reg(arg) for arg in op_args])
-        mtype = native_function_type(method, self.emitter)
+        mtype = native_function_type_from_decl(method_decl, self.emitter)
         version = "_TRAIT" if rtype.class_ir.is_trait else ""
         if is_direct:
             # Directly call method, without going through the vtable.
-            lib = self.emitter.get_group_prefix(method.decl)
-            self.emit_line(f"{dest}{lib}{NATIVE_PREFIX}{method.cname(self.names)}({args});")
+            self.emit_line(f"{dest}{self.emitter.native_function_call(method_decl)}({args});")
         else:
             # Call using vtable.
             method_idx = rtype.method_index(name)

mypyc/codegen/emitmodule.py

@@ -317,6 +317,14 @@ def compile_modules_to_ir(
         else:
             scc_ir = compile_scc_to_ir(trees, result, mapper, compiler_options, errors)
             modules.update(scc_ir)
+            # A later SCC loaded from cache may reference classes/functions
+            # defined in this freshly-built SCC; populate deser_ctx so the
+            # cached IR deserializer can resolve those cross-SCC references.
+            for module_ir in scc_ir.values():
+                for cl in module_ir.classes:
+                    deser_ctx.classes.setdefault(cl.fullname, cl)
+                for fn in module_ir.functions:
+                    deser_ctx.functions.setdefault(fn.decl.id, fn)
 
     return modules
 
@@ -517,13 +525,16 @@ def generate_function_declaration(fn: FuncIR, emitter: Emitter) -> None:
         f"{native_function_header(fn.decl, emitter)};", needs_export=True
     )
     if fn.name != TOP_LEVEL_NAME and not fn.internal:
+        # needs_export=True so Python-wrapper (CPyPy_) symbols are reachable from
+        # other groups via the export table — needed for cross-group inherited
+        # __init__ / __new__ slot dispatch under `separate=True`.
         if is_fastcall_supported(fn, emitter.capi_version):
             emitter.context.declarations[PREFIX + fn.cname(emitter.names)] = HeaderDeclaration(
-                f"{wrapper_function_header(fn, emitter.names)};"
+                f"{wrapper_function_header(fn, emitter.names)};", needs_export=True
             )
         else:
             emitter.context.declarations[PREFIX + fn.cname(emitter.names)] = HeaderDeclaration(
-                f"{legacy_wrapper_function_header(fn, emitter.names)};"
+                f"{legacy_wrapper_function_header(fn, emitter.names)};", needs_export=True
             )
 
 
@@ -886,6 +897,21 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None:
                 "goto fail;",
                 "}",
                 "",
+                # Expose ensure_deps_<short> as a capsule so the shim can call
+                # it before invoking the per-module init.
+                f"extern int ensure_deps_{short_name}(void);",
+                'capsule = PyCapsule_New((void *)ensure_deps_{sh}, "{lib}.ensure_deps", NULL);'.format(
+                    sh=short_name, lib=shared_lib_name(self.group_name)
+                ),
+                "if (!capsule) {",
+                "goto fail;",
+                "}",
+                'res = PyObject_SetAttrString(module, "ensure_deps", capsule);',
+                "Py_DECREF(capsule);",
+                "if (res < 0) {",
+                "goto fail;",
+                "}",
+                "",
             )
 
         for mod in self.modules:
@@ -917,25 +943,58 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None:
                 "",
             )
 
-        for group in sorted(self.context.group_deps):
-            egroup = exported_name(group)
+        # End of exec_<short_name>: only sets up capsules/module attributes.
+        # Cross-group imports (populating `exports_<dep>` tables) are split
+        # out into ensure_deps_<short_name>() below and run later, from the
+        # shim's PyInit. See generate_shared_lib_init for details.
+        emitter.emit_lines("return 0;", "fail:", "return -1;", "}")
+
+        if self.compiler_options.separate:
+            # ensure_deps_<short>(): populates cross-group exports tables. Run
+            # once, lazily, from the shim's PyInit just before invoking the
+            # per-module init capsule. This defers cross-group imports out of
+            # the shared-lib PyInit so they can't transitively trigger a
+            # sibling package's __init__.py while another package __init__.py
+            # is still mid-flight.
             emitter.emit_lines(
-                'tmp = PyImport_ImportModule("{}"); if (!tmp) goto fail; Py_DECREF(tmp);'.format(
-                    shared_lib_name(group)
-                ),
-                'struct export_table_{} *pexports_{} = PyCapsule_Import("{}.exports", 0);'.format(
-                    egroup, egroup, shared_lib_name(group)
-                ),
-                f"if (!pexports_{egroup}) {{",
-                "goto fail;",
-                "}",
-                "memcpy(&exports_{group}, pexports_{group}, sizeof(exports_{group}));".format(
-                    group=egroup
-                ),
                 "",
+                f"int ensure_deps_{short_name}(void)",
+                "{",
+                "static int done = 0;",
+                "if (done) return 0;",
             )
-
-        emitter.emit_lines("return 0;", "fail:", "return -1;", "}")
+            if self.context.group_deps:
+                emitter.emit_line(
+                    "static PyObject *_mypyc_fromlist = NULL; "
+                    "if (!_mypyc_fromlist) { "
+                    '_mypyc_fromlist = Py_BuildValue("(s)", "*"); '
+                    "if (!_mypyc_fromlist) return -1; }"
+                )
+                emitter.emit_line("PyObject *tmp;")
+                emitter.emit_line("PyObject *caps;")
+            for group in sorted(self.context.group_deps):
+                egroup = exported_name(group)
+                # ImportModuleLevel with fromlist returns the leaf via
+                # sys.modules (no dotted getattr walk), and fetching the
+                # `exports` capsule directly off that module bypasses
+                # PyCapsule_Import (which would redo the attribute walk).
+                emitter.emit_lines(
+                    'tmp = PyImport_ImportModuleLevel("{}", NULL, NULL, _mypyc_fromlist, 0);'.format(
+                        shared_lib_name(group)
+                    ),
+                    "if (!tmp) return -1;",
+                    'caps = PyObject_GetAttrString(tmp, "exports");',
+                    "Py_DECREF(tmp);",
+                    "if (!caps) return -1;",
+                    "struct export_table_{g} *pexports_{g} = "
+                    '(struct export_table_{g} *)PyCapsule_GetPointer(caps, "{lib}.exports");'.format(
+                        g=egroup, lib=shared_lib_name(group)
+                    ),
+                    "Py_DECREF(caps);",
+                    f"if (!pexports_{egroup}) return -1;",
+                    "memcpy(&exports_{g}, pexports_{g}, sizeof(exports_{g}));".format(g=egroup),
+                )
+            emitter.emit_lines("done = 1;", "return 0;", "}")
 
         if self.multi_phase_init:
             emitter.emit_lines(
@@ -980,6 +1039,7 @@ def generate_shared_lib_init(self, emitter: Emitter) -> None:
                 "}",
                 f"if (exec_{short_name}(module) < 0) {{",
                 "Py_DECREF(module);",
+                "module = NULL;",
                 "return NULL;",
                 "}",
                 "return module;",

mypyc/codegen/emitwrapper.py

@@ -537,7 +537,7 @@ def generate_get_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str:
         )
     )
     emitter.emit_line("instance = instance ? instance : Py_None;")
-    emitter.emit_line(f"return {NATIVE_PREFIX}{fn.cname(emitter.names)}(self, instance, owner);")
+    emitter.emit_line(f"return {emitter.native_function_call(fn.decl)}(self, instance, owner);")
     emitter.emit_line("}")
 
     return name
@@ -600,8 +600,8 @@ def generate_bool_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str:
     name = f"{DUNDER_PREFIX}{fn.name}{cl.name_prefix(emitter.names)}"
     emitter.emit_line(f"static int {name}(PyObject *self) {{")
     emitter.emit_line(
-        "{}val = {}{}(self);".format(
-            emitter.ctype_spaced(fn.ret_type), NATIVE_PREFIX, fn.cname(emitter.names)
+        "{}val = {}(self);".format(
+            emitter.ctype_spaced(fn.ret_type), emitter.native_function_call(fn.decl)
         )
     )
     emitter.emit_error_check("val", fn.ret_type, "return -1;")
@@ -704,8 +704,8 @@ def generate_set_del_item_wrapper_inner(
         generate_arg_check(arg.name, arg.type, emitter, GotoHandler("fail"))
     native_args = ", ".join(f"arg_{arg.name}" for arg in args)
     emitter.emit_line(
-        "{}val = {}{}({});".format(
-            emitter.ctype_spaced(fn.ret_type), NATIVE_PREFIX, fn.cname(emitter.names), native_args
+        "{}val = {}({});".format(
+            emitter.ctype_spaced(fn.ret_type), emitter.native_function_call(fn.decl), native_args
         )
     )
     emitter.emit_error_check("val", fn.ret_type, "goto fail;")
@@ -722,8 +722,8 @@ def generate_contains_wrapper(cl: ClassIR, fn: FuncIR, emitter: Emitter) -> str:
     emitter.emit_line(f"static int {name}(PyObject *self, PyObject *obj_item) {{")
     generate_arg_check("item", fn.args[1].type, emitter, ReturnHandler("-1"))
     emitter.emit_line(
-        "{}val = {}{}(self, arg_item);".format(
-            emitter.ctype_spaced(fn.ret_type), NATIVE_PREFIX, fn.cname(emitter.names)
+        "{}val = {}(self, arg_item);".format(
+            emitter.ctype_spaced(fn.ret_type), emitter.native_function_call(fn.decl)
         )
     )
     emitter.emit_error_check("val", fn.ret_type, "return -1;")
@@ -857,6 +857,9 @@ def set_target(self, fn: FuncIR) -> None:
         """
         self.target_name = fn.name
         self.target_cname = fn.cname(self.emitter.names)
+        # Cached native-call expression so cross-group targets go through the
+        # exports table; same as `NATIVE_PREFIX + cname` for in-group calls.
+        self.target_native_call = self.emitter.native_function_call(fn.decl)
         self.num_bitmap_args = fn.sig.num_bitmap_args
         if self.num_bitmap_args:
             self.args = fn.args[: -self.num_bitmap_args]
@@ -927,8 +930,8 @@ def emit_call(self, not_implemented_handler: str = "") -> None:
             # TODO: The Py_RETURN macros return the correct PyObject * with reference count
             #       handling. Are they relevant?
             emitter.emit_line(
-                "{}retval = {}{}({});".format(
-                    emitter.ctype_spaced(ret_type), NATIVE_PREFIX, self.target_cname, native_args
+                "{}retval = {}({});".format(
+                    emitter.ctype_spaced(ret_type), self.target_native_call, native_args
                 )
             )
             emitter.emit_lines(*self.cleanups)
@@ -940,19 +943,15 @@ def emit_call(self, not_implemented_handler: str = "") -> None:
         else:
             if not_implemented_handler and not isinstance(ret_type, RInstance):
                 # The return value type may overlap with NotImplemented.
-                emitter.emit_line(
-                    "PyObject *retbox = {}{}({});".format(
-                        NATIVE_PREFIX, self.target_cname, native_args
-                    )
-                )
+                emitter.emit_line(f"PyObject *retbox = {self.target_native_call}({native_args});")
                 emitter.emit_lines(
                     "if (retbox == Py_NotImplemented) {",
                     not_implemented_handler,
                     "}",
                     "return retbox;",
                 )
             else:
-                emitter.emit_line(f"return {NATIVE_PREFIX}{self.target_cname}({native_args});")
+                emitter.emit_line(f"return {self.target_native_call}({native_args});")
             # TODO: Tracebacks?
 
     def error(self) -> ErrorHandler:

mypyc/ir/class_ir.py

@@ -284,6 +284,12 @@ def has_method(self, name: str) -> bool:
         return True
 
     def is_method_final(self, name: str) -> bool:
+        if not self.is_ext_class:
+            # Non-extension classes don't use vtable dispatch; their mypyc-compiled
+            # "fast" methods are always called directly by C name. Treating them as
+            # final here keeps codegen from trying to index into a vtable that was
+            # never computed (non-ext classes skip compute_vtable).
+            return True
         subs = self.subclasses()
         if subs is None:
             return self.is_final_class