jython · Stewori · Sep 15, 2024 · Oct 16, 2024 · Mar 2, 2026 · Mar 4, 2026
diff --git a/Lib/test/large_methods_272$py.class b/Lib/test/large_methods_272$py.class
diff --git a/Lib/test/large_module.py b/Lib/test/large_module.py
@@ -65935,4 +65935,5 @@ def large_function(self):
 count += 1
 count += 1
 count += 1 # +10
+msg = 'large_module '+str(count)
 
diff --git a/Lib/test/large_module.pyc b/Lib/test/large_module.pyc
diff --git a/Lib/test/large_module_272$py.class b/Lib/test/large_module_272$py.class
diff --git a/Lib/test/test_large_method_bytecode_jy.py b/Lib/test/test_large_method_bytecode_jy.py
@@ -16,6 +16,7 @@
 import unittest
 from test import test_support
 
+
 class large_method_tests(unittest.TestCase):
     '''Tests some oversized functions and methods.
     '''
@@ -52,6 +53,7 @@ def test_small_func(self):
         '''
         self.assertEqual(large_methods.small_function(), 'small 10')
 
+
 class large_module_tests(unittest.TestCase):
     '''Tests a module with oversized main-code.
     So the whole module is represented as a single PyBytecode object.
@@ -68,6 +70,7 @@ def test_large_module_main(self):
         '''Tests the module's oversized main-code.
         '''
         self.assertEqual(large_module.count, 2310)
+        self.assertEqual(large_module.msg, 'large_module 2310')
 
     def test_large_module_method(self):
         cl2 = large_module.OversizedMethodHolder()
@@ -82,10 +85,89 @@ def test_large_module_very_large_func(self):
     def test_large_module_small_func(self):
         self.assertEqual(large_module.small_function(), 'small 10')
 
+
+class large_method_back_compat_tests(unittest.TestCase):
+    '''Tests backwards compatibility of the mechanism that handles oversized functions.
+    '''
+
+    @classmethod
+    def setUpClass(cls):
+        from org.python.core import imp as _imp
+        from org.python.core import BytecodeLoader as _bcl
+        global imp_class, bytecode_loader_class
+        imp_class = _imp
+        bytecode_loader_class = _bcl
+
+    def test_backwards_compat(self):
+        # To avoid unloading, we test everything in one go. This way, import failures are
+        # tested before import succeeds for the first time. To verify that the tested issue
+        # is real, we assert import failures on large_methods_272$py.class and
+        # large_module_272$py.class which were compiled by Jython 2.7.2.
+        # We later turn on the mitigation to see that it restores compatibility.
+
+        self.assertFalse(imp_class.getIgnoreAPIVersion())
+        with self.assertRaises(ImportError) as cm_methods:
+            import large_methods_272
+        msg_start = 'compiled unit contains version 38 code (%s required): '
+        msg_end = 'large_%s_272$py.class'
+        api_v = imp_class.getAPIVersion()
+        self.assertTrue(cm_methods.exception.message.startswith(msg_start % api_v))
+        self.assertTrue(cm_methods.exception.message.endswith(msg_end % 'methods'))
+        with self.assertRaises(ImportError) as cm_module:
+            import large_module_272
+        self.assertTrue(cm_module.exception.message.startswith(msg_start % api_v))
+        self.assertTrue(cm_module.exception.message.endswith(msg_end % 'module'))
+
+        # Now we bypass the API version check, but deliberately turn off the serialVersionUID
+        # mismatch mitigation. This way we confirm that the issue is real and serialized objects
+        # from older Jython might cause this import error.
+
+        imp_class.setIgnoreAPIVersion(True)
+        self.assertTrue(bytecode_loader_class.getIgnoreSerialVersionUID())
+        bytecode_loader_class.setIgnoreSerialVersionUID(False)
+
+        s = 'compiled unit contains incompatible serialized objects (for oversized function handling): '
+        s2 = 'local class incompatible: stream classdesc serialVersionUID = '
+        with self.assertRaises(ImportError) as cm2_methods:
+            import large_methods_272
+        self.assertTrue(cm2_methods.exception.message.startswith(s))
+        self.assertTrue(s2 in cm2_methods.exception.message)
+
+        with self.assertRaises(ImportError) as cm2_module:
+            import large_module_272
+        self.assertTrue(cm2_module.exception.message.startswith(s))
+        self.assertTrue(s2 in cm2_module.exception.message)
+
+        # Finally, we activate full mitigation of the issue and verify that everything passes now.
+
+        bytecode_loader_class.setIgnoreSerialVersionUID(True)
+
+        import large_methods_272
+        self.assertEqual(large_methods_272.large_function(), 'large 2300')
+        cl_272 = large_methods_272.OversizedMethodHolder()
+        self.assertEqual(cl_272.large_function(), 'large_method 2300')
+        self.assertEqual(large_methods_272.very_large_function(), 'very large 58900')
+        self.assertEqual(large_methods_272.small_function(), 'small 10')
+
+        import large_module_272
+        self.assertEqual(large_module_272.count, 2310)
+        self.assertEqual(large_module_272.msg, 'large_module 2310')
+        cl2_272 = large_module_272.OversizedMethodHolder()
+        self.assertEqual(cl2_272.large_function(), 'large_method 2300')
+        self.assertEqual(large_module_272.large_function(), 'large 2300')
+        self.assertEqual(large_module_272.very_large_function(), 'very large 58900')
+        self.assertEqual(large_module_272.small_function(), 'small 10')
+
+        # Restore default setting:
+        imp_class.setIgnoreAPIVersion(False)
+        self.assertFalse(imp_class.getIgnoreAPIVersion())
+
+
 def test_main():
     test_support.run_unittest(
         large_method_tests,
-        large_module_tests
+        large_module_tests,
+        large_method_back_compat_tests
     )
 
 if __name__ == "__main__":

diff --git a/NEWS b/NEWS
@@ -29,6 +29,8 @@ New Features
       deprecated for removal in Java 26. See https://github.com/jnr/jffi/issues/165
 
 Jython 2.7.5a1 Bugs fixed
+    - [ GH-416 ] Solve serialization backward incompatibility of oversized methods and functions
+    - [ GH-234 ] Clarification on compile (Module or method too large)
     - [ GH-388 ] @ExposedType annotation should be @Documented
     - [ GH-404 ] Windows code page 65001 (UTF-8) not supported by Jython
     - [ GH-384 ] Console encoding inferred incorrectly on Java 21

diff --git a/build.gradle b/build.gradle
@@ -483,8 +483,14 @@ task mergePythonLib(
 
     // Copy Jython Lib, with precedence over CPython files of the same name
     duplicatesStrategy = DuplicatesStrategy.INCLUDE
-    from libJython
-    exclude '**/*.class'
+    from(libJython) {
+        exclude '**/*.class'
+    }
+    from(libJython) {
+        // re-include these in a separate block to avoid interference with exclude
+        include 'test/large_methods_272$py.class'
+        include 'test/large_module_272$py.class'
+    }
 
     // Allow Gradle to infer the need to regenerate the outputs
     inputs.dir libJython
@@ -539,10 +545,17 @@ task copyTestLib(
         dependsOn: mergePythonLib,
         description: 'Copy merged Python library (tests only)') {
     into buildTestLibDir
-    from pythonLibDir
-    exclude '**/*.pyd', '**/*.class' // test material includes .pyc files
-    // Include only tests and test material
-    include libTestSpecs
+
+    from(pythonLibDir) {
+        exclude '**/*.pyd', '**/*.class' // test material includes .pyc files
+        // Include only tests and test material
+        include libTestSpecs
+    }
+    from(pythonLibDir) {
+        // re-include these in a separate block to avoid interference with exclude
+        include 'test/large_methods_272$py.class'
+        include 'test/large_module_272$py.class'
+    }
 }
 
 // Attach this task to processResources

diff --git a/build.xml b/build.xml
@@ -1167,6 +1167,10 @@ The text for an official release would continue like ...
             <fileset dir="${basedir}/Lib">
                 <exclude name="**/*.class"/>
             </fileset>
+            <fileset dir="${basedir}/Lib">
+                <include name="test/large_methods_272$py.class"/>
+                <include name="test/large_module_272$py.class"/>
+            </fileset>
         </copy>
     </target>
 

diff --git a/src/org/python/compiler/Module.java b/src/org/python/compiler/Module.java
@@ -783,52 +783,84 @@ private static PyBytecode loadPyBytecode(String filename, boolean try_cpython)
         } else {
             String CPython_command = System.getProperty(PYTHON_CPYTHON);
             if (try_cpython && CPython_command != null) {
-                // check version...
-                String command_ver = CPython_command + " --version";
-                String command = CPython_command + " -m py_compile " + filename;
+                // check that the command is an absolute path or resolves to a standard system location...
+                File pythonExec = new File(CPython_command);
+                if (!pythonExec.isAbsolute()) {
+                    boolean foundSystemPython = false;
+                    // for convenience, some bare standard commands are still permitted in this case:
+                    if (CPython_command.equals("python") || CPython_command.equals("python2") 
+                            || CPython_command.equals("python2.7") || CPython_command.equals("python.exe")
+                            || CPython_command.equals("python2.exe")) {
+                        // canonical locations for these standard commands on Linux, macOS, and Windows
+                        String[] systemDirectories = {
+                            "/usr/bin", "/usr/local/bin", "/opt/homebrew/bin", // Linux & macOS
+                            "C:\\Python27", "C:\\Program Files\\Python27"};    // Windows
+                        for (String dir : systemDirectories) {
+                            pythonExec = new File(dir, CPython_command);
+                            if (!pythonExec.exists() && dir.startsWith("C:\\") && !CPython_command.endsWith(".exe")) {
+                                // one more try for Windows dirs by adding .exe postfix
+                                pythonExec = new File(dir, CPython_command+".exe");
+                            }
+                            if (pythonExec.exists() && pythonExec.isFile() && pythonExec.canExecute()) {
+                                // resolve absolute path
+                                CPython_command = pythonExec.getAbsolutePath();
+                                foundSystemPython = true;
+                                break;
+                            }
+                        }
+                    }
+                    if (!foundSystemPython) {
+                        throw new RuntimeException(
+                            "The CPython command must be an absolute path or resolve to a standard system location. " +
+                            "Please provide a full absolute path instead of: " + CPython_command);
+                    }
+                }
+
                 Exception exc = null;
-                int result = 0;
+                int result = -1;
                 String reason;
                 try {
-                    Process p = Runtime.getRuntime().exec(command_ver);
-                    // Python 2.7 writes version to error-stream for some reason:
-                    BufferedReader br =
-                            new BufferedReader(new InputStreamReader(p.getErrorStream()));
-                    String cp_version = br.readLine();
-                    while (br.readLine() != null) {}
-                    br.close();
-                    if (cp_version == null) {
-                        // Also try input-stream as fallback, just in case...
-                        br = new BufferedReader(new InputStreamReader(p.getInputStream()));
+                    // check version...
+                    ProcessBuilder pbVersion = new ProcessBuilder(CPython_command, "--version");
+                    // merge stderr into stdout to prevent stream buffer deadlocks
+                    pbVersion.redirectErrorStream(true); 
+                    Process pVersion = pbVersion.start();
+                    String cp_version = null;
+                    try (BufferedReader br = new BufferedReader(new InputStreamReader(pVersion.getInputStream()))) {
                         cp_version = br.readLine();
+                        // consume any remaining output so the process can terminate cleanly
                         while (br.readLine() != null) {}
-                        br.close();
                     }
-                    result = p.waitFor();
-                    if (!cp_version.startsWith("Python 2.7.")) {
-                        reason = cp_version + " has been provided, but 2.7.x is required.";
+                    result = pVersion.waitFor();
+                    if (cp_version == null || !cp_version.startsWith("Python 2.7.")) {
+                        reason  = (cp_version == null ? "No version output" : cp_version) 
+                                + " has been provided, but 2.7.x is required.";
                         throw new RuntimeException(String.format(LARGE_METHOD_MSG, filename)
-                                + String.format(TRIED_CREATE_PYC_MSG, command, reason)
+                                + String.format(TRIED_CREATE_PYC_MSG,
+                                CPython_command + " -m py_compile " + filename, reason)
                                 + String.format(PLEASE_PROVIDE_MSG, filename) + CPYTHON_CMD_MSG);
                     }
-                } catch (InterruptedException | IOException e) {
-                    exc = e;
-                }
-
-                if (exc == null && result == 0) {
-                    try {
-                        Process p = Runtime.getRuntime().exec(command);
-                        result = p.waitFor();
+                    // compile...
+                    if (result == 0) {
+                        ProcessBuilder pbCompile = new ProcessBuilder(CPython_command, "-m", "py_compile", filename);
+                        pbCompile.redirectErrorStream(true);
+                        Process pCompile = pbCompile.start();
+                        // consume stream to prevent hanging
+                        try (BufferedReader br = new BufferedReader(new InputStreamReader(pCompile.getInputStream()))) {
+                            while (br.readLine() != null) {}
+                        }
+                        result = pCompile.waitFor();
                         if (result == 0) {
                             return loadPyBytecode(filename, false);
                         }
-                    } catch (InterruptedException | IOException e) {
-                        exc = e;
                     }
+                } catch (InterruptedException | IOException e) {
+                    exc = e;
                 }
                 reason = exc != null ? "of " + exc.toString() : "of a bad return: " + result;
                 String exc_msg = String.format(LARGE_METHOD_MSG, filename)
-                        + String.format(TRIED_CREATE_PYC_MSG, command, reason)
+                        + String.format(TRIED_CREATE_PYC_MSG,
+                        CPython_command + " -m py_compile " + filename, reason)
                         + String.format(PLEASE_PROVIDE_MSG, filename) + CPYTHON_CMD_MSG;
                 throw exc != null ? new RuntimeException(exc_msg, exc)
                         : new RuntimeException(exc_msg);