Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
201 changes: 201 additions & 0 deletions build-tools/python/tool/opcode_gen.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,201 @@
# Copyright (c)2026 Jython Developers.
# Licensed to PSF under a contributor agreement.
#
# Generate the Java opcode constants class for the CPython version running this script
Comment thread
jeff5 marked this conversation as resolved.
Resolved
#
# Usage: python opcode_gen.py [--package PKG] [dstdir]
#
# The class written is named OpcodeNNN (e.g. Opcode315 for CPython 3.15)
# in package PKG (default org.python.core). With dstdir (the root of a
# Java source tree) it is written to dstdir/<PKG as path>/OpcodeNNN.java,
# otherwise to standard output. The Gradle build runs this (task
# core:generateOpcodes) with the CPython whose byte code the interpreter
# supports, writing to core/build/generated/sources/opcodes/java/main.
# The target version is simply that of the Python running the script.
#
# Everything is taken from the opcode and dis modules, so that a move
# to a new version of CPython is (as far as the constants go) a re-run.
#
# Specialized ("quickened") and instrumented opcodes are deliberately
# omitted: CPython only creates them at run time, so they never appear
# in the code objects we load (from .pyc files or marshal).

import argparse, sys
from pathlib import Path
import opcode, dis

VER = f"{sys.version_info.major}{sys.version_info.minor}"
CLASS = f"Opcode{VER}"
VERSION = f"{sys.version_info.major}.{sys.version_info.minor}"


def opcodes():
"Real (not pseudo, not instrumented) opcodes in numerical order"
ops = [(v, k) for k, v in opcode.opmap.items()
if v < 256 and not k.startswith("INSTRUMENTED_")]
return sorted(ops)


def generate(out, package="org.python.core"):
ops = "".join(const(f"CPython opcode {k}", k, v) for v, k in opcodes())
out.write(f"""\
package {package};

// Generated by build-tools/python/tool/opcode_gen.py (Gradle task core:generateOpcodes)
// from CPython {sys.version.split()[0]}. Do not edit by hand.
// Specialized and instrumented opcodes are omitted: they never
// appear in code loaded from .pyc files or marshal.

/** Constants for opcodes taken from CPython {VERSION} {{@code opcode}} module. */
class {CLASS} {{

private {CLASS}() {{}} // no instances

/* Instruction opcodes for compiled code */

{ops}
/** Opcodes at or above this value use their argument. */
static final int HAVE_ARGUMENT = {opcode.HAVE_ARGUMENT};

{binary_op_section()}
{compare_section()}
{intrinsic_section()}{common_section()}{cache_section()}}}
""")


def const(doc, name, value):
"A documented Java int constant declaration"
return f" /** {doc} */\n static final int {name} = {value};\n"


def binary_op_section():
nb_ops = "".join(const(f"Encoding of {sym} used in BINARY_OP opcode.", name, i)
for i, (name, sym) in enumerate(dis._nb_ops))
return f"""\
// An encoding of binary operations used only by BINARY_OP

{nb_ops}"""


def compare_section():
cmps = "".join(const(f"Encoding of {sym} in COMPARE_OP opcode.",
f"CMP_{i_name(sym)}", i)
for i, sym in enumerate(dis.cmp_op))
# From 3.13 the comparison is in the upper bits of the argument
if sys.version_info >= (3, 13):
how = "oparg >> 5"
cmps += (const("Shift to extract comparison from COMPARE_OP argument.",
"CMP_SHIFT", 5)
+ const("Flag in COMPARE_OP argument: convert result to bool.",
"CMP_TO_BOOL", 16))
else:
how = "oparg"
return f"""\
// Comparisons encoded in COMPARE_OP (as {how})

{cmps}"""


def intrinsic_section():
if not hasattr(dis, "_intrinsic_1_descs"): # 3.12+
return ""
intrinsics = "".join(
const(f"Encoding of {name} in CALL_INTRINSIC_{n} opcode.", name, i)
for n, descs in ((1, dis._intrinsic_1_descs), (2, dis._intrinsic_2_descs))
for i, name in enumerate(descs))
return f"""\
// Operations encoded in CALL_INTRINSIC_1 and CALL_INTRINSIC_2

{intrinsics}
"""


def common_section():
if not hasattr(dis, "_common_constants"): # 3.14+
return ""
consts = dis._common_constants
commons = "".join(
const(f"Index of {common_repr(c)} in LOAD_COMMON_CONSTANT.",
f"COMMON_{common_name(c)}", i)
for i, c in enumerate(consts))
return f"""\
// Constants indexed by LOAD_COMMON_CONSTANT

{commons}\
{const("Number of LOAD_COMMON_CONSTANT constants.", "COMMON_CONSTANT_COUNT", len(consts))}
"""


def cache_section():
caches = "".join(
const(f"Cache following {k} opcode (words).", f"INLINE_CACHE_ENTRIES_{k}", n)
for k, n in inline_cache_entries() if n and k in opcode.opmap)
return f"""\
/*
* Various CPython opcodes are followed by an in-line cache, which
* is zero in the byte code initially. We do not implement this
* cache, but end those instructions with an advance of the IP of
* the right size. Jumps are relative to the end of the cache.
*/

{caches}"""


def inline_cache_entries():
"(name, size) of the in-line cache of each opcode, sorted by name"
entries = opcode._inline_cache_entries
if isinstance(entries, list):
# Before 3.12 this is a list indexed by opcode
entries = {opcode.opname[i]: n for i, n in enumerate(entries)}
return sorted(entries.items())


def i_name(sym):
return {"<": "LT", "<=": "LE", "==": "EQ", "!=": "NE", ">": "GT",
">=": "GE"}[sym]


def common_repr(c):
if isinstance(c, type) or callable(c):
return "{@code " + c.__name__ + "}"
return "{@code " + repr(c) + "}"


def common_name(c):
if isinstance(c, type) or callable(c):
# AssertionError -> ASSERTION_ERROR, all -> ALL
n = c.__name__
return "".join("_" + ch if ch.isupper() and i else ch
for i, ch in enumerate(n)).upper()
return {None: "NONE", "": "EMPTY_STR", True: "TRUE", False: "FALSE",
-1: "MINUS_ONE"}[c]


def get_parser():
parser = argparse.ArgumentParser(
prog='opcode_gen',
description='Generate Java opcode constants for the running CPython.'
)

parser.add_argument('dest_dir', nargs='?', type=Path,
help='Root of Java source tree (default: write to standard output)')
parser.add_argument('--package', '-p', default='org.python.core',
help='Java package of the generated class (default: %(default)s)')
return parser


def main():
args = get_parser().parse_args()
if args.dest_dir:
pkgdir = args.dest_dir.joinpath(*args.package.split("."))
pkgdir.mkdir(parents=True, exist_ok=True)
dst = pkgdir / f"{CLASS}.java"
with dst.open("w", encoding="utf-8") as f:
generate(f, args.package)
print(f" Generate: {dst}")
else:
generate(sys.stdout, args.package)


if __name__ == "__main__":
main()
20 changes: 20 additions & 0 deletions core/core.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ def genJavaDir = "$genOutputDir/java/main"
def genTestJavaDir = "$genOutputDir/java/test"

def genInputDir = getProject().file('src/main/javaTemplate')

// Destination of opcode constants generated from the supported CPython
def genOpcodeJavaDir = "$genSourcesDir/opcodes/java/main"
def genTestInputDir = getProject().file('src/test/javaTemplate')


Expand All @@ -51,6 +54,7 @@ sourceSets {
main {
java {
srcDir genJavaDir
srcDir genOpcodeJavaDir
}
}

Expand Down Expand Up @@ -91,12 +95,28 @@ task generateObjectDefinitions(type: PythonExec,
compileJava.dependsOn(generateObjectDefinitions)


// Generate opcode constants (OpcodeNNN.java) from the supported CPython

task generateOpcodes(type: PythonExec,
description: 'Generate opcode constants from CPython') {
def tool = getProject().getRootProject().file(
'build-tools/python/tool/opcode_gen.py')
// Clues for the incremental build
inputs.file(tool)
outputs.dir(genOpcodeJavaDir)
args tool
args genOpcodeJavaDir
}
compileJava.dependsOn(generateOpcodes)


// Report the version of Python available

task pythonVersion(type: PythonExec) {
args '-V'
}
generateObjectDefinitions.dependsOn(pythonVersion)
generateOpcodes.dependsOn(pythonVersion)


// Copy and compile Python source examples to the build tree
Expand Down
6 changes: 0 additions & 6 deletions core/src/main/java/org/python/core/CPython311Frame.java
Original file line number Diff line number Diff line change
Expand Up @@ -471,12 +471,6 @@ Object eval() {
break;
}

case Opcode311.JUMP_BACKWARD_QUICK: {
// Same as plain JUMP_BACKWARD for us
ip -= oparg;
break;
}

case Opcode311.LOAD_METHOD:
/*
* Emitted when compiling obj.meth(...). Works in tandem with CALL.
Expand Down
Loading
Loading