Skip to content

Commit 68763ae

Browse files
TH3CHARLieTH3CHARLie
authored
[mypyc] Implement builtins.len primitive for list (python#9271)
* implement list len primitive * use pointer type * rename * add PyObject * use CPyPtr * revert RStruct design and fix tests * list_len helper and updates according to review comments * fix * remove size_t_to_short_int
1 parent 1bbcd53 commit 68763ae

File tree

15 files changed

+417
-388
lines changed

15 files changed

+417
-388
lines changed

‎mypyc/codegen/emitfunc.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -477,7 +477,8 @@ def visit_get_element_ptr(self, op: GetElementPtr) -> None:
477477
# TODO: support tuple type
478478
assert isinstance(op.src_type, RStruct)
479479
assert op.field in op.src_type.names, "Invalid field name."
480-
self.emit_line('%s = &%s.%s;' % (dest, src, op.field))
480+
self.emit_line('%s = (%s)&((%s *)%s)->%s;' % (dest, op.type._ctype, op.src_type.name,
481+
src, op.field))
481482

482483
# Helpers
483484

‎mypyc/ir/ops.py‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,7 @@
2525
from mypyc.ir.rtypes import (
2626
RType, RInstance, RTuple, RVoid, is_bool_rprimitive, is_int_rprimitive,
2727
is_short_int_rprimitive, is_none_rprimitive, object_rprimitive, bool_rprimitive,
28-
short_int_rprimitive, int_rprimitive, void_rtype, is_c_py_ssize_t_rprimitive,
29-
c_pyssize_t_rprimitive
28+
short_int_rprimitive, int_rprimitive, void_rtype, pointer_rprimitive, is_pointer_rprimitive
3029
)
3130
from mypyc.common import short_name
3231

@@ -1360,7 +1359,7 @@ def __init__(self, type: RType, src: Value, line: int = -1) -> None:
13601359
self.type = type
13611360
# TODO: for now we enforce that the src memory address should be Py_ssize_t
13621361
# later we should also support same width unsigned int
1363-
assert is_c_py_ssize_t_rprimitive(src.type)
1362+
assert is_pointer_rprimitive(src.type)
13641363
self.src = src
13651364

13661365
def sources(self) -> List[Value]:
@@ -1379,7 +1378,7 @@ class GetElementPtr(RegisterOp):
13791378

13801379
def __init__(self, src: Value, src_type: RType, field: str, line: int = -1) -> None:
13811380
super().__init__(line)
1382-
self.type = c_pyssize_t_rprimitive
1381+
self.type = pointer_rprimitive
13831382
self.src = src
13841383
self.src_type = src_type
13851384
self.field = field
@@ -1388,7 +1387,7 @@ def sources(self) -> List[Value]:
13881387
return [self.src]
13891388

13901389
def to_str(self, env: Environment) -> str:
1391-
return env.format("%r = get_element_ptr %r %r :: %r", self, self.src,
1390+
return env.format("%r = get_element_ptr %r %s :: %r", self, self.src,
13921391
self.field, self.src_type)
13931392

13941393
def accept(self, visitor: 'OpVisitor[T]') -> T:

‎mypyc/ir/rtypes.py‎

Lines changed: 28 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -182,8 +182,10 @@ def __init__(self,
182182
self.size = size
183183
# TODO: For low-level integers, they actually don't have undefined values
184184
# we need to figure out some way to represent here.
185-
if ctype in ('CPyTagged', 'int32_t', 'int64_t'):
185+
if ctype == 'CPyTagged':
186186
self.c_undefined = 'CPY_INT_TAG'
187+
elif ctype in ('int32_t', 'int64_t', 'CPyPtr'):
188+
self.c_undefined = '0'
187189
elif ctype == 'PyObject *':
188190
# Boxed types use the null pointer as the error value.
189191
self.c_undefined = 'NULL'
@@ -254,6 +256,10 @@ def __repr__(self) -> str:
254256
else:
255257
c_pyssize_t_rprimitive = int64_rprimitive
256258

259+
# low level pointer, represented as integer in C backends
260+
pointer_rprimitive = RPrimitive('ptr', is_unboxed=True, is_refcounted=False,
261+
ctype='CPyPtr') # type: Final
262+
257263
# Floats are represent as 'float' PyObject * values. (In the future
258264
# we'll likely switch to a more efficient, unboxed representation.)
259265
float_rprimitive = RPrimitive('builtins.float', is_unboxed=False,
@@ -311,6 +317,10 @@ def is_c_py_ssize_t_rprimitive(rtype: RType) -> bool:
311317
return rtype is c_pyssize_t_rprimitive
312318

313319

320+
def is_pointer_rprimitive(rtype: RType) -> bool:
321+
return rtype is pointer_rprimitive
322+
323+
314324
def is_float_rprimitive(rtype: RType) -> bool:
315325
return isinstance(rtype, RPrimitive) and rtype.name == 'builtins.float'
316326

@@ -514,12 +524,8 @@ def compute_aligned_offsets_and_size(types: List[RType]) -> Tuple[List[int], int
514524
return offsets, final_size
515525

516526

517-
class StructInfo:
518-
"""Struct-like type Infomation
519-
520-
StructInfo should work with registry to ensure constraints like the unique naming
521-
constraint for struct type
522-
"""
527+
class RStruct(RType):
528+
"""Represent CPython structs"""
523529
def __init__(self,
524530
name: str,
525531
names: List[str],
@@ -532,31 +538,7 @@ def __init__(self,
532538
for i in range(len(self.types) - len(self.names)):
533539
self.names.append('_item' + str(i))
534540
self.offsets, self.size = compute_aligned_offsets_and_size(types)
535-
536-
537-
class RStruct(RType):
538-
"""Represent CPython structs"""
539-
def __init__(self,
540-
info: StructInfo) -> None:
541-
self.info = info
542-
self.name = self.info.name
543-
self._ctype = self.info.name
544-
545-
@property
546-
def names(self) -> List[str]:
547-
return self.info.names
548-
549-
@property
550-
def types(self) -> List[RType]:
551-
return self.info.types
552-
553-
@property
554-
def offsets(self) -> List[int]:
555-
return self.info.offsets
556-
557-
@property
558-
def size(self) -> int:
559-
return self.info.size
541+
self._ctype = name
560542

561543
def accept(self, visitor: 'RTypeVisitor[T]') -> T:
562544
return visitor.visit_rstruct(self)
@@ -571,10 +553,11 @@ def __repr__(self) -> str:
571553
in zip(self.names, self.types)))
572554

573555
def __eq__(self, other: object) -> bool:
574-
return isinstance(other, RStruct) and self.info == other.info
556+
return (isinstance(other, RStruct) and self.name == other.name
557+
and self.names == other.names and self.types == other.types)
575558

576559
def __hash__(self) -> int:
577-
return hash(self.info)
560+
return hash((self.name, tuple(self.names), tuple(self.types)))
578561

579562
def serialize(self) -> JsonDict:
580563
assert False
@@ -687,3 +670,14 @@ def optional_value_type(rtype: RType) -> Optional[RType]:
687670
def is_optional_type(rtype: RType) -> bool:
688671
"""Is rtype an optional type with exactly two union items?"""
689672
return optional_value_type(rtype) is not None
673+
674+
675+
PyObject = RStruct(
676+
name='PyObject',
677+
names=['ob_refcnt', 'ob_type'],
678+
types=[c_pyssize_t_rprimitive, pointer_rprimitive])
679+
680+
PyVarObject = RStruct(
681+
name='PyVarObject',
682+
names=['ob_base', 'ob_size'],
683+
types=[PyObject, c_pyssize_t_rprimitive])

‎mypyc/irbuild/builder.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444
from mypyc.ir.func_ir import FuncIR, INVALID_FUNC_DEF
4545
from mypyc.ir.class_ir import ClassIR, NonExtClassInfo
4646
from mypyc.primitives.registry import func_ops, CFunctionDescription, c_function_ops
47-
from mypyc.primitives.list_ops import list_len_op, to_list, list_pop_last
47+
from mypyc.primitives.list_ops import to_list, list_pop_last
4848
from mypyc.primitives.dict_ops import dict_get_item_op, dict_set_item_op
4949
from mypyc.primitives.generic_ops import py_setattr_op, iter_op, next_op
5050
from mypyc.primitives.misc_ops import true_op, false_op, import_op
@@ -238,6 +238,9 @@ def binary_int_op(self, type: RType, lhs: Value, rhs: Value, op: int, line: int)
238238
def compare_tagged(self, lhs: Value, rhs: Value, op: str, line: int) -> Value:
239239
return self.builder.compare_tagged(lhs, rhs, op, line)
240240

241+
def list_len(self, val: Value, line: int) -> Value:
242+
return self.builder.list_len(val, line)
243+
241244
@property
242245
def environment(self) -> Environment:
243246
return self.builder.environment
@@ -508,7 +511,7 @@ def process_iterator_tuple_assignment(self,
508511
if target.star_idx is not None:
509512
post_star_vals = target.items[split_idx + 1:]
510513
iter_list = self.call_c(to_list, [iterator], line)
511-
iter_list_len = self.primitive_op(list_len_op, [iter_list], line)
514+
iter_list_len = self.list_len(iter_list, line)
512515
post_star_len = self.add(LoadInt(len(post_star_vals)))
513516
condition = self.binary_op(post_star_len, iter_list_len, '<=', line)
514517

‎mypyc/irbuild/for_helpers.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
from mypyc.primitives.exc_ops import no_err_occurred_op
3030
from mypyc.irbuild.builder import IRBuilder
3131

32-
3332
GenFunc = Callable[[], None]
3433

3534

@@ -333,6 +332,9 @@ def gen_cleanup(self) -> None:
333332

334333
def load_len(self, expr: Union[Value, AssignmentTarget]) -> Value:
335334
"""A helper to get collection length, used by several subclasses."""
335+
val = self.builder.read(expr, self.line)
336+
if is_list_rprimitive(val.type):
337+
return self.builder.builder.list_len(self.builder.read(expr, self.line), self.line)
336338
return self.builder.builder.builtin_call(
337339
[self.builder.read(expr, self.line)],
338340
'builtins.len',

‎mypyc/irbuild/ll_builder.py‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,13 @@
2121
Assign, Branch, Goto, Call, Box, Unbox, Cast, GetAttr,
2222
LoadStatic, MethodCall, PrimitiveOp, OpDescription, RegisterOp, CallC, Truncate,
2323
RaiseStandardError, Unreachable, LoadErrorValue, LoadGlobal,
24-
NAMESPACE_TYPE, NAMESPACE_MODULE, NAMESPACE_STATIC, BinaryIntOp
24+
NAMESPACE_TYPE, NAMESPACE_MODULE, NAMESPACE_STATIC, BinaryIntOp, GetElementPtr,
25+
LoadMem
2526
)
2627
from mypyc.ir.rtypes import (
2728
RType, RUnion, RInstance, optional_value_type, int_rprimitive, float_rprimitive,
2829
bool_rprimitive, list_rprimitive, str_rprimitive, is_none_rprimitive, object_rprimitive,
29-
c_pyssize_t_rprimitive, is_short_int_rprimitive, is_tagged
30+
c_pyssize_t_rprimitive, is_short_int_rprimitive, is_tagged, PyVarObject, short_int_rprimitive
3031
)
3132
from mypyc.ir.func_ir import FuncDecl, FuncSignature
3233
from mypyc.ir.class_ir import ClassIR, all_concrete_classes
@@ -40,7 +41,7 @@
4041
c_binary_ops, c_unary_ops
4142
)
4243
from mypyc.primitives.list_ops import (
43-
list_extend_op, list_len_op, new_list_op
44+
list_extend_op, new_list_op
4445
)
4546
from mypyc.primitives.tuple_ops import list_tuple_op, new_tuple_op
4647
from mypyc.primitives.dict_ops import (
@@ -703,7 +704,7 @@ def add_bool_branch(self, value: Value, true: BasicBlock, false: BasicBlock) ->
703704
zero = self.add(LoadInt(0))
704705
value = self.binary_op(value, zero, '!=', value.line)
705706
elif is_same_type(value.type, list_rprimitive):
706-
length = self.primitive_op(list_len_op, [value], value.line)
707+
length = self.list_len(value, value.line)
707708
zero = self.add(LoadInt(0))
708709
value = self.binary_op(length, zero, '!=', value.line)
709710
elif (isinstance(value.type, RInstance) and value.type.class_ir.is_ext_class
@@ -810,6 +811,12 @@ def matching_call_c(self,
810811
def binary_int_op(self, type: RType, lhs: Value, rhs: Value, op: int, line: int) -> Value:
811812
return self.add(BinaryIntOp(type, lhs, rhs, op, line))
812813

814+
def list_len(self, val: Value, line: int) -> Value:
815+
elem_address = self.add(GetElementPtr(val, PyVarObject, 'ob_size'))
816+
size_value = self.add(LoadMem(c_pyssize_t_rprimitive, elem_address))
817+
offset = self.add(LoadInt(1, -1, rtype=c_pyssize_t_rprimitive))
818+
return self.binary_int_op(short_int_rprimitive, size_value, offset,
819+
BinaryIntOp.LEFT_SHIFT, -1)
813820
# Internal helpers
814821

815822
def decompose_union_helper(self,

‎mypyc/irbuild/specialize.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,11 @@
1818
from mypy.types import AnyType, TypeOfAny
1919

2020
from mypyc.ir.ops import (
21-
Value, BasicBlock, LoadInt, RaiseStandardError, Unreachable, OpDescription
21+
Value, BasicBlock, LoadInt, RaiseStandardError, Unreachable, OpDescription,
2222
)
2323
from mypyc.ir.rtypes import (
2424
RType, RTuple, str_rprimitive, list_rprimitive, dict_rprimitive, set_rprimitive,
25-
bool_rprimitive, is_dict_rprimitive
25+
bool_rprimitive, is_dict_rprimitive, is_list_rprimitive,
2626
)
2727
from mypyc.primitives.dict_ops import dict_keys_op, dict_values_op, dict_items_op
2828
from mypyc.primitives.misc_ops import true_op, false_op
@@ -75,6 +75,9 @@ def translate_len(
7575
# though we still need to evaluate it.
7676
builder.accept(expr.args[0])
7777
return builder.add(LoadInt(len(expr_rtype.types)))
78+
elif is_list_rprimitive(expr_rtype):
79+
obj = builder.accept(expr.args[0])
80+
return builder.list_len(obj, -1)
7881
return None
7982

8083

‎mypyc/lib-rt/mypyc_util.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
#define CPy_XDECREF(p) Py_XDECREF(p)
3333

3434
typedef size_t CPyTagged;
35+
typedef size_t CPyPtr;
3536

3637
#define CPY_INT_BITS (CHAR_BIT * sizeof(CPyTagged))
3738

‎mypyc/primitives/list_ops.py‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
c_int_rprimitive
99
)
1010
from mypyc.primitives.registry import (
11-
name_ref_op, func_op, custom_op, name_emit,
11+
name_ref_op, custom_op, name_emit,
1212
call_emit, c_function_op, c_binary_op, c_method_op
1313
)
1414

@@ -146,11 +146,3 @@ def emit_len(emitter: EmitterInterface, args: List[str], dest: str) -> None:
146146
emitter.emit_declaration('Py_ssize_t %s;' % temp)
147147
emitter.emit_line('%s = PyList_GET_SIZE(%s);' % (temp, args[0]))
148148
emitter.emit_line('%s = CPyTagged_ShortFromSsize_t(%s);' % (dest, temp))
149-
150-
151-
# len(list)
152-
list_len_op = func_op(name='builtins.len',
153-
arg_types=[list_rprimitive],
154-
result_type=short_int_rprimitive,
155-
error_kind=ERR_NEVER,
156-
emit=emit_len)

‎mypyc/primitives/struct_regsitry.py‎

Lines changed: 0 additions & 24 deletions
This file was deleted.

0 commit comments

Comments
 (0)