Skip to content

Commit 767c281

Browse files
sbinetwesm
sbinet
authored and
wesm
committed
[Go] add support for nullbitmap with offset
The new implementation has a moderate speed bump: name old time/op new time/op delta CountSetBits_3-8 8.16ns ± 1% 8.18ns ± 0% +0.31% (p=0.000 n=19+18) CountSetBits_32-8 7.62ns ± 1% 7.79ns ± 1% +2.24% (p=0.000 n=18+18) CountSetBits_128-8 6.80ns ± 1% 7.03ns ± 1% +3.40% (p=0.000 n=18+19) CountSetBits_1000-8 18.7ns ± 0% 17.8ns ± 1% -4.92% (p=0.000 n=16+20) CountSetBits_1024-8 16.1ns ± 0% 16.1ns ± 1% +0.44% (p=0.000 n=20+20) name old alloc/op new alloc/op delta CountSetBits_3-8 0.00B 0.00B ~ (all equal) CountSetBits_32-8 0.00B 0.00B ~ (all equal) CountSetBits_128-8 0.00B 0.00B ~ (all equal) CountSetBits_1000-8 0.00B 0.00B ~ (all equal) CountSetBits_1024-8 0.00B 0.00B ~ (all equal) name old allocs/op new allocs/op delta CountSetBits_3-8 0.00 0.00 ~ (all equal) CountSetBits_32-8 0.00 0.00 ~ (all equal) CountSetBits_128-8 0.00 0.00 ~ (all equal) CountSetBits_1000-8 0.00 0.00 ~ (all equal) CountSetBits_1024-8 0.00 0.00 ~ (all equal)
1 parent cb6cdbb commit 767c281

13 files changed

Lines changed: 171 additions & 43 deletions

‎go/arrow/array/array_test.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -62,14 +62,14 @@ func TestMakeFromData(t *testing.T) {
6262
{name: "timestamp", d: &testDataType{arrow.TIMESTAMP}},
6363

6464
{name: "list", d: &testDataType{arrow.LIST}, child: []*array.Data{
65-
array.NewData(&testDataType{arrow.INT64}, 0, make([]*memory.Buffer, 4), nil, 0),
66-
array.NewData(&testDataType{arrow.INT64}, 0, make([]*memory.Buffer, 4), nil, 0),
65+
array.NewData(&testDataType{arrow.INT64}, 0, make([]*memory.Buffer, 4), nil, 0, 0),
66+
array.NewData(&testDataType{arrow.INT64}, 0, make([]*memory.Buffer, 4), nil, 0, 0),
6767
}},
6868

6969
{name: "struct", d: &testDataType{arrow.STRUCT}},
7070
{name: "struct", d: &testDataType{arrow.STRUCT}, child: []*array.Data{
71-
array.NewData(&testDataType{arrow.INT64}, 0, make([]*memory.Buffer, 4), nil, 0),
72-
array.NewData(&testDataType{arrow.INT64}, 0, make([]*memory.Buffer, 4), nil, 0),
71+
array.NewData(&testDataType{arrow.INT64}, 0, make([]*memory.Buffer, 4), nil, 0, 0),
72+
array.NewData(&testDataType{arrow.INT64}, 0, make([]*memory.Buffer, 4), nil, 0, 0),
7373
}},
7474

7575
// invalid types
@@ -83,7 +83,7 @@ func TestMakeFromData(t *testing.T) {
8383
if test.size != 0 {
8484
n = test.size
8585
}
86-
data := array.NewData(test.d, 0, b[:n], test.child, 0)
86+
data := array.NewData(test.d, 0, b[:n], test.child, 0, 0)
8787

8888
if test.expPanic {
8989
assert.PanicsWithValue(t, test.expError, func() {
@@ -116,7 +116,7 @@ func TestArray_NullN(t *testing.T) {
116116
for _, test := range tests {
117117
t.Run(test.name, func(t *testing.T) {
118118
buf := memory.NewBufferBytes(test.bm)
119-
data := array.NewData(arrow.FixedWidthTypes.Boolean, test.l, []*memory.Buffer{buf, nil}, nil, test.n)
119+
data := array.NewData(arrow.FixedWidthTypes.Boolean, test.l, []*memory.Buffer{buf, nil}, nil, test.n, 0)
120120
buf.Release()
121121
ar := array.MakeFromData(data)
122122
data.Release()

‎go/arrow/array/binarybuilder.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ func (b *BinaryBuilder) NewBinaryArray() (a *Binary) {
163163
func (b *BinaryBuilder) newData() (data *Data) {
164164
b.appendNextOffset()
165165
offsets, values := b.offsets.Finish(), b.values.Finish()
166-
data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, offsets, values}, nil, b.nulls)
166+
data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, offsets, values}, nil, b.nulls, 0)
167167
if offsets != nil {
168168
offsets.Release()
169169
}

‎go/arrow/array/boolean.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ type Boolean struct {
3232
// The nullBitmap buffer can be nil of there are no null values.
3333
// If nulls is not known, use UnknownNullCount to calculate the value of NullN at runtime from the nullBitmap buffer.
3434
func NewBoolean(length int, data *memory.Buffer, nullBitmap *memory.Buffer, nulls int) *Boolean {
35-
return NewBooleanData(NewData(arrow.FixedWidthTypes.Boolean, length, []*memory.Buffer{nullBitmap, data}, nil, nulls))
35+
return NewBooleanData(NewData(arrow.FixedWidthTypes.Boolean, length, []*memory.Buffer{nullBitmap, data}, nil, nulls, 0))
3636
}
3737

3838
func NewBooleanData(data *Data) *Boolean {

‎go/arrow/array/booleanbuilder.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -144,7 +144,7 @@ func (b *BooleanBuilder) newData() *Data {
144144
// trim buffers
145145
b.data.Resize(bytesRequired)
146146
}
147-
res := NewData(arrow.FixedWidthTypes.Boolean, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls)
147+
res := NewData(arrow.FixedWidthTypes.Boolean, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0)
148148
b.reset()
149149

150150
if b.data != nil {

‎go/arrow/array/data.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,13 @@ type Data struct {
2929
refCount int64
3030
dtype arrow.DataType
3131
nulls int
32+
offset int
3233
length int
3334
buffers []*memory.Buffer // TODO(sgc): should this be an interface?
3435
childData []*Data // TODO(sgc): managed by ListArray, StructArray and UnionArray types
3536
}
3637

37-
func NewData(dtype arrow.DataType, length int, buffers []*memory.Buffer, childData []*Data, nulls int) *Data {
38+
func NewData(dtype arrow.DataType, length int, buffers []*memory.Buffer, childData []*Data, nulls, offset int) *Data {
3839
for _, b := range buffers {
3940
if b != nil {
4041
b.Retain()
@@ -52,6 +53,7 @@ func NewData(dtype arrow.DataType, length int, buffers []*memory.Buffer, childDa
5253
dtype: dtype,
5354
nulls: nulls,
5455
length: length,
56+
offset: offset,
5557
buffers: buffers,
5658
childData: childData,
5759
}

‎go/arrow/array/list.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,7 @@ func (b *ListBuilder) newData() (data *Data) {
199199
},
200200
[]*Data{values.Data()},
201201
b.nulls,
202+
0,
202203
)
203204
b.reset()
204205

‎go/arrow/array/numeric.gen.go.tmpl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,4 +45,4 @@ func (a *{{.Name}}) setData(data *Data) {
4545
a.values = arrow.{{.Name}}Traits.CastFromBytes(vals.Bytes())
4646
}
4747
}
48-
{{end}}
48+
{{end}}

‎go/arrow/array/numeric_test.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,11 @@ import (
2828
func TestNewFloat64Data(t *testing.T) {
2929
exp := []float64{1.0, 2.0, 4.0, 8.0, 16.0}
3030

31-
ad := array.NewData(arrow.PrimitiveTypes.Float64, len(exp), []*memory.Buffer{nil, memory.NewBufferBytes(arrow.Float64Traits.CastToBytes(exp))}, nil, 0)
31+
ad := array.NewData(
32+
arrow.PrimitiveTypes.Float64, len(exp),
33+
[]*memory.Buffer{nil, memory.NewBufferBytes(arrow.Float64Traits.CastToBytes(exp))},
34+
nil, 0, 0,
35+
)
3236
fa := array.NewFloat64Data(ad)
3337

3438
assert.Equal(t, len(exp), fa.Len(), "unexpected Len()")

‎go/arrow/array/numericbuilder.gen.go‎

Lines changed: 11 additions & 11 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎go/arrow/array/numericbuilder.gen.go.tmpl‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -155,9 +155,9 @@ func (b *{{.Name}}Builder) newData() (data *Data) {
155155
b.data.Resize(bytesRequired)
156156
}
157157
{{if .Opt.Parametric -}}
158-
data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls)
158+
data = NewData(b.dtype, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0)
159159
{{else -}}
160-
data = NewData(arrow.PrimitiveTypes.{{.Name}}, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls)
160+
data = NewData(arrow.PrimitiveTypes.{{.Name}}, b.length, []*memory.Buffer{b.nullBitmap, b.data}, nil, b.nulls, 0)
161161
{{end -}}
162162
b.reset()
163163

0 commit comments

Comments
 (0)