Skip to content

Commit f33e294

Browse files
eerhardteerhardt
committed
ARROW-4502: [C#] Add support for zero-copy reads
Update to the latest Google FlatBuffers code to support Spans/Memory. Add a constructor for ArrowStreamReader that takes a ReadOnlyMemory<byte>. Add a synchronous ReadNextRecordBatch() method. Since we are now enabling Spans with FlatBuffers, we need to change the way we write to streams in the ArrowStreamWriter to use Memory<byte> instead of byte[]. This API is in netcoreapp2.1, but not in netstandard, so cross compile for netcoreapp2.1 and add a shim for netstandard.
1 parent dd3ba0c commit f33e294

12 files changed

Lines changed: 959 additions & 580 deletions

‎csharp/src/Apache.Arrow/Apache.Arrow.csproj‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<Import Project="../../build/Common.props" />
44

55
<PropertyGroup>
6-
<TargetFramework>netstandard1.3</TargetFramework>
6+
<TargetFrameworks>netstandard1.3;netcoreapp2.1</TargetFrameworks>
77
<LangVersion>7.2</LangVersion>
88
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
99
<Authors>Apache</Authors>
@@ -15,6 +15,7 @@
1515
<PackageTags>apache arrow</PackageTags>
1616
<Company>Apache</Company>
1717
<Version>0.0.1</Version>
18+
<DefineConstants>$(DefineConstants);UNSAFE_BYTEBUFFER;BYTEBUFFER_NO_BOUNDS_CHECK;ENABLE_SPAN_T</DefineConstants>
1819
</PropertyGroup>
1920

2021
<ItemGroup>
@@ -39,4 +40,7 @@
3940
</EmbeddedResource>
4041
</ItemGroup>
4142

43+
<ItemGroup Condition="'$(TargetFramework)' == 'netcoreapp2.1'">
44+
<Compile Remove="Extensions\StreamExtensions.netstandard.cs" />
45+
</ItemGroup>
4246
</Project>

‎csharp/src/Apache.Arrow/ArrowBuffer.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ namespace Apache.Arrow
2323
{
2424
public static ArrowBuffer Empty => new ArrowBuffer(Memory<byte>.Empty);
2525

26-
private ArrowBuffer(Memory<byte> data)
26+
internal ArrowBuffer(ReadOnlyMemory<byte> data)
2727
{
2828
Memory = data;
2929
}

‎csharp/src/Apache.Arrow/Extensions/StreamExtensions.netstandard.cs‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one or more
2+
// contributor license agreements. See the NOTICE file distributed with
3+
// this work for additional information regarding copyright ownership.
4+
// The ASF licenses this file to You under the Apache License, Version 2.0
5+
// (the "License"); you may not use this file except in compliance with
6+
// the License. You may obtain a copy of the License at
7+
//
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
//
10+
// Unless required by applicable law or agreed to in writing, software
11+
// distributed under the License is distributed on an "AS IS" BASIS,
12+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
// See the License for the specific language governing permissions and
14+
// limitations under the License.
15+
16+
using System;
17+
using System.Buffers;
18+
using System.IO;
19+
using System.Runtime.InteropServices;
20+
using System.Threading;
21+
using System.Threading.Tasks;
22+
23+
namespace Apache.Arrow
24+
{
25+
// Helpers to write Memory<byte> to Stream on netstandard
26+
internal static class StreamExtensions
27+
{
28+
public static Task WriteAsync(this Stream stream, ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default)
29+
{
30+
if (MemoryMarshal.TryGetArray(buffer, out ArraySegment<byte> array))
31+
{
32+
return stream.WriteAsync(array.Array, array.Offset, array.Count, cancellationToken);
33+
}
34+
else
35+
{
36+
byte[] sharedBuffer = ArrayPool<byte>.Shared.Rent(buffer.Length);
37+
buffer.Span.CopyTo(sharedBuffer);
38+
return FinishWriteAsync(stream.WriteAsync(sharedBuffer, 0, buffer.Length, cancellationToken), sharedBuffer);
39+
}
40+
}
41+
42+
private static async Task FinishWriteAsync(Task writeTask, byte[] localBuffer)
43+
{
44+
try
45+
{
46+
await writeTask.ConfigureAwait(false);
47+
}
48+
finally
49+
{
50+
ArrayPool<byte>.Shared.Return(localBuffer);
51+
}
52+
}
53+
}
54+
}

0 commit comments

Comments
 (0)