forked from apache/arrow
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathprint-buffer-alignment.js
More file actions
executable file
·122 lines (110 loc) · 4.81 KB
/
Copy pathprint-buffer-alignment.js
File metadata and controls
executable file
·122 lines (110 loc) · 4.81 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
#! /usr/bin/env node
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
// @ts-check
const fs = require('fs');
const path = require('path');
const extension = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : '.cjs';
const { VectorLoader } = require(`../targets/apache-arrow/visitor/vectorloader`);
const { RecordBatch, AsyncMessageReader, makeData, Struct, Schema, Field } = require(`../index${extension}`);
(async () => {
const readable = process.argv.length < 3 ? process.stdin : fs.createReadStream(path.resolve(process.argv[2]));
const reader = new AsyncMessageReader(readable);
let schema, metadataLength, message;
let byteOffset = 0;
let recordBatchCount = 0;
let dictionaryBatchCount = 0;
while (1) {
if ((metadataLength = (await reader.readMetadataLength())).done) { break; }
if (metadataLength.value === -1) {
if ((metadataLength = (await reader.readMetadataLength())).done) { break; }
}
if ((message = (await reader.readMetadata(metadataLength.value))).done) { break; }
if (message.value.isSchema()) {
console.log(
`Schema:`,
{
byteOffset,
metadataLength: metadataLength.value,
});
schema = message.value.header();
byteOffset += metadataLength.value;
} else if (message.value.isRecordBatch()) {
const header = message.value.header();
const bufferRegions = header.buffers;
const body = await reader.readMessageBody(message.value.bodyLength);
const recordBatch = loadRecordBatch(schema, header, body);
console.log(
`RecordBatch ${++recordBatchCount}:`,
{
numRows: recordBatch.numRows,
byteOffset,
metadataLength: metadataLength.value,
bodyByteLength: body.byteLength,
});
byteOffset += metadataLength.value;
bufferRegions.forEach(({ offset, length: byteLength }, i) => {
console.log(`\tbuffer ${i + 1}:`, { byteOffset: byteOffset + offset, byteLength });
});
byteOffset += body.byteLength;
} else if (message.value.isDictionaryBatch()) {
const header = message.value.header();
const bufferRegions = header.data.buffers;
const type = schema.dictionaries.get(header.id);
const body = await reader.readMessageBody(message.value.bodyLength);
const recordBatch = loadDictionaryBatch(header.data, body, type);
console.log(
`DictionaryBatch ${++dictionaryBatchCount}:`,
{
id: header.id,
numRows: recordBatch.numRows,
byteOffset,
metadataLength: metadataLength.value,
bodyByteLength: body.byteLength,
});
byteOffset += metadataLength.value;
bufferRegions.forEach(({ offset, length: byteLength }, i) => {
console.log(`\tbuffer ${i + 1}:`, { byteOffset: byteOffset + offset, byteLength });
});
byteOffset += body.byteLength;
}
}
await reader.return();
})().catch((e) => { console.error(e); process.exit(1); });
function loadRecordBatch(schema, header, body) {
const children = new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany(schema.fields);
return new RecordBatch(
schema,
makeData({
type: new Struct(schema.fields),
length: header.length,
children: children
})
);
}
function loadDictionaryBatch(header, body, dictionaryType) {
const schema = new Schema([new Field('', dictionaryType)]);
const children = new VectorLoader(body, header.nodes, header.buffers, new Map()).visitMany([dictionaryType]);
return new RecordBatch(
schema,
makeData({
type: new Struct(schema.fields),
length: header.length,
children: children
})
);
}