Skip to content

Commit aaf42c8

Browse files
trxcllntwesm
trxcllnt
authored and
wesm
committed
Consolidated JS data handling refactor
Change-Id: I423b49d58842a88b8a26c7fa646ed4771f9e31a0
1 parent 46b1bc7 commit aaf42c8

160 files changed

Lines changed: 17169 additions & 12420 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.travis.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ matrix:
229229
- if [ $ARROW_CI_INTEGRATION_AFFECTED != "1" ]; then exit; fi
230230
- $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh
231231
- $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
232-
- nvm install 10.1
232+
- nvm install 11.6
233233
- $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
234234
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
235235
script:
@@ -239,7 +239,7 @@ matrix:
239239
language: node_js
240240
os: linux
241241
node_js:
242-
- '10.1'
242+
- '11.6'
243243
before_script:
244244
- if [ $ARROW_CI_JS_AFFECTED != "1" ]; then exit; fi
245245
- $TRAVIS_BUILD_DIR/ci/travis_install_linux.sh

‎ci/travis_script_integration.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ pushd $ARROW_JS_DIR
3636

3737
# lint and compile JS source
3838
npm run lint
39-
npm run build
39+
npm run build -- -t apache-arrow
4040

4141
popd
4242

‎ci/travis_script_js.sh‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,10 @@ source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
2323

2424
pushd $ARROW_JS_DIR
2525

26-
npm run lint
26+
npm run lint:ci
2727
npm run build
28-
# run the non-snapshot unit tests
2928
npm test
29+
npm run test:coverage
30+
bash <(curl -s https://codecov.io/bash) || echo "Codecov did not collect coverage reports"
3031

3132
popd

‎integration/integration_test.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1122,7 +1122,7 @@ def _run(self, exe_cmd, arrow_path=None, json_path=None,
11221122
if json_path is not None:
11231123
cmd.extend(['-j', json_path])
11241124

1125-
cmd.extend(['--mode', command, '-t', 'es5', '-m', 'umd'])
1125+
cmd.extend(['--mode', command])
11261126

11271127
if self.debug:
11281128
print(' '.join(cmd))

‎js/.gitignore‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ npm-debug.log*
2323
yarn-debug.log*
2424
yarn-error.log*
2525

26-
.vscode
26+
.vscode/**
27+
!.vscode/launch.json
2728

2829
# Runtime data
2930
pids
@@ -78,10 +79,14 @@ yarn.lock
7879
.env
7980

8081
# compilation targets
82+
doc
8183
dist
8284
targets
8385

8486
# test data files
85-
test/data/
87+
test/data/cpp
88+
test/data/java
89+
test/data/json
90+
8691
# jest snapshots (too big)
8792
test/__snapshots__/

‎js/.vscode/launch.json‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
{
2+
// Use IntelliSense to learn about possible attributes.
3+
// Hover to view descriptions of existing attributes.
4+
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
5+
"version": "0.2.0",
6+
"configurations": [
7+
{
8+
"type": "node",
9+
"request": "launch",
10+
"name": "Debug Gulp Build",
11+
"program": "${workspaceFolder}/node_modules/gulp/bin/gulp.js",
12+
"args": [
13+
"build",
14+
// Specify we want to debug the "src" target, which won't clean or build -- essentially a "dry-run" of the gulp build
15+
"--target", "src"
16+
]
17+
},
18+
{
19+
"type": "node",
20+
"request": "launch",
21+
"name": "Debug Unit Tests",
22+
"cwd": "${workspaceRoot}",
23+
"program": "${workspaceFolder}/node_modules/.bin/jest",
24+
"skipFiles": [
25+
"<node_internals>/**/*.js",
26+
"${workspaceFolder}/node_modules/**/*.js"
27+
],
28+
"env": {
29+
"NODE_NO_WARNINGS": "1",
30+
"READABLE_STREAM": "disable",
31+
"TEST_DOM_STREAMS": "true",
32+
"TEST_NODE_STREAMS": "true",
33+
// Modify these environment variables to run tests on a specific compilation target + module format combo
34+
"TEST_TS_SOURCE": "true",
35+
// "TEST_TS_SOURCE": "false",
36+
// "TEST_TARGET": "es5",
37+
// "TEST_MODULE": "umd"
38+
},
39+
"args": [
40+
// "-i",
41+
"test/unit/",
42+
43+
// Uncomment any of these to run individual test suites
44+
// "test/unit/int-tests.ts",
45+
// "test/unit/table-tests.ts",
46+
// "test/unit/generated-data-tests.ts",
47+
48+
// "test/unit/vector/vector-tests.ts",
49+
// "test/unit/vector/bool-vector-tests.ts",
50+
// "test/unit/vector/date-vector-tests.ts",
51+
// "test/unit/vector/float16-vector-tests.ts",
52+
// "test/unit/vector/numeric-vector-tests.ts",
53+
54+
// "test/unit/visitor-tests.ts",
55+
56+
// "test/unit/ipc/message-reader-tests.ts",
57+
// "test/unit/ipc/reader/file-reader-tests.ts",
58+
// "test/unit/ipc/reader/json-reader-tests.ts",
59+
// "test/unit/ipc/reader/from-inference-tests.ts",
60+
// "test/unit/ipc/reader/stream-reader-tests.ts",
61+
// "test/unit/ipc/reader/streams-dom-tests.ts",
62+
// "test/unit/ipc/reader/streams-node-tests.ts",
63+
// "test/unit/ipc/writer/file-writer-tests.ts",
64+
// "test/unit/ipc/writer/json-writer-tests.ts",
65+
// "test/unit/ipc/writer/stream-writer-tests.ts",
66+
// "test/unit/ipc/writer/streams-dom-tests.ts",
67+
// "test/unit/ipc/writer/streams-node-tests.ts",
68+
]
69+
},
70+
{
71+
"type": "node",
72+
"request": "launch",
73+
"name": "Debug Integration Tests",
74+
"cwd": "${workspaceRoot}",
75+
"program": "${workspaceFolder}/bin/integration.js",
76+
"skipFiles": [
77+
"<node_internals>/**/*.js",
78+
"${workspaceFolder}/node_modules/**/*.js"
79+
],
80+
"env": {
81+
"NODE_NO_WARNINGS": "1",
82+
"READABLE_STREAM": "disable"
83+
},
84+
"args": [
85+
"--mode", "VALIDATE"
86+
]
87+
},
88+
{
89+
"type": "node",
90+
"request": "launch",
91+
"name": "Debug bin/arrow2csv",
92+
"env": { "ARROW_JS_DEBUG": "src", "TS_NODE_CACHE": "false" },
93+
"runtimeArgs": ["-r", "ts-node/register"],
94+
"console": "integratedTerminal",
95+
"skipFiles": [
96+
"<node_internals>/**/*.js",
97+
"${workspaceFolder}/node_modules/**/*.js"
98+
],
99+
"args": [
100+
"${workspaceFolder}/src/bin/arrow2csv.ts",
101+
"-f", "./test/data/cpp/stream/simple.arrow"
102+
]
103+
},
104+
{
105+
"type": "node",
106+
"request": "launch",
107+
"name": "Debug bin/file-to-stream",
108+
"env": { "ARROW_JS_DEBUG": "src", "TS_NODE_CACHE": "false" },
109+
"runtimeArgs": ["-r", "ts-node/register"],
110+
"skipFiles": [
111+
"<node_internals>/**/*.js",
112+
"${workspaceFolder}/node_modules/**/*.js"
113+
],
114+
"args": [
115+
"${workspaceFolder}/bin/file-to-stream.js",
116+
"./test/data/cpp/file/struct_example.arrow",
117+
"./struct_example-stream-out.arrow",
118+
]
119+
},
120+
{
121+
"type": "node",
122+
"request": "launch",
123+
"name": "Debug bin/stream-to-file",
124+
"env": { "ARROW_JS_DEBUG": "src", "TS_NODE_CACHE": "false" },
125+
"runtimeArgs": ["-r", "ts-node/register"],
126+
"skipFiles": [
127+
"<node_internals>/**/*.js",
128+
"${workspaceFolder}/node_modules/**/*.js"
129+
],
130+
"args": [
131+
"${workspaceFolder}/bin/stream-to-file.js",
132+
"./test/data/cpp/stream/struct_example.arrow",
133+
"./struct_example-file-out.arrow",
134+
]
135+
},
136+
{
137+
"type": "node",
138+
"request": "launch",
139+
"name": "Debug bin/json-to-arrow",
140+
"env": { "ARROW_JS_DEBUG": "src", "TS_NODE_CACHE": "false" },
141+
"runtimeArgs": ["-r", "ts-node/register"],
142+
"skipFiles": [
143+
"<node_internals>/**/*.js",
144+
"${workspaceFolder}/node_modules/**/*.js"
145+
],
146+
"args": [
147+
"${workspaceFolder}/bin/json-to-arrow.js",
148+
"-j", "./test/data/json/struct_example.json",
149+
"-a", "./struct_example-stream-out.arrow",
150+
"-f", "stream"
151+
]
152+
},
153+
{
154+
"type": "node",
155+
"request": "launch",
156+
"name": "Debug bin/print-buffer-alignment",
157+
"env": { "ARROW_JS_DEBUG": "src", "TS_NODE_CACHE": "false" },
158+
"runtimeArgs": ["-r", "ts-node/register"],
159+
"skipFiles": [
160+
"<node_internals>/**/*.js",
161+
"${workspaceFolder}/node_modules/**/*.js"
162+
],
163+
"args": [
164+
"${workspaceFolder}/bin/print-buffer-alignment.js",
165+
"./test/data/cpp/stream/struct_example.arrow"
166+
]
167+
}
168+
]
169+
}

‎js/README.md‎

Lines changed: 26 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ Check out our [API documentation][7] to learn more about how to use Apache Arrow
4949

5050
### Get a table from an Arrow file on disk (in IPC format)
5151

52-
```es6
52+
```js
5353
import { readFileSync } from 'fs';
5454
import { Table } from 'apache-arrow';
5555

@@ -70,7 +70,7 @@ null, null, null
7070

7171
### Create a Table when the Arrow file is split across buffers
7272

73-
```es6
73+
```js
7474
import { readFileSync } from 'fs';
7575
import { Table } from 'apache-arrow';
7676

@@ -93,33 +93,42 @@ console.log(table.toString());
9393

9494
### Create a Table from JavaScript arrays
9595

96-
```es6
96+
```js
97+
import {
98+
Table,
99+
FloatVector,
100+
DateVector
101+
} from 'apache-arrow';
102+
97103
const LENGTH = 2000;
98-
const rainAmounts = Float32Array.from({length: LENGTH}, () => Number((Math.random() * 20).toFixed(1)));
99-
const rainDates = Array.from({length: LENGTH}, (_, i) => new Date(Date.now() - 1000 * 60 * 60 * 24 * i));
100104

101-
const rainfall = arrow.Table.fromVectors(
105+
const rainAmounts = Float32Array.from(
106+
{ length: LENGTH },
107+
() => Number((Math.random() * 20).toFixed(1)));
108+
109+
const rainDates = Array.from(
110+
{ length: LENGTH },
111+
(_, i) => new Date(Date.now() - 1000 * 60 * 60 * 24 * i));
112+
113+
const rainfall = Table.fromVectors(
102114
[FloatVector.from(rainAmounts), DateVector.from(rainDates)],
103115
['precipitation', 'date']
104116
);
105117
```
106118

107119
### Load data with `fetch`
108120

109-
```es6
121+
```js
110122
import { Table } from "apache-arrow";
111123

112-
fetch(require("simple.arrow")).then(response => {
113-
response.arrayBuffer().then(buffer => {
114-
const table = Table.from(new Uint8Array(buffer));
115-
console.log(table.toString());
116-
});
117-
});
124+
const table = await Table.from(fetch(("/simple.arrow")));
125+
console.log(table.toString());
126+
118127
```
119128

120129
### Columns look like JS Arrays
121130

122-
```es6
131+
```js
123132
import { readFileSync } from 'fs';
124133
import { Table } from 'apache-arrow';
125134

@@ -131,7 +140,7 @@ const table = Table.from([
131140
const column = table.getColumn('origin_lat');
132141

133142
// Copy the data into a TypedArray
134-
const typed = column.slice();
143+
const typed = column.toArray();
135144
assert(typed instanceof Float32Array);
136145

137146
for (let i = -1, n = column.length; ++i < n;) {
@@ -141,7 +150,7 @@ for (let i = -1, n = column.length; ++i < n;) {
141150

142151
### Usage with MapD Core
143152

144-
```es6
153+
```js
145154
import MapD from 'rxjs-mapd';
146155
import { Table } from 'apache-arrow';
147156

@@ -164,7 +173,7 @@ MapD.open(host, port)
164173
)
165174
.map(([schema, records]) =>
166175
// Create Arrow Table from results
167-
Table.from(schema, records))
176+
Table.from([schema, records]))
168177
.map((table) =>
169178
// Stringify the table to CSV with row numbers
170179
table.toString({ index: true }))

‎js/bin/arrow2csv.js‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#! /usr/bin/env node
2+
3+
// Licensed to the Apache Software Foundation (ASF) under one
4+
// or more contributor license agreements. See the NOTICE file
5+
// distributed with this work for additional information
6+
// regarding copyright ownership. The ASF licenses this file
7+
// to you under the Apache License, Version 2.0 (the
8+
// "License"); you may not use this file except in compliance
9+
// with the License. You may obtain a copy of the License at
10+
//
11+
// http://www.apache.org/licenses/LICENSE-2.0
12+
//
13+
// Unless required by applicable law or agreed to in writing,
14+
// software distributed under the License is distributed on an
15+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
// KIND, either express or implied. See the License for the
17+
// specific language governing permissions and limitations
18+
// under the License.
19+
20+
const Path = require(`path`);
21+
const here = Path.resolve(__dirname, '../');
22+
const tsnode = require.resolve(`ts-node/register`);
23+
const arrow2csv = Path.join(here, `src/bin/arrow2csv.ts`);
24+
25+
require('child_process').spawn(`node`, [
26+
`-r`, tsnode, arrow2csv, ...process.argv.slice(2)
27+
], { cwd: here, env: process.env, stdio: `inherit` });

0 commit comments

Comments
 (0)