Skip to content

Commit 5489bbf

Browse files
trxcllntkszucs
trxcllnt
authored and
kszucs
committed
ARROW-10255: [JS] Reorganize exports for ESM tree-shaking
Related JIRA: https://issues.apache.org/jira/browse/ARROW-10255 Opening this as a draft PR for ongoing work to make our ESM exports more friendly to [tree-shaking](https://webpack.js.org/guides/tree-shaking/). Closes apache#8418 from trxcllnt/fea/tree-shaking Lead-authored-by: ptaylor <paul.e.taylor@me.com> Co-authored-by: Paul Taylor <paul.e.taylor@me.com> Signed-off-by: Krisztián Szűcs <szucs.krisztian@gmail.com>
1 parent 3258f12 commit 5489bbf

21 files changed

Lines changed: 3283 additions & 2659 deletions

‎js/DEVELOP.md‎

Lines changed: 36 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
-->
1919

2020
# Getting Involved
21+
2122
Even if you do not plan to contribute to Apache Arrow itself or Arrow
2223
integrations in other projects, we'd be happy to have you involved:
2324

@@ -68,29 +69,41 @@ Uses [lerna](https://github.com/lerna/lerna) to publish each build target to npm
6869

6970
1. Once generated, the flatbuffers format code needs to be adjusted for our build scripts (assumes `gnu-sed`):
7071

71-
```shell
72-
cd $ARROW_HOME
73-
74-
flatc --ts -o ./js/src/fb ./format/{File,Schema,Message}.fbs
75-
76-
cd ./js/src/fb
77-
78-
# Rename the existing files to <filename>.bak.ts
79-
mv File{,.bak}.ts && mv Schema{,.bak}.ts && mv Message{,.bak}.ts
80-
81-
# Remove `_generated` from the ES6 imports of the generated files
82-
sed -i '+s+_generated\";+\";+ig' *_generated.ts
83-
# Fix all the `flatbuffers` imports
84-
sed -i '+s+./flatbuffers+flatbuffers+ig' *_generated.ts
85-
# Fix the Union createTypeIdsVector typings
86-
sed -i -r '+s+static createTypeIdsVector\(builder: flatbuffers.Builder, data: number\[\] \| Uint8Array+static createTypeIdsVector\(builder: flatbuffers.Builder, data: number\[\] \| Int32Array+ig' Schema_generated.ts
87-
# Add `/* tslint:disable:class-name */` to the top of `Schema.ts`
88-
echo -e '/* tslint:disable:class-name */\n' | cat - Schema_generated.ts > Schema1.ts && mv Schema1.ts Schema_generated.ts
89-
# Remove "_generated" suffix from TS files
90-
mv File{_generated,}.ts && mv Schema{_generated,}.ts && mv Message{_generated,}.ts
91-
```
92-
2. Manually remove `Tensor` and `SparseTensor` imports and exports
93-
3. Execute `npm run lint` from the `js` directory to fix the linting errors
72+
```shell
73+
cd $ARROW_HOME
74+
75+
# Create a tmpdir to store modified flatbuffers schemas
76+
tmp_format_dir=$(mktemp -d)
77+
cp ./format/*.fbs $tmp_format_dir
78+
79+
# Remove namespaces from the flatbuffers schemas
80+
sed -i '+s+namespace org.apache.arrow.flatbuf;++ig' $tmp_format_dir/*.fbs
81+
sed -i '+s+org.apache.arrow.flatbuf.++ig' $tmp_format_dir/*.fbs
82+
83+
# Generate TS source from the modified Arrow flatbuffers schemas
84+
flatc --ts --no-ts-reexport -o ./js/src/fb $tmp_format_dir/{File,Schema,Message}.fbs
85+
86+
# Remove the tmpdir
87+
rm -rf $tmp_format_dir
88+
89+
cd ./js/src/fb
90+
91+
# Rename the existing files to <filename>.bak.ts
92+
mv File{,.bak}.ts && mv Schema{,.bak}.ts && mv Message{,.bak}.ts
93+
94+
# Remove `_generated` from the ES6 imports of the generated files
95+
sed -i '+s+_generated\";+\";+ig' *_generated.ts
96+
# Fix all the `flatbuffers` imports
97+
sed -i '+s+./flatbuffers+flatbuffers+ig' *_generated.ts
98+
# Fix the Union createTypeIdsVector typings
99+
sed -i -r '+s+static createTypeIdsVector\(builder: flatbuffers.Builder, data: number\[\] \| Uint8Array+static createTypeIdsVector\(builder: flatbuffers.Builder, data: number\[\] \| Int32Array+ig' Schema_generated.ts
100+
# Add `/* tslint:disable:class-name */` to the top of `Schema.ts`
101+
echo -e '/* tslint:disable:class-name */\n' | cat - Schema_generated.ts > Schema1.ts && mv Schema1.ts Schema_generated.ts
102+
# Remove "_generated" suffix from TS files
103+
mv File{_generated,}.ts && mv Schema{_generated,}.ts && mv Message{_generated,}.ts
104+
```
105+
106+
2. Execute `npm run lint` from the `js` directory to fix the linting errors
94107

95108
[1]: mailto:dev-subscribe@arrow.apache.org
96109
[2]: https://github.com/apache/arrow/tree/master/format

‎js/gulp/package-task.js‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,8 @@ const createMainPackageJson = (target, format) => (orig) => ({
5252
types: `${mainExport}.node.d.ts`,
5353
unpkg: `${mainExport}.es5.min.js`,
5454
jsdelivr: `${mainExport}.es5.min.js`,
55-
esm: { mode: `all`, sourceMap: true }
55+
sideEffects: false,
56+
esm: { mode: `all`, sourceMap: true },
5657
});
5758

5859
const createTypeScriptPackageJson = (target, format) => (orig) => ({
@@ -86,6 +87,8 @@ const createScopedPackageJSON = (target, format) => (({ name, ...orig }) =>
8687
main: format === 'umd' ? `${mainExport}.js` : `${mainExport}.node`,
8788
// set "module" (for https://www.npmjs.com/package/@pika/pack) if building scoped ESM target
8889
module: format === 'esm' ? `${mainExport}.dom.js` : undefined,
90+
// set "sideEffects" to false as a hint to Webpack that it's safe to tree-shake the ESM target
91+
sideEffects: format === 'esm' ? false : undefined,
8992
// include "esm" settings for https://www.npmjs.com/package/esm if building scoped ESM target
9093
esm: format === `esm` ? { mode: `auto`, sourceMap: true } : undefined,
9194
// set "types" (for TypeScript/VSCode)

‎js/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@
6262
"@types/text-encoding-utf-8": "^1.0.1",
6363
"command-line-args": "5.0.2",
6464
"command-line-usage": "5.0.5",
65-
"flatbuffers": "1.11.0",
65+
"flatbuffers": "1.12.0",
6666
"json-bignum": "^0.0.3",
6767
"pad-left": "^2.1.0",
6868
"text-encoding-utf-8": "^1.0.2",

‎js/src/Arrow.dom.ts‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@
1717

1818
import streamAdapters from './io/adapters';
1919
import { Builder } from './builder/index';
20-
import { RecordBatchReader } from './ipc/reader';
21-
import { RecordBatchWriter } from './ipc/writer';
20+
import { RecordBatchReader, RecordBatchFileReader, RecordBatchStreamReader, } from './ipc/reader';
21+
import { RecordBatchWriter, RecordBatchFileWriter, RecordBatchStreamWriter, } from './ipc/writer';
2222
import { toDOMStream } from './io/whatwg/iterable';
2323
import { builderThroughDOMStream } from './io/whatwg/builder';
2424
import { recordBatchReaderThroughDOMStream } from './io/whatwg/reader';
@@ -27,10 +27,14 @@ import { recordBatchWriterThroughDOMStream } from './io/whatwg/writer';
2727
streamAdapters.toDOMStream = toDOMStream;
2828
Builder['throughDOM'] = builderThroughDOMStream;
2929
RecordBatchReader['throughDOM'] = recordBatchReaderThroughDOMStream;
30+
RecordBatchFileReader['throughDOM'] = recordBatchReaderThroughDOMStream;
31+
RecordBatchStreamReader['throughDOM'] = recordBatchReaderThroughDOMStream;
3032
RecordBatchWriter['throughDOM'] = recordBatchWriterThroughDOMStream;
33+
RecordBatchFileWriter['throughDOM'] = recordBatchWriterThroughDOMStream;
34+
RecordBatchStreamWriter['throughDOM'] = recordBatchWriterThroughDOMStream;
3135

3236
export {
33-
ArrowType, DateUnit, IntervalUnit, MessageHeader, MetadataVersion, Precision, TimeUnit, Type, UnionMode, BufferType,
37+
DateUnit, IntervalUnit, MessageHeader, MetadataVersion, Precision, TimeUnit, Type, UnionMode, BufferType,
3438
Data,
3539
DataType,
3640
Null,

‎js/src/Arrow.ts‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,19 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
export { ArrowType, DateUnit, IntervalUnit, MessageHeader, MetadataVersion, Precision, TimeUnit, Type, UnionMode, BufferType } from './enum';
18+
export {
19+
DateUnit,
20+
TimeUnit,
21+
Precision,
22+
UnionMode,
23+
IntervalUnit,
24+
MetadataVersion,
25+
} from './fb/Schema';
26+
27+
export { MessageHeader } from './fb/Message';
28+
29+
export { Type, BufferType } from './enum';
30+
1931
export { Data } from './data';
2032
export {
2133
DataType,
@@ -105,6 +117,7 @@ import * as util_math_ from './util/math';
105117
import * as util_buffer_ from './util/buffer';
106118
import * as util_vector_ from './util/vector';
107119
import * as predicate from './compute/predicate';
120+
import { compareSchemas, compareFields, compareTypes } from './visitor/typecomparator';
108121

109122
export { predicate };
110123
/** @ignore */
@@ -114,5 +127,8 @@ export const util = {
114127
...util_bit_,
115128
...util_math_,
116129
...util_buffer_,
117-
...util_vector_
130+
...util_vector_,
131+
compareSchemas,
132+
compareFields,
133+
compareTypes,
118134
};

‎js/src/enum.ts‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,16 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
import * as Schema_ from './fb/Schema';
19-
import * as Message_ from './fb/Message';
18+
export {
19+
DateUnit,
20+
TimeUnit,
21+
Precision,
22+
UnionMode,
23+
IntervalUnit,
24+
MetadataVersion,
25+
} from './fb/Schema';
2026

21-
export import ArrowType = Schema_.org.apache.arrow.flatbuf.Type;
22-
export import DateUnit = Schema_.org.apache.arrow.flatbuf.DateUnit;
23-
export import TimeUnit = Schema_.org.apache.arrow.flatbuf.TimeUnit;
24-
export import Precision = Schema_.org.apache.arrow.flatbuf.Precision;
25-
export import UnionMode = Schema_.org.apache.arrow.flatbuf.UnionMode;
26-
export import IntervalUnit = Schema_.org.apache.arrow.flatbuf.IntervalUnit;
27-
export import MessageHeader = Message_.org.apache.arrow.flatbuf.MessageHeader;
28-
export import MetadataVersion = Schema_.org.apache.arrow.flatbuf.MetadataVersion;
27+
export { MessageHeader } from './fb/Message';
2928

3029
/**
3130
* Main data type enumeration.

0 commit comments

Comments
 (0)