fix stream -> file, file -> stream, add tests

trxcllnt · trxcllnt · commit 73a2fa9b86cb · 2018-05-13T16:30:33.000-07:00
diff --git a/js/src/ipc/writer/binary.ts b/js/src/ipc/writer/binary.ts
@@ -268,7 +268,7 @@ class RecordBatchSerializer extends VectorVisitor {
         const { data, length } = vector;
         const { offset, values, valueOffsets } = data;
         const firstOffset = valueOffsets[0];
-        const lastOffset = valueOffsets[valueOffsets.length - 1];
+        const lastOffset = valueOffsets[length];
         const byteLength = Math.min(lastOffset - firstOffset, values.byteLength - firstOffset);
         // Push in the order FlatList types read their buffers
         // valueOffsets buffer first
@@ -536,9 +536,9 @@ export class TypeSerializer extends TypeVisitor {
 function concatBuffersWithMetadata(byteLength: number, buffers: Uint8Array[], buffersMeta: BufferMetadata[]) {
     const data = new Uint8Array(byteLength);
     for (let bufferIndex = -1, buffersLen = buffers.length; ++bufferIndex < buffersLen;) {
-        const { buffer, byteLength } = buffers[bufferIndex];
-        const { offset: byteOffset } = buffersMeta[bufferIndex];
-        data.set(new Uint8Array(buffer, 0, byteLength), byteOffset);
+        const { offset } = buffersMeta[bufferIndex];
+        const { buffer, byteOffset, byteLength } = buffers[bufferIndex];
+        data.set(new Uint8Array(buffer, byteOffset, byteLength), offset);
     }
     return data;
 }
diff --git a/js/test/integration/validate-tests.ts b/js/test/integration/validate-tests.ts
@@ -55,7 +55,7 @@ const jsonAndArrowPaths = toArray(zip(
 .filter(([p1, p2]) => p1 !== undefined && p2 !== undefined) as [string, string][];
 
 expect.extend({
-    toEqualVector(v1: any, v2: any) {
+    toEqualVector([v1, format1, columnName]: [any, string, string], [v2, format2]: [any, string]) {
 
         const format = (x: any, y: any, msg= ' ') => `${
             this.utils.printExpected(x)}${
@@ -102,7 +102,7 @@ expect.extend({
         return {
             pass: allFailures.every(({ failures }) => failures.length === 0),
             message: () => [
-                `${v1.name}: (${format('json', 'arrow', ' !== ')})\n`,
+                `${columnName}: (${format(format1, format2, ' !== ')})\n`,
                 ...allFailures.map(({ failures, title }) =>
                     !failures.length ? `` : [`${title}:`, ...failures].join(`\n`))
             ].join('\n')
@@ -119,8 +119,10 @@ describe(`Integration`, () => {
         describe(path.join(dir, name), () => {
             testReaderIntegration(json, arrowBuffer);
             testTableFromBuffersIntegration(json, arrowBuffer);
-            testTableToBuffersIntegration('file')(json, arrowBuffer);
-            testTableToBuffersIntegration('stream')(json, arrowBuffer);
+            testTableToBuffersIntegration('json', 'file')(json, arrowBuffer);
+            testTableToBuffersIntegration('json', 'stream')(json, arrowBuffer);
+            testTableToBuffersIntegration('binary', 'file')(json, arrowBuffer);
+            testTableToBuffersIntegration('binary', 'stream')(json, arrowBuffer);
         });
     }
 });
@@ -134,8 +136,11 @@ function testReaderIntegration(jsonData: any, arrowBuffer: Uint8Array) {
             expect(jsonRecordBatch.length).toEqual(binaryRecordBatch.length);
             expect(jsonRecordBatch.numCols).toEqual(binaryRecordBatch.numCols);
             for (let i = -1, n = jsonRecordBatch.numCols; ++i < n;) {
-                (jsonRecordBatch.getChildAt(i) as any).name = jsonRecordBatch.schema.fields[i].name;
-                (expect(jsonRecordBatch.getChildAt(i)) as any).toEqualVector(binaryRecordBatch.getChildAt(i));
+                const v1 = jsonRecordBatch.getChildAt(i);
+                const v2 = binaryRecordBatch.getChildAt(i);
+                const name = jsonRecordBatch.schema.fields[i].name;
+                (expect([v1, `json`, name]) as any)
+                    .toEqualVector([v2, `binary`]);
             }
         }
     });
@@ -149,25 +154,31 @@ function testTableFromBuffersIntegration(jsonData: any, arrowBuffer: Uint8Array)
         expect(jsonTable.length).toEqual(binaryTable.length);
         expect(jsonTable.numCols).toEqual(binaryTable.numCols);
         for (let i = -1, n = jsonTable.numCols; ++i < n;) {
-            (jsonTable.getColumnAt(i) as any).name = jsonTable.schema.fields[i].name;
-            (expect(jsonTable.getColumnAt(i)) as any).toEqualVector(binaryTable.getColumnAt(i));
+            const v1 = jsonTable.getColumnAt(i);
+            const v2 = binaryTable.getColumnAt(i);
+            const name = jsonTable.schema.fields[i].name;
+            (expect([v1, `json`, name]) as any)
+                .toEqualVector([v2, `binary`]);
         }
     });
 }
 
-function testTableToBuffersIntegration(arrowFormat: 'stream' | 'file') {
+function testTableToBuffersIntegration(srcFormat: 'json' | 'binary', arrowFormat: 'stream' | 'file') {
     return function testTableToBuffersIntegration(jsonData: any, arrowBuffer: Uint8Array) {
-        test(`serializing json to binary reports the same values as the original binary arrow table`, () => {
+        test(`serializing ${srcFormat} to a ${arrowFormat} reports the same values as the alternate format`, () => {
             expect.hasAssertions();
-            const fromJSON = Table.from(jsonData);
-            const serialized = fromJSON.serialize('binary', arrowFormat === 'stream');
-            const jsonTable = Table.from(serialized);
-            const binaryTable = Table.from(arrowBuffer);
-            expect(jsonTable.length).toEqual(binaryTable.length);
-            expect(jsonTable.numCols).toEqual(binaryTable.numCols);
-            for (let i = -1, n = jsonTable.numCols; ++i < n;) {
-                (jsonTable.getColumnAt(i) as any).name = jsonTable.schema.fields[i].name;
-                (expect(jsonTable.getColumnAt(i)) as any).toEqualVector(binaryTable.getColumnAt(i));
+            const refFormat = srcFormat === 'json' ? `binary` : `json`;
+            const refTable = Table.from(refFormat === `json` ? jsonData : arrowBuffer);
+            const srcTable = Table.from(srcFormat === `json` ? jsonData : arrowBuffer);
+            const dstTable = Table.from(srcTable.serialize(`binary`, arrowFormat === `stream`));
+            expect(dstTable.length).toEqual(refTable.length);
+            expect(dstTable.numCols).toEqual(refTable.numCols);
+            for (let i = -1, n = dstTable.numCols; ++i < n;) {
+                const v1 = dstTable.getColumnAt(i);
+                const v2 = refTable.getColumnAt(i);
+                const name = dstTable.schema.fields[i].name;
+                (expect([v1, srcFormat, name]) as any)
+                    .toEqualVector([v2, refFormat]);
             }
         });
     }
