Skip to content

Commit 0882d8e

Browse files
trxcllnttrxcllnt
committed
separate JS unit tests from integration tests in CI
1 parent 1f6a81b commit 0882d8e

8 files changed

Lines changed: 60 additions & 48 deletions

File tree

‎.travis.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,9 @@ matrix:
108108
- source $TRAVIS_BUILD_DIR/ci/travis_install_clang_tools.sh
109109
- export CC="clang-4.0"
110110
- export CXX="clang++-4.0"
111+
- nvm install node
111112
- $TRAVIS_BUILD_DIR/ci/travis_lint.sh
113+
- $TRAVIS_BUILD_DIR/ci/travis_before_script_js.sh
112114
- $TRAVIS_BUILD_DIR/ci/travis_before_script_cpp.sh
113115
script:
114116
- $TRAVIS_BUILD_DIR/ci/travis_script_integration.sh

‎ci/travis_script_integration.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,3 +44,19 @@ conda install -y pip numpy six
4444
python integration_test.py --debug
4545

4646
popd
47+
48+
pushd $ARROW_JS_DIR
49+
50+
# create initial test data
51+
npm run test:createTestData
52+
# run once to write the snapshots
53+
npm test -- -t ts -u --integration
54+
# run again to test all builds against the snapshots
55+
npm test -- --integration
56+
# run tests against source to generate coverage data
57+
npm run test:coverage -- --integration
58+
# Uncomment to upload to coveralls
59+
# cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js;
60+
61+
62+
popd

‎ci/travis_script_js.sh‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,21 +17,15 @@
1717
# specific language governing permissions and limitations
1818
# under the License.
1919

20-
set -e
20+
set -ex
2121

22-
JS_DIR=${TRAVIS_BUILD_DIR}/js
22+
source $TRAVIS_BUILD_DIR/ci/travis_env_common.sh
2323

24-
pushd $JS_DIR
24+
pushd $ARROW_JS_DIR
2525

2626
npm run lint
2727
npm run build
28-
# run once to write the snapshots
29-
npm test -- -t ts -u
30-
# run again to test all builds against the snapshots
28+
# run the non-snapshot unit tests
3129
npm test
32-
# run tests against source to generate coverage data
33-
npm run test:coverage
34-
# Uncomment to upload to coveralls
35-
# cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js;
3630

3731
popd

‎js/gulp/argv.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ const argv = require(`command-line-args`)([
2222
{ name: `target`, type: String, defaultValue: `` },
2323
{ name: `module`, type: String, defaultValue: `` },
2424
{ name: `coverage`, type: Boolean, defaultValue: false },
25+
{ name: `integration`, alias: `i`, type: Boolean, defaultValue: false },
2526
{ name: `targets`, alias: `t`, type: String, multiple: true, defaultValue: [] },
2627
{ name: `modules`, alias: `m`, type: String, multiple: true, defaultValue: [] },
2728
{ name: `sources`, alias: `s`, type: String, multiple: true, defaultValue: [`cpp`, `java`] },

‎js/gulp/test-task.js‎

Lines changed: 28 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -38,20 +38,23 @@ const debugArgv = [`--runInBand`, `--env`, `jest-environment-node-debug`];
3838
const jest = require.resolve(path.join(`..`, `node_modules`, `.bin`, `jest`));
3939

4040
const testTask = ((cache, execArgv, testOptions) => memoizeTask(cache, function test(target, format, debug = false) {
41-
const opts = Object.assign({}, testOptions);
41+
const opts = { ...testOptions };
4242
const args = !debug ? [...execArgv] : [...debugArgv, ...execArgv];
43-
opts.env = Object.assign({}, opts.env, {
43+
if (!argv.integration) {
44+
args.push('test/vector-tests.ts');
45+
}
46+
opts.env = { ...opts.env,
4447
TEST_TARGET: target,
4548
TEST_MODULE: format,
4649
TEST_TS_SOURCE: !!argv.coverage,
4750
TEST_SOURCES: JSON.stringify(Array.isArray(argv.sources) ? argv.sources : [argv.sources]),
4851
TEST_FORMATS: JSON.stringify(Array.isArray(argv.formats) ? argv.formats : [argv.formats]),
49-
});
52+
};
5053
return !debug ?
5154
child_process.spawn(jest, args, opts) :
5255
child_process.exec(`node --inspect-brk ${jest} ${args.join(` `)}`, opts);
5356
}))({}, jestArgv, {
54-
env: Object.assign({}, process.env),
57+
env: { ...process.env },
5558
stdio: [`ignore`, `inherit`, `inherit`],
5659
});
5760

@@ -60,9 +63,16 @@ module.exports.testTask = testTask;
6063
module.exports.cleanTestData = cleanTestData;
6164
module.exports.createTestData = createTestData;
6265

63-
const ARROW_HOME = path.resolve('../');
64-
const integrationDir = path.resolve(ARROW_HOME, 'integration');
65-
const testFilesDir = path.resolve(ARROW_HOME, 'js/test/data');
66+
// Pull C++ and Java paths from environment vars first, otherwise sane defaults
67+
const ARROW_HOME = process.env.ARROW_HOME || path.resolve('../');
68+
const ARROW_JAVA_DIR = process.env.ARROW_JAVA_DIR || path.join(ARROW_HOME, 'java');
69+
const CPP_EXE_PATH = process.env.ARROW_CPP_EXE_PATH || path.join(ARROW_HOME, 'cpp/build/debug');
70+
const ARROW_INTEGRATION_DIR = process.env.ARROW_INTEGRATION_DIR || path.join(ARROW_HOME, 'integration');
71+
const CPP_JSON_TO_ARROW = path.join(CPP_EXE_PATH, 'json-integration-test');
72+
const CPP_STREAM_TO_FILE = path.join(CPP_EXE_PATH, 'stream-to-file');
73+
const CPP_FILE_TO_STREAM = path.join(CPP_EXE_PATH, 'file-to-stream');
74+
75+
const testFilesDir = path.join(ARROW_HOME, 'js/test/data');
6676
const cppFilesDir = path.join(testFilesDir, 'cpp');
6777
const javaFilesDir = path.join(testFilesDir, 'java');
6878
const jsonFilesDir = path.join(testFilesDir, 'json');
@@ -73,34 +83,21 @@ async function cleanTestData() {
7383

7484
async function createTestJSON() {
7585
await mkdirp(jsonFilesDir);
76-
await exec(`shx cp ${integrationDir}/data/*.json ${jsonFilesDir}`);
77-
await exec(`python ${integrationDir}/integration_test.py --write_generated_json ${jsonFilesDir}`);
86+
await exec(`shx cp ${ARROW_INTEGRATION_DIR}/data/*.json ${jsonFilesDir}`);
87+
await exec(`python ${ARROW_INTEGRATION_DIR}/integration_test.py --write_generated_json ${jsonFilesDir}`);
7888
}
7989

8090
async function createTestData() {
8191

82-
// Only re-create test data if the test data folder doesn't exist
83-
// This should be the case on first checkout, and on the CI server
84-
try {
85-
const testFilesExist = await stat(testFilesDir);
86-
if (testFilesExist && testFilesExist.isDirectory()) {
87-
return;
88-
}
89-
} catch (e) {
90-
// continue
92+
let JAVA_TOOLS_JAR = process.env.ARROW_JAVA_INTEGRATION_JAR;
93+
if (!JAVA_TOOLS_JAR) {
94+
const pom_version = await
95+
readFile(path.join(ARROW_JAVA_DIR, 'pom.xml'))
96+
.then((pom) => parseXML(pom.toString()))
97+
.then((pomXML) => pomXML.project.version[0]);
98+
JAVA_TOOLS_JAR = path.join(ARROW_JAVA_DIR, `/tools/target/arrow-tools-${pom_version}-jar-with-dependencies.jar`);
9199
}
92100

93-
// Pull C++ and Java paths from environment vars first, otherwise sane defaults
94-
const CPP_EXE_PATH = process.env.ARROW_CPP_EXE_PATH || path.resolve(ARROW_HOME, 'cpp/build/debug');
95-
const CPP_JSON_TO_ARROW = path.join(CPP_EXE_PATH, 'json-integration-test');
96-
const CPP_STREAM_TO_FILE = path.join(CPP_EXE_PATH, 'stream-to-file');
97-
const CPP_FILE_TO_STREAM = path.join(CPP_EXE_PATH, 'file-to-stream');
98-
99-
const pomString = await readFile(path.join(ARROW_HOME, 'java', 'pom.xml'));
100-
const pomObject = await parseXML(pomString.toString());
101-
const _arrow_version = pomObject.project.version[0];
102-
const JAVA_TOOLS_JAR = process.env.ARROW_JAVA_INTEGRATION_JAR || path.resolve(ARROW_HOME, `java/tools/target/arrow-tools-${_arrow_version}-jar-with-dependencies.jar`);
103-
104101
await cleanTestData().then(createTestJSON);
105102
await mkdirp(path.join(cppFilesDir, 'file'));
106103
await mkdirp(path.join(javaFilesDir, 'file'));
@@ -119,11 +116,11 @@ async function createTestData() {
119116
try {
120117
await generateCPPFile(path.resolve(jsonPath), arrowCppFilePath);
121118
await generateCPPStream(arrowCppFilePath, arrowCppStreamPath);
122-
} catch (e) { errors.push(e.message); }
119+
} catch (e) { errors.push(`${e.stdout}\n${e.message}`); }
123120
try {
124121
await generateJavaFile(path.resolve(jsonPath), arrowJavaFilePath);
125122
await generateJavaStream(arrowJavaFilePath, arrowJavaStreamPath);
126-
} catch (e) { errors.push(e.message); }
123+
} catch (e) { errors.push(`${e.stdout}\n${e.message}`); }
127124
}
128125
if (errors.length) {
129126
console.error(errors.join(`\n`));

‎js/gulp/uglify-task.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ const uglifyTask = ((cache, commonConfig) => memoizeTask(cache, function uglifyJ
3939

4040
const targetConfig = { ...commonConfig,
4141
output: { ...commonConfig.output,
42-
path: path.resolve(`./${out}`) } };
42+
path: path.resolve(`./${out}`) } };
4343

4444
const webpackConfigs = [
4545
[mainExport, PublicNames]

‎js/gulpfile.js‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ knownTargets.forEach((target) =>
5050
gulp.series(
5151
gulp.parallel(
5252
cleanTask(target, `umd`),
53-
cleanTask(UMDSourceTargets[target], `cls`),
53+
cleanTask(UMDSourceTargets[target], `cls`)
5454
),
5555
buildTask(UMDSourceTargets[target], `cls`),
5656
buildTask(target, `umd`), packageTask(target, `umd`)
@@ -86,11 +86,11 @@ const buildConcurrent = (tasks) => () =>
8686
.merge(...knownTargets.map((target) =>
8787
del(`${targetDir(target, `cls`)}/**`)))));
8888

89-
gulp.task(`test:clean`, cleanTestData);
90-
gulp.task(`test:create`, createTestData);
91-
gulp.task( `test`, gulp.series(createTestData, getTasks(`test`)/*, cleanTestData*/));
92-
gulp.task(`debug`, gulp.series(createTestData, getTasks(`debug`)/*, cleanTestData*/));
93-
gulp.task(`clean`, gulp.parallel(/*cleanTestData,*/ getTasks(`clean`)));
89+
gulp.task(`test:cleanTestData`, cleanTestData);
90+
gulp.task(`test:createTestData`, createTestData);
91+
gulp.task( `test`, gulp.series(getTasks(`test`)));
92+
gulp.task(`debug`, gulp.series(getTasks(`debug`)));
93+
gulp.task(`clean`, gulp.parallel(getTasks(`clean`)));
9494
gulp.task(`build`, buildConcurrent(getTasks(`build`)));
9595
gulp.task(`default`, gulp.series(`build`, `test`));
9696

‎js/package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
"release": "./npm-release.sh",
2020
"validate": "run-s --silent lint build clean",
2121
"test:coverage": "gulp test -t ts --coverage",
22+
"test:cleanTestData": "gulp test:cleanTestData",
23+
"test:createTestData": "gulp test:createTestData",
2224
"doc": "shx rm -rf ./doc && esdoc",
2325
"lint": "npm-run-all -p lint:*",
2426
"lint:src": "tslint --fix --project -p tsconfig.json -c tslint.json \"src/**/*.ts\"",

0 commit comments

Comments
 (0)