microsoft · DonJayamanne · Apr 22, 2019 · Apr 22, 2019 · Apr 22, 2019 · Apr 22, 2019
diff --git a/.gitignore b/.gitignore
@@ -16,6 +16,7 @@ npm-debug.log
 coverage/
 .vscode-test/**
 **/.venv*/
+port.txt
 precommit.hook
 pythonFiles/experimental/ptvsd/**
 pythonFiles/lib/**

diff --git a/news/2 Fixes/5129.md b/news/2 Fixes/5129.md
@@ -0,0 +1 @@
+Reliably end test tasks in Azure Pipelines.
diff --git a/package.json b/package.json
@@ -2115,10 +2115,10 @@
         "test:unittests:cover": "npm run test:unittests",
         "test:functional": "mocha --require source-map-support/register --opts ./build/.mocha.functional.opts",
         "test:functional:cover": "npm run test:functional",
-        "testDebugger": "node ./out/test/debuggerTest.js",
-        "testSingleWorkspace": "node ./out/test/standardTest.js",
-        "testMultiWorkspace": "node ./out/test/multiRootTest.js",
-        "testPerformance": "node ./out/test/performanceTest.js",
+        "testDebugger": "node ./out/test/testBootstrap.js ./out/test/debuggerTest.js",
+        "testSingleWorkspace": "node ./out/test/testBootstrap.js ./out/test/standardTest.js",
+        "testMultiWorkspace": "node ./out/test/testBootstrap.js ./out/test/multiRootTest.js",
+        "testPerformance": "node ./out/test/testBootstrap.js ./out/test/performanceTest.js",
         "testSmoke": "node ./out/test/smokeTest.js",
         "lint-staged": "node gulpfile.js",
         "lint": "tslint src/**/*.ts -t verbose",

diff --git a/src/test/common/exitCIAfterTestReporter.ts b/src/test/common/exitCIAfterTestReporter.ts
@@ -8,45 +8,65 @@
 // The hack is to force it to die when tests are done, if this doesn't work we've got a bigger problem on our hands.
 
 // tslint:disable:no-var-requires no-require-imports no-any no-console no-unnecessary-class no-default-export
-const log = require('why-is-node-running');
+import * as fs from 'fs-extra';
+import * as net from 'net';
+import * as path from 'path';
+import { EXTENSION_ROOT_DIR } from '../../client/constants';
+import { noop } from '../core';
+
+let client: net.Socket | undefined;
 const mochaTests: any = require('mocha');
 const { EVENT_RUN_BEGIN, EVENT_RUN_END } = mochaTests.Runner.constants;
+
+async function connectToServer() {
+    const portFile = path.join(EXTENSION_ROOT_DIR, 'port.txt');
+    if (!(await fs.pathExists(portFile))) {
+        return;
+    }
+    const port = parseInt(await fs.readFile(portFile, 'utf-8'), 10);
+    console.log(`Need to connect to port ${port}`);
+    return new Promise(resolve => {
+        try {
+            client = new net.Socket();
+            client.connect({ port }, () => {
+                console.log(`Connected to port ${port}`);
+                resolve();
+            });
+        } catch {
+            console.error('Failed to connect to socket server to notify completion of tests');
+            resolve();
+        }
+    });
+}
+function notifyCompleted(hasFailures: boolean) {
+    if (!client || client.destroyed || !client.writable) {
+        console.error('No client to write from');
+        return;
+    }
+    try {
+        const exitCode = hasFailures ? 1 : 0;
+        console.log(`Notify server of test completion with code ${exitCode}`);
+        // If there are failures, send a code of 1 else 0.
+        client.write(exitCode.toString());
+        client.end();
+        console.log('Notified server of test completion');
+    } catch (ex) {
+        console.error('Socket client error', ex);
+    }
+}
+
 class ExitReporter {
     constructor(runner: any) {
         console.log('Initialize Exit Reporter for Mocha (PVSC).');
+        connectToServer().catch(noop);
         const stats = runner.stats;
         runner
             .once(EVENT_RUN_BEGIN, () => {
                 console.info('Start Exit Reporter for Mocha.');
             })
-            .once(EVENT_RUN_END, () => {
-                process.stdout.cork();
+            .once(EVENT_RUN_END, async () => {
+                notifyCompleted(stats.failures > 0);
                 console.info('End Exit Reporter for Mocha.');
-                process.stdout.write('If process does not die in 30s, then log and kill.');
-                process.stdout.uncork();
-                // NodeJs generally waits for pending timeouts, however the process running Mocha
-                // No idea why it times, out. Once again, this is a hack.
-                // Solution (i.e. hack), lets add a timeout with a delay of 30 seconds,
-                // & if this process doesn't die, lets kill it.
-                function die() {
-                    setTimeout(() => {
-                        console.info('Exiting from custom PVSC Mocha Reporter.');
-                        try {
-                            log();
-                        } catch (ex) {
-                            // Do nothing.
-                        }
-                        process.exit(stats.failures === 0 ? 0 : 1);
-                        try {
-                            // Lets just close VSC, hopefully that'll be sufficient (more graceful).
-                            const vscode = require('vscode');
-                            vscode.commands.executeCommand('workbench.action.closeWindow');
-                        } catch (ex) {
-                            // Do nothing.
-                        }
-                    }, 30000);
-                }
-                die();
             });
     }
 }

diff --git a/src/test/debugger/attach.ptvsd.test.ts b/src/test/debugger/attach.ptvsd.test.ts
@@ -16,6 +16,7 @@ import { IS_WINDOWS } from '../../client/common/platform/constants';
 import { FileSystem } from '../../client/common/platform/fileSystem';
 import { IPlatformService } from '../../client/common/platform/types';
 import { IConfigurationService } from '../../client/common/types';
+import { MultiStepInputFactory } from '../../client/common/utils/multiStepInput';
 import { DebuggerTypeName, PTVSD_PATH } from '../../client/debugger/constants';
 import { PythonDebugConfigurationService } from '../../client/debugger/extension/configuration/debugConfigurationService';
 import { AttachConfigurationResolver } from '../../client/debugger/extension/configuration/resolvers/attach';
@@ -25,7 +26,6 @@ import { IServiceContainer } from '../../client/ioc/types';
 import { PYTHON_PATH, sleep } from '../common';
 import { IS_MULTI_ROOT_TEST, TEST_DEBUGGER } from '../initialize';
 import { continueDebugging, createDebugAdapter } from './utils';
-import { MultiStepInputFactory } from '../../client/common/utils/multiStepInput';
 
 // tslint:disable:no-invalid-this max-func-body-length no-empty no-increment-decrement no-unused-variable no-console
 const fileToDebug = path.join(EXTENSION_ROOT_DIR, 'src', 'testMultiRootWkspc', 'workspace5', 'remoteDebugger-start-with-ptvsd.py');

diff --git a/src/test/testBootstrap.ts b/src/test/testBootstrap.ts
@@ -0,0 +1,103 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+'use strict';
+
+import { ChildProcess, spawn, SpawnOptions } from 'child_process';
+import * as fs from 'fs-extra';
+import { createServer, Server } from 'net';
+import * as path from 'path';
+import { EXTENSION_ROOT_DIR } from '../client/constants';
+import { noop, sleep } from './core';
+
+// tslint:disable:no-console
+
+/*
+This is a simple work around for tests tasks not completing on Azure Pipelines.
+What's been happening is, the tests run however for some readon the Node propcess (VS Code) does not exit.
+Here's what we've tried thus far:
+* Dispose all timers
+* Close all open streams/sockets.
+* Use `process.exit` and use the VSC commands to close itself.
+
+Final solution:
+* Start a node.js procecss
+    * This process will start a socket server
+    * This procecss will start the tests in a separate procecss (spawn)
+* When the tests have completed,
+    * Send a message to the socket server with a flag (true/false whether tests passed/failed)
+* Socket server (main procecss) will receive the test status flag.
+    * This will kill the spawned process
+    * This main process will kill itself with exit code 0 if tests pass succesfully, else 1.
+*/
+
+const testFile = process.argv[2];
+const portFile = path.join(EXTENSION_ROOT_DIR, 'port.txt');
+
+let proc: ChildProcess | undefined;
+let server: Server | undefined;
+
+async function deletePortFile() {
+    try {
+        if (await fs.pathExists(portFile)) {
+            await fs.unlink(portFile);
+        }
+    } catch {
+        noop();
+    }
+}
+async function end(exitCode: number) {
+    if (exitCode === 0) {
+        console.log('Exiting without errors');
+    } else {
+        console.error('Exiting with test failures');
+    }
+    if (proc) {
+        try {
+            const procToKill = proc;
+            proc = undefined;
+            console.log('Killing VSC');
+            await deletePortFile();
+            // Wait for the std buffers to get flushed before killing.
+            await sleep(5_000);
+            procToKill.kill();
+        } catch {
+            noop();
+        }
+    }
+    if (server) {
+        server.close();
+    }
+    // Exit with required code.
+    process.exit(exitCode);
+}
+
+async function startSocketServer() {
+    return new Promise(resolve => {
+        server = createServer(socket => {
+            socket.on('data', buffer => {
+                const data = buffer.toString('utf8');
+                console.log(`Exit code from Tests is ${data}`);
+                const code = parseInt(data.substring(0, 1), 10);
+                end(code).catch(noop);
+            });
+        });
+
+        server.listen({ host: '127.0.0.1', port: 0 }, async () => {
+            const port = server!.address().port;
+            console.log(`Test server listening on port ${port}`);
+            await deletePortFile();
+            await fs.writeFile(portFile, port.toString());
+            resolve();
+        });
+    });
+}
+
+async function start() {
+    await startSocketServer();
+    const options: SpawnOptions = { cwd: process.cwd(), env: process.env, detached: true, stdio: 'inherit' };
+    proc = spawn(process.execPath, [testFile], options);
+    proc.once('close', end);
+}
+
+start().catch(ex => console.error(ex));