Skip to content

Commit 498325c

Browse files
author
Benjamin Pasero
committed
Load iconv-lite module async (fix microsoft#40147)
1 parent 3edd0f9 commit 498325c

6 files changed

Lines changed: 59 additions & 58 deletions

File tree

‎src/vs/base/node/encoding.ts‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Licensed under the MIT License. See License.txt in the project root for license information.
44
*--------------------------------------------------------------------------------------------*/
55

6-
import * as iconv from 'iconv-lite';
6+
import { DecoderStream } from 'iconv-lite';
77
import { Readable, ReadableStream, newWriteableStream } from 'vs/base/common/stream';
88
import { VSBuffer, VSBufferReadable, VSBufferReadableStream } from 'vs/base/common/buffer';
99

@@ -31,7 +31,7 @@ export interface IDecodeStreamOptions {
3131
guessEncoding: boolean;
3232
minBytesRequiredForDetection?: number;
3333

34-
overwriteEncoding(detectedEncoding: string | null): string;
34+
overwriteEncoding(detectedEncoding: string | null): Promise<string>;
3535
}
3636

3737
export interface IDecodeStreamResult {
@@ -48,7 +48,7 @@ export function toDecodeStream(source: VSBufferReadableStream, options: IDecodeS
4848
const bufferedChunks: VSBuffer[] = [];
4949
let bytesBuffered = 0;
5050

51-
let decoder: iconv.DecoderStream | undefined = undefined;
51+
let decoder: DecoderStream | undefined = undefined;
5252

5353
const createDecoder = async () => {
5454
try {
@@ -60,9 +60,10 @@ export function toDecodeStream(source: VSBufferReadableStream, options: IDecodeS
6060
}, options.guessEncoding);
6161

6262
// ensure to respect overwrite of encoding
63-
detected.encoding = options.overwriteEncoding(detected.encoding);
63+
detected.encoding = await options.overwriteEncoding(detected.encoding);
6464

6565
// decode and write buffered content
66+
const iconv = await import('iconv-lite');
6667
decoder = iconv.getDecoder(toNodeEncoding(detected.encoding));
6768
const decoded = decoder.write(Buffer.from(VSBuffer.concat(bufferedChunks).buffer));
6869
target.write(decoded);
@@ -127,8 +128,10 @@ export function toDecodeStream(source: VSBufferReadableStream, options: IDecodeS
127128
});
128129
}
129130

130-
export function toEncodeReadable(readable: Readable<string>, encoding: string, options?: { addBOM?: boolean }): VSBufferReadable {
131+
export async function toEncodeReadable(readable: Readable<string>, encoding: string, options?: { addBOM?: boolean }): Promise<VSBufferReadable> {
132+
const iconv = await import('iconv-lite');
131133
const encoder = iconv.getEncoder(toNodeEncoding(encoding), options);
134+
132135
let bytesRead = 0;
133136
let done = false;
134137

@@ -172,7 +175,9 @@ export function toEncodeReadable(readable: Readable<string>, encoding: string, o
172175
};
173176
}
174177

175-
export function encodingExists(encoding: string): boolean {
178+
export async function encodingExists(encoding: string): Promise<boolean> {
179+
const iconv = await import('iconv-lite');
180+
176181
return iconv.encodingExists(toNodeEncoding(encoding));
177182
}
178183

‎src/vs/base/test/node/encoding/encoding.test.ts‎

Lines changed: 33 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ suite('Encoding', () => {
129129
process.env['VSCODE_CLI_ENCODING'] = 'utf16le';
130130

131131
const enc = await terminalEncoding.resolveTerminalEncoding();
132-
assert.ok(encoding.encodingExists(enc));
132+
assert.ok(await encoding.encodingExists(enc));
133133
assert.equal(enc, 'utf16le');
134134
});
135135

@@ -252,14 +252,13 @@ suite('Encoding', () => {
252252
}
253253

254254
test('toDecodeStream - some stream', async function () {
255-
256-
let source = newTestReadableStream([
255+
const source = newTestReadableStream([
257256
Buffer.from([65, 66, 67]),
258257
Buffer.from([65, 66, 67]),
259258
Buffer.from([65, 66, 67]),
260259
]);
261260

262-
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
261+
const { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 });
263262

264263
assert.ok(detected);
265264
assert.ok(stream);
@@ -269,14 +268,13 @@ suite('Encoding', () => {
269268
});
270269

271270
test('toDecodeStream - some stream, expect too much data', async function () {
272-
273-
let source = newTestReadableStream([
271+
const source = newTestReadableStream([
274272
Buffer.from([65, 66, 67]),
275273
Buffer.from([65, 66, 67]),
276274
Buffer.from([65, 66, 67]),
277275
]);
278276

279-
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
277+
const { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 });
280278

281279
assert.ok(detected);
282280
assert.ok(stream);
@@ -286,11 +284,10 @@ suite('Encoding', () => {
286284
});
287285

288286
test('toDecodeStream - some stream, no data', async function () {
289-
290-
let source = newWriteableBufferStream();
287+
const source = newWriteableBufferStream();
291288
source.end();
292289

293-
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 512, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
290+
const { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 512, guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 });
294291

295292
assert.ok(detected);
296293
assert.ok(stream);
@@ -301,76 +298,73 @@ suite('Encoding', () => {
301298

302299

303300
test('toDecodeStream - encoding, utf16be', async function () {
301+
const path = getPathFromAmdModule(require, './fixtures/some_utf16be.css');
302+
const source = streamToBufferReadableStream(fs.createReadStream(path));
304303

305-
let path = getPathFromAmdModule(require, './fixtures/some_utf16be.css');
306-
let source = streamToBufferReadableStream(fs.createReadStream(path));
307-
308-
let { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
304+
const { detected, stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 64, guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 });
309305

310306
assert.equal(detected.encoding, 'utf16be');
311307
assert.equal(detected.seemsBinary, false);
312308

313-
let expected = await readAndDecodeFromDisk(path, detected.encoding);
314-
let actual = await readAllAsString(stream);
309+
const expected = await readAndDecodeFromDisk(path, detected.encoding);
310+
const actual = await readAllAsString(stream);
315311
assert.equal(actual, expected);
316312
});
317313

318314

319315
test('toDecodeStream - empty file', async function () {
316+
const path = getPathFromAmdModule(require, './fixtures/empty.txt');
317+
const source = streamToBufferReadableStream(fs.createReadStream(path));
318+
const { detected, stream } = await encoding.toDecodeStream(source, { guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 });
320319

321-
let path = getPathFromAmdModule(require, './fixtures/empty.txt');
322-
let source = streamToBufferReadableStream(fs.createReadStream(path));
323-
let { detected, stream } = await encoding.toDecodeStream(source, { guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
324-
325-
let expected = await readAndDecodeFromDisk(path, detected.encoding);
326-
let actual = await readAllAsString(stream);
320+
const expected = await readAndDecodeFromDisk(path, detected.encoding);
321+
const actual = await readAllAsString(stream);
327322
assert.equal(actual, expected);
328323
});
329324

330325
test('toDecodeStream - decodes buffer entirely', async function () {
331-
let emojis = Buffer.from('🖥️💻💾');
332-
let incompleteEmojis = emojis.slice(0, emojis.length - 1);
326+
const emojis = Buffer.from('🖥️💻💾');
327+
const incompleteEmojis = emojis.slice(0, emojis.length - 1);
333328

334-
let buffers = [];
329+
const buffers: Buffer[] = [];
335330
for (let i = 0; i < incompleteEmojis.length; i++) {
336331
buffers.push(incompleteEmojis.slice(i, i + 1));
337332
}
338333

339334
const source = newTestReadableStream(buffers);
340-
let { stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: detected => detected || encoding.UTF8 });
335+
const { stream } = await encoding.toDecodeStream(source, { minBytesRequiredForDetection: 4, guessEncoding: false, overwriteEncoding: async detected => detected || encoding.UTF8 });
341336

342-
let expected = incompleteEmojis.toString(encoding.UTF8);
343-
let actual = await readAllAsString(stream);
337+
const expected = incompleteEmojis.toString(encoding.UTF8);
338+
const actual = await readAllAsString(stream);
344339

345340
assert.equal(actual, expected);
346341
});
347342

348343
test('toEncodeReadable - encoding, utf16be', async function () {
344+
const path = getPathFromAmdModule(require, './fixtures/some_utf16be.css');
345+
const source = await readAndDecodeFromDisk(path, encoding.UTF16be);
349346

350-
let path = getPathFromAmdModule(require, './fixtures/some_utf16be.css');
351-
let source = await readAndDecodeFromDisk(path, encoding.UTF16be);
352-
353-
let expected = VSBuffer.wrap(
347+
const expected = VSBuffer.wrap(
354348
iconv.encode(source, encoding.toNodeEncoding(encoding.UTF16be))
355349
).toString();
356-
let actual = streams.consumeReadable(
357-
encoding.toEncodeReadable(streams.toReadable(source), encoding.UTF16be),
350+
351+
const actual = streams.consumeReadable(
352+
await encoding.toEncodeReadable(streams.toReadable(source), encoding.UTF16be),
358353
VSBuffer.concat
359354
).toString();
360355

361356
assert.equal(actual, expected);
362357
});
363358

364359
test('toEncodeReadable - empty readable to utf8', async function () {
365-
366360
const source: streams.Readable<string> = {
367361
read() {
368362
return null;
369363
}
370364
};
371365

372-
let actual = streams.consumeReadable(
373-
encoding.toEncodeReadable(source, encoding.UTF8),
366+
const actual = streams.consumeReadable(
367+
await encoding.toEncodeReadable(source, encoding.UTF8),
374368
VSBuffer.concat
375369
).toString();
376370

@@ -391,17 +385,16 @@ suite('Encoding', () => {
391385
relatedBom: encoding.UTF16le_BOM
392386
}].forEach(({ utfEncoding, relatedBom }) => {
393387
test(`toEncodeReadable - empty readable to ${utfEncoding} with BOM`, async function () {
394-
395388
const source: streams.Readable<string> = {
396389
read() {
397390
return null;
398391
}
399392
};
400393

401-
let encodedReadable = encoding.toEncodeReadable(source, utfEncoding, { addBOM: true });
394+
const encodedReadable = encoding.toEncodeReadable(source, utfEncoding, { addBOM: true });
402395

403396
const expected = VSBuffer.wrap(Buffer.from(relatedBom)).toString();
404-
const actual = streams.consumeReadable(encodedReadable, VSBuffer.concat).toString();
397+
const actual = streams.consumeReadable(await encodedReadable, VSBuffer.concat).toString();
405398

406399
assert.equal(actual, expected);
407400
});

‎src/vs/workbench/services/textfile/browser/browserTextFileService.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import { ShutdownReason } from 'vs/platform/lifecycle/common/lifecycle';
1111
export class BrowserTextFileService extends AbstractTextFileService {
1212

1313
readonly encoding: IResourceEncodings = {
14-
getPreferredWriteEncoding(): IResourceEncoding {
14+
async getPreferredWriteEncoding(): Promise<IResourceEncoding> {
1515
return { encoding: 'utf8', hasBOM: false };
1616
}
1717
};

‎src/vs/workbench/services/textfile/common/textFileEditorModel.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,7 @@ export class TextFileEditorModel extends BaseTextEditorModel implements ITextFil
286286
}
287287

288288
private async loadFromBackup(backup: IResolvedBackup<IBackupMetaData>, options?: ITextFileLoadOptions): Promise<TextFileEditorModel> {
289+
const preferredEncoding = await this.textFileService.encoding.getPreferredWriteEncoding(this.resource, this.preferredEncoding);
289290

290291
// Load with backup
291292
this.loadFromContent({
@@ -296,7 +297,7 @@ export class TextFileEditorModel extends BaseTextEditorModel implements ITextFil
296297
size: backup.meta ? backup.meta.size : 0,
297298
etag: backup.meta ? backup.meta.etag : ETAG_DISABLED, // etag disabled if unknown!
298299
value: backup.value,
299-
encoding: this.textFileService.encoding.getPreferredWriteEncoding(this.resource, this.preferredEncoding).encoding
300+
encoding: preferredEncoding.encoding
300301
}, options, true /* from backup */);
301302

302303
// Restore orphaned flag based on state

‎src/vs/workbench/services/textfile/common/textfiles.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,7 @@ export class TextFileOperationError extends FileOperationError {
171171
}
172172

173173
export interface IResourceEncodings {
174-
getPreferredWriteEncoding(resource: URI, preferredEncoding?: string): IResourceEncoding;
174+
getPreferredWriteEncoding(resource: URI, preferredEncoding?: string): Promise<IResourceEncoding>;
175175
}
176176

177177
export interface IResourceEncoding {

‎src/vs/workbench/services/textfile/electron-browser/nativeTextFileService.ts‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,8 @@ export class NativeTextFileService extends AbstractTextFileService {
170170
}
171171

172172
// otherwise create with encoding
173-
return this.fileService.createFile(resource, this.getEncodedReadable(value || '', encoding, addBOM), options);
173+
const encodedReadable = await this.getEncodedReadable(value || '', encoding, addBOM);
174+
return this.fileService.createFile(resource, encodedReadable, options);
174175
}
175176

176177
async write(resource: URI, value: string | ITextSnapshot, options?: IWriteTextFileOptions): Promise<IFileStatWithMetadata> {
@@ -204,7 +205,8 @@ export class NativeTextFileService extends AbstractTextFileService {
204205

205206
// otherwise save with encoding
206207
else {
207-
return await this.fileService.writeFile(resource, this.getEncodedReadable(value, encoding, addBOM), options);
208+
const encodedReadable = await this.getEncodedReadable(value, encoding, addBOM);
209+
return await this.fileService.writeFile(resource, encodedReadable, options);
208210
}
209211
} catch (error) {
210212

@@ -229,7 +231,7 @@ export class NativeTextFileService extends AbstractTextFileService {
229231
}
230232
}
231233

232-
private getEncodedReadable(value: string | ITextSnapshot, encoding: string, addBOM: boolean): VSBufferReadable {
234+
private getEncodedReadable(value: string | ITextSnapshot, encoding: string, addBOM: boolean): Promise<VSBufferReadable> {
233235
const snapshot = typeof value === 'string' ? stringToSnapshot(value) : value;
234236
return toEncodeReadable(snapshot, encoding, { addBOM });
235237
}
@@ -340,7 +342,7 @@ export class EncodingOracle extends Disposable implements IResourceEncodings {
340342
}
341343

342344
async getWriteEncoding(resource: URI, options?: IWriteTextFileOptions): Promise<{ encoding: string, addBOM: boolean }> {
343-
const { encoding, hasBOM } = this.getPreferredWriteEncoding(resource, options ? options.encoding : undefined);
345+
const { encoding, hasBOM } = await this.getPreferredWriteEncoding(resource, options ? options.encoding : undefined);
344346

345347
// Some encodings come with a BOM automatically
346348
if (hasBOM) {
@@ -364,16 +366,16 @@ export class EncodingOracle extends Disposable implements IResourceEncodings {
364366
return { encoding, addBOM: false };
365367
}
366368

367-
getPreferredWriteEncoding(resource: URI, preferredEncoding?: string): IResourceEncoding {
368-
const resourceEncoding = this.getEncodingForResource(resource, preferredEncoding);
369+
async getPreferredWriteEncoding(resource: URI, preferredEncoding?: string): Promise<IResourceEncoding> {
370+
const resourceEncoding = await this.getEncodingForResource(resource, preferredEncoding);
369371

370372
return {
371373
encoding: resourceEncoding,
372374
hasBOM: resourceEncoding === UTF16be || resourceEncoding === UTF16le || resourceEncoding === UTF8_with_bom // enforce BOM for certain encodings
373375
};
374376
}
375377

376-
getReadEncoding(resource: URI, options: IReadTextFileOptions | undefined, detectedEncoding: string | null): string {
378+
getReadEncoding(resource: URI, options: IReadTextFileOptions | undefined, detectedEncoding: string | null): Promise<string> {
377379
let preferredEncoding: string | undefined;
378380

379381
// Encoding passed in as option
@@ -398,7 +400,7 @@ export class EncodingOracle extends Disposable implements IResourceEncodings {
398400
return this.getEncodingForResource(resource, preferredEncoding);
399401
}
400402

401-
private getEncodingForResource(resource: URI, preferredEncoding?: string): string {
403+
private async getEncodingForResource(resource: URI, preferredEncoding?: string): Promise<string> {
402404
let fileEncoding: string;
403405

404406
const override = this.getEncodingOverride(resource);
@@ -410,7 +412,7 @@ export class EncodingOracle extends Disposable implements IResourceEncodings {
410412
fileEncoding = this.textResourceConfigurationService.getValue(resource, 'files.encoding'); // and last we check for settings
411413
}
412414

413-
if (!fileEncoding || !encodingExists(fileEncoding)) {
415+
if (!fileEncoding || !(await encodingExists(fileEncoding))) {
414416
fileEncoding = UTF8; // the default is UTF 8
415417
}
416418

0 commit comments

Comments
 (0)