Skip to content

Commit 96d0395

Browse files
author
Benjamin Pasero
committed
files2 - implement read() and readStream()
1 parent b91293e commit 96d0395

9 files changed

Lines changed: 205 additions & 74 deletions

File tree

‎src/vs/base/node/encoding.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,12 @@ export interface IDecodeStreamOptions {
2424
overwriteEncoding?(detectedEncoding: string | null): string;
2525
}
2626

27-
export function toDecodeStream(readable: Readable, options: IDecodeStreamOptions): Promise<{ detected: IDetectedEncodingResult, stream: NodeJS.ReadableStream }> {
27+
export interface IDecodeStreamResult {
28+
detected: IDetectedEncodingResult;
29+
stream: NodeJS.ReadableStream;
30+
}
31+
32+
export function toDecodeStream(readable: Readable, options: IDecodeStreamOptions): Promise<IDecodeStreamResult> {
2833
if (!options.minBytesRequiredForDetection) {
2934
options.minBytesRequiredForDetection = options.guessEncoding ? AUTO_GUESS_BUFFER_MAX_LEN : NO_GUESS_BUFFER_MAX_LEN;
3035
}
@@ -33,7 +38,7 @@ export function toDecodeStream(readable: Readable, options: IDecodeStreamOptions
3338
options.overwriteEncoding = detected => detected || UTF8;
3439
}
3540

36-
return new Promise<{ detected: IDetectedEncodingResult, stream: NodeJS.ReadableStream }>((resolve, reject) => {
41+
return new Promise<IDecodeStreamResult>((resolve, reject) => {
3742

3843
readable.on('error', reject);
3944

‎src/vs/workbench/contrib/files/common/files.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ export class FileOnDiskContentProvider implements ITextModelContentProvider {
176176
private resolveEditorModel(resource: URI, createAsNeeded: boolean = true): Promise<ITextModel | null> {
177177
const savedFileResource = toLocalResource(resource, this.environmentService.configuration.remoteAuthority);
178178

179-
return this.textFileService.read(savedFileResource).then(content => {
179+
return this.textFileService.readStream(savedFileResource).then(content => {
180180
let codeEditorModel = this.modelService.getModel(resource);
181181
if (codeEditorModel) {
182182
this.modelService.updateModel(codeEditorModel, content.value);

‎src/vs/workbench/contrib/welcome/walkThrough/common/walkThroughContentProvider.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ export class WalkThroughContentProvider implements ITextModelContentProvider, IW
3535
reject(err);
3636
}
3737
});
38-
}) : this.textFileService.read(URI.file(resource.fsPath)).then(content => content.value));
38+
}) : this.textFileService.readStream(URI.file(resource.fsPath)).then(content => content.value));
3939
return content.then(content => {
4040
let codeEditorModel = this.modelService.getModel(resource);
4141
if (!codeEditorModel) {
@@ -61,7 +61,7 @@ export class WalkThroughSnippetContentProvider implements ITextModelContentProvi
6161
}
6262

6363
public provideTextContent(resource: URI): Promise<ITextModel> {
64-
return this.textFileService.read(URI.file(resource.fsPath)).then(content => {
64+
return this.textFileService.readStream(URI.file(resource.fsPath)).then(content => {
6565
let codeEditorModel = this.modelService.getModel(resource);
6666
if (!codeEditorModel) {
6767
const j = parseInt(resource.fragment);

‎src/vs/workbench/services/textfile/common/textFileEditorModel.ts‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import { URI } from 'vs/base/common/uri';
1212
import { isUndefinedOrNull, withUndefinedAsNull } from 'vs/base/common/types';
1313
import { IWorkspaceContextService } from 'vs/platform/workspace/common/workspace';
1414
import { IEnvironmentService } from 'vs/platform/environment/common/environment';
15-
import { ITextFileService, IAutoSaveConfiguration, ModelState, ITextFileEditorModel, ISaveOptions, ISaveErrorHandler, ISaveParticipant, StateChange, SaveReason, ITextFileContent, ILoadOptions, LoadReason, IResolvedTextFileEditorModel } from 'vs/workbench/services/textfile/common/textfiles';
15+
import { ITextFileService, IAutoSaveConfiguration, ModelState, ITextFileEditorModel, ISaveOptions, ISaveErrorHandler, ISaveParticipant, StateChange, SaveReason, ITextFileStreamContent, ILoadOptions, LoadReason, IResolvedTextFileEditorModel } from 'vs/workbench/services/textfile/common/textfiles';
1616
import { EncodingMode } from 'vs/workbench/common/editor';
1717
import { BaseTextEditorModel } from 'vs/workbench/common/editor/textEditorModel';
1818
import { IBackupFileService } from 'vs/workbench/services/backup/common/backup';
@@ -266,7 +266,7 @@ export class TextFileEditorModel extends BaseTextEditorModel implements ITextFil
266266

267267
// If we have a backup, continue loading with it
268268
if (!!backup) {
269-
const content: ITextFileContent = {
269+
const content: ITextFileStreamContent = {
270270
resource: this.resource,
271271
name: basename(this.resource),
272272
mtime: Date.now(),
@@ -306,7 +306,7 @@ export class TextFileEditorModel extends BaseTextEditorModel implements ITextFil
306306

307307
// Resolve Content
308308
try {
309-
const content = await this.textFileService.read(this.resource, { acceptTextOnly: !allowBinary, etag, encoding: this.preferredEncoding });
309+
const content = await this.textFileService.readStream(this.resource, { acceptTextOnly: !allowBinary, etag, encoding: this.preferredEncoding });
310310

311311
// Clear orphaned state when loading was successful
312312
this.setOrphaned(false);
@@ -346,7 +346,7 @@ export class TextFileEditorModel extends BaseTextEditorModel implements ITextFil
346346
}
347347
}
348348

349-
private async loadWithContent(content: ITextFileContent, options?: ILoadOptions, backup?: URI): Promise<TextFileEditorModel> {
349+
private async loadWithContent(content: ITextFileStreamContent, options?: ILoadOptions, backup?: URI): Promise<TextFileEditorModel> {
350350
const model = await this.doLoadWithContent(content, backup);
351351

352352
// Telemetry: We log the fileGet telemetry event after the model has been loaded to ensure a good mimetype
@@ -372,7 +372,7 @@ export class TextFileEditorModel extends BaseTextEditorModel implements ITextFil
372372
return model;
373373
}
374374

375-
private doLoadWithContent(content: ITextFileContent, backup?: URI): Promise<TextFileEditorModel> {
375+
private doLoadWithContent(content: ITextFileStreamContent, backup?: URI): Promise<TextFileEditorModel> {
376376
this.logService.trace('load() - resolved content', this.resource);
377377

378378
// Update our resolved disk stat model

‎src/vs/workbench/services/textfile/common/textFileService.ts‎

Lines changed: 32 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import { Event, Emitter } from 'vs/base/common/event';
1111
import * as platform from 'vs/base/common/platform';
1212
import { IWindowsService } from 'vs/platform/windows/common/windows';
1313
import { IBackupFileService } from 'vs/workbench/services/backup/common/backup';
14-
import { IResult, ITextFileOperationResult, ITextFileService, ITextFileContent, IAutoSaveConfiguration, AutoSaveMode, SaveReason, ITextFileEditorModelManager, ITextFileEditorModel, ModelState, ISaveOptions, AutoSaveContext, IWillMoveEvent } from 'vs/workbench/services/textfile/common/textfiles';
14+
import { IResult, ITextFileOperationResult, ITextFileService, ITextFileStreamContent, IAutoSaveConfiguration, AutoSaveMode, SaveReason, ITextFileEditorModelManager, ITextFileEditorModel, ModelState, ISaveOptions, AutoSaveContext, IWillMoveEvent, ITextFileContent } from 'vs/workbench/services/textfile/common/textfiles';
1515
import { ConfirmResult, IRevertOptions } from 'vs/workbench/common/editor';
1616
import { ILifecycleService, ShutdownReason, LifecyclePhase } from 'vs/platform/lifecycle/common/lifecycle';
1717
import { IWorkspaceContextService, WorkbenchState } from 'vs/platform/workspace/common/workspace';
@@ -370,6 +370,21 @@ export abstract class TextFileService extends Disposable implements ITextFileSer
370370
//#region primitives (read, create, move, delete, update)
371371

372372
async read(resource: URI, options?: IReadTextFileOptions): Promise<ITextFileContent> {
373+
const content = await this.fileService.readFile(resource, options);
374+
375+
// in case of acceptTextOnly: true, we check the first
376+
// chunk for possibly being binary by looking for 0-bytes
377+
// we limit this check to the first 512 bytes
378+
this.validateBinary(content.value, options);
379+
380+
return {
381+
...content,
382+
encoding: 'utf8',
383+
value: content.value.toString()
384+
};
385+
}
386+
387+
async readStream(resource: URI, options?: IReadTextFileOptions): Promise<ITextFileStreamContent> {
373388
const stream = await this.fileService.readFileStream(resource, options);
374389

375390
// in case of acceptTextOnly: true, we check the first
@@ -380,11 +395,7 @@ export abstract class TextFileService extends Disposable implements ITextFileSer
380395
if (!checkedForBinary) {
381396
checkedForBinary = true;
382397

383-
for (let i = 0; i < data.byteLength && i < 512; i++) {
384-
if (data.readUint8(i) === 0) {
385-
throw new FileOperationError(nls.localize('fileBinaryError', "File seems to be binary and cannot be opened as text"), FileOperationResult.FILE_IS_BINARY, options);
386-
}
387-
}
398+
this.validateBinary(data, options);
388399
}
389400

390401
return undefined;
@@ -397,6 +408,21 @@ export abstract class TextFileService extends Disposable implements ITextFileSer
397408
};
398409
}
399410

411+
private validateBinary(buffer: VSBuffer, options?: IReadTextFileOptions): void {
412+
if (!options || !options.acceptTextOnly) {
413+
return; // no validation needed
414+
}
415+
416+
// in case of acceptTextOnly: true, we check the first
417+
// chunk for possibly being binary by looking for 0-bytes
418+
// we limit this check to the first 512 bytes
419+
for (let i = 0; i < buffer.byteLength && i < 512; i++) {
420+
if (buffer.readUint8(i) === 0) {
421+
throw new FileOperationError(nls.localize('fileBinaryError', "File seems to be binary and cannot be opened as text"), FileOperationResult.FILE_IS_BINARY, options);
422+
}
423+
}
424+
}
425+
400426
async create(resource: URI, value?: string | ITextSnapshot, options?: ICreateFileOptions): Promise<IFileStatWithMetadata> {
401427
const stat = await this.doCreate(resource, value, options);
402428

‎src/vs/workbench/services/textfile/common/textfiles.ts‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,11 @@ export interface ITextFileService extends IDisposable {
105105
*/
106106
read(resource: URI, options?: IReadTextFileOptions): Promise<ITextFileContent>;
107107

108+
/**
109+
* Read the contents of a file identified by the resource as stream.
110+
*/
111+
readStream(resource: URI, options?: IReadTextFileOptions): Promise<ITextFileStreamContent>;
112+
108113
/**
109114
* Update a file with given contents.
110115
*/
@@ -262,17 +267,28 @@ export const enum LoadReason {
262267
OTHER = 3
263268
}
264269

265-
export interface ITextFileContent extends IBaseStatWithMetadata {
270+
interface IBaseTextFileContent extends IBaseStatWithMetadata {
266271

267272
/**
268-
* The line grouped content of a text file.
273+
* The encoding of the content if known.
269274
*/
270-
value: ITextBufferFactory;
275+
encoding: string;
276+
}
277+
278+
export interface ITextFileContent extends IBaseTextFileContent {
271279

272280
/**
273-
* The encoding of the content if known.
281+
* The content of a text file.
274282
*/
275-
encoding: string;
283+
value: string;
284+
}
285+
286+
export interface ITextFileStreamContent extends IBaseTextFileContent {
287+
288+
/**
289+
* The line grouped content of a text file.
290+
*/
291+
value: ITextBufferFactory;
276292
}
277293

278294
export interface IModelLoadOrCreateOptions {

‎src/vs/workbench/services/textfile/node/textFileService.ts‎

Lines changed: 72 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,18 @@
66
import { tmpdir } from 'os';
77
import { localize } from 'vs/nls';
88
import { TextFileService } from 'vs/workbench/services/textfile/common/textFileService';
9-
import { ITextFileService, ITextFileContent } from 'vs/workbench/services/textfile/common/textfiles';
9+
import { ITextFileService, ITextFileStreamContent, ITextFileContent } from 'vs/workbench/services/textfile/common/textfiles';
1010
import { registerSingleton } from 'vs/platform/instantiation/common/extensions';
1111
import { URI } from 'vs/base/common/uri';
12-
import { ITextSnapshot, IWriteTextFileOptions, IFileStatWithMetadata, IResourceEncoding, IReadTextFileOptions, stringToSnapshot, ICreateFileOptions, FileOperationError, FileOperationResult, IResourceEncodings } from 'vs/platform/files/common/files';
12+
import { ITextSnapshot, IWriteTextFileOptions, IFileStatWithMetadata, IResourceEncoding, IReadTextFileOptions, stringToSnapshot, ICreateFileOptions, FileOperationError, FileOperationResult, IResourceEncodings, IFileStreamContent } from 'vs/platform/files/common/files';
1313
import { Schemas } from 'vs/base/common/network';
1414
import { exists, stat, chmod, rimraf } from 'vs/base/node/pfs';
1515
import { join, dirname } from 'vs/base/common/path';
1616
import { isMacintosh, isLinux } from 'vs/base/common/platform';
1717
import product from 'vs/platform/product/node/product';
1818
import { ITextResourceConfigurationService } from 'vs/editor/common/services/resourceConfiguration';
1919
import { IWorkspaceContextService } from 'vs/platform/workspace/common/workspace';
20-
import { UTF8, UTF8_with_bom, UTF16be, UTF16le, encodingExists, IDetectedEncodingResult, detectEncodingByBOM, encodeStream, UTF8_BOM, UTF16be_BOM, UTF16le_BOM, toDecodeStream, IDecodeStreamOptions } from 'vs/base/node/encoding';
20+
import { UTF8, UTF8_with_bom, UTF16be, UTF16le, encodingExists, IDetectedEncodingResult, detectEncodingByBOM, encodeStream, UTF8_BOM, UTF16be_BOM, UTF16le_BOM, toDecodeStream, IDecodeStreamResult } from 'vs/base/node/encoding';
2121
import { WORKSPACE_EXTENSION } from 'vs/platform/workspaces/common/workspaces';
2222
import { joinPath, extname, isEqualOrParent } from 'vs/base/common/resources';
2323
import { Disposable } from 'vs/base/common/lifecycle';
@@ -26,6 +26,7 @@ import { VSBufferReadable, VSBuffer, VSBufferReadableStream } from 'vs/base/comm
2626
import { Readable } from 'stream';
2727
import { isUndefinedOrNull } from 'vs/base/common/types';
2828
import { createTextBufferFactoryFromStream } from 'vs/editor/common/model/textModel';
29+
import { MAX_FILE_SIZE, MAX_HEAP_SIZE } from 'vs/platform/files/node/fileConstants';
2930

3031
export class NodeTextFileService extends TextFileService {
3132

@@ -39,28 +40,72 @@ export class NodeTextFileService extends TextFileService {
3940
}
4041

4142
async read(resource: URI, options?: IReadTextFileOptions): Promise<ITextFileContent> {
42-
const stream = await this.fileService.readFileStream(resource, options);
43+
const [bufferStream, decoder] = await this.doRead(resource, options);
4344

44-
const readable = this.streamToNodeReadable(stream.value);
45+
return {
46+
...bufferStream,
47+
encoding: decoder.detected.encoding || UTF8,
48+
value: await this.nodeReadableToString(decoder.stream)
49+
};
50+
}
4551

46-
const decodeStreamOpts: IDecodeStreamOptions = {
47-
guessEncoding: options && options.autoGuessEncoding,
48-
overwriteEncoding: detected => {
49-
return this.encoding.getReadEncoding(resource, options, { encoding: detected, seemsBinary: false });
50-
}
52+
async readStream(resource: URI, options?: IReadTextFileOptions): Promise<ITextFileStreamContent> {
53+
const [bufferStream, decoder] = await this.doRead(resource, options);
54+
55+
return {
56+
...bufferStream,
57+
encoding: decoder.detected.encoding || UTF8,
58+
value: await createTextBufferFactoryFromStream(decoder.stream)
5159
};
60+
}
5261

53-
const result = await toDecodeStream(readable, decodeStreamOpts);
62+
private async doRead(resource: URI, options?: IReadTextFileOptions): Promise<[IFileStreamContent, IDecodeStreamResult]> {
5463

55-
if (options && options.acceptTextOnly && result.detected.seemsBinary) {
64+
// ensure limits
65+
options = this.ensureLimits(options);
66+
67+
// read stream raw
68+
const bufferStream = await this.fileService.readFileStream(resource, options);
69+
70+
// read through encoding library
71+
const decoder = await toDecodeStream(this.streamToNodeReadable(bufferStream.value), {
72+
guessEncoding: options && options.autoGuessEncoding,
73+
overwriteEncoding: detected => this.encoding.getReadEncoding(resource, options, { encoding: detected, seemsBinary: false })
74+
});
75+
76+
// validate binary
77+
if (options && options.acceptTextOnly && decoder.detected.seemsBinary) {
5678
throw new FileOperationError(localize('fileBinaryError', "File seems to be binary and cannot be opened as text"), FileOperationResult.FILE_IS_BINARY, options);
5779
}
5880

59-
return {
60-
...stream,
61-
encoding: result.detected.encoding || UTF8,
62-
value: await createTextBufferFactoryFromStream(result.stream)
63-
};
81+
return [bufferStream, decoder];
82+
}
83+
84+
private ensureLimits(options?: IReadTextFileOptions): IReadTextFileOptions {
85+
let ensuredOptions: IReadTextFileOptions;
86+
if (!options) {
87+
ensuredOptions = Object.create(null);
88+
} else {
89+
ensuredOptions = options;
90+
}
91+
92+
let ensuredLimits: { size?: number; memory?: number; };
93+
if (!ensuredOptions.limits) {
94+
ensuredLimits = Object.create(null);
95+
ensuredOptions.limits = ensuredLimits;
96+
} else {
97+
ensuredLimits = ensuredOptions.limits;
98+
}
99+
100+
if (typeof ensuredLimits.size !== 'number') {
101+
ensuredLimits.size = MAX_FILE_SIZE;
102+
}
103+
104+
if (typeof ensuredLimits.memory !== 'number') {
105+
ensuredLimits.memory = Math.max(typeof this.environmentService.args['max-memory'] === 'string' ? parseInt(this.environmentService.args['max-memory']) * 1024 * 1024 || 0 : 0, MAX_HEAP_SIZE);
106+
}
107+
108+
return ensuredOptions;
64109
}
65110

66111
private streamToNodeReadable(stream: VSBufferReadableStream): Readable {
@@ -107,6 +152,16 @@ export class NodeTextFileService extends TextFileService {
107152
};
108153
}
109154

155+
private nodeReadableToString(stream: NodeJS.ReadableStream): Promise<string> {
156+
return new Promise((resolve, reject) => {
157+
let result = '';
158+
159+
stream.on('data', chunk => result += chunk);
160+
stream.on('error', reject);
161+
stream.on('end', () => resolve(result));
162+
});
163+
}
164+
110165
protected async doCreate(resource: URI, value?: string, options?: ICreateFileOptions): Promise<IFileStatWithMetadata> {
111166

112167
// check for encoding

0 commit comments

Comments
 (0)