Skip to content

Commit cbb1610

Browse files
author
Benjamin Pasero
committed
files - add support for tracking bytes written and adopt for upload in web
1 parent 0c73b69 commit cbb1610

6 files changed

Lines changed: 108 additions & 67 deletions

File tree

‎src/vs/base/test/common/stream.test.ts‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -76,12 +76,6 @@ suite('Stream', () => {
7676
};
7777
stream.on('error', errorListener);
7878

79-
let end = false;
80-
const endListener = () => {
81-
end = true;
82-
};
83-
stream.on('end', endListener);
84-
8579
let data = false;
8680
const dataListener = () => {
8781
data = true;

‎src/vs/platform/files/common/fileService.ts‎

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -330,12 +330,12 @@ export class FileService extends Disposable implements IFileService {
330330

331331
// write file: unbuffered (only if data to write is a buffer, or the provider has no buffered write capability)
332332
if (!hasOpenReadWriteCloseCapability(provider) || (hasReadWriteCapability(provider) && bufferOrReadableOrStream instanceof VSBuffer)) {
333-
await this.doWriteUnbuffered(provider, resource, bufferOrReadableOrStream);
333+
await this.doWriteUnbuffered(provider, resource, bufferOrReadableOrStream, options);
334334
}
335335

336336
// write file: buffered
337337
else {
338-
await this.doWriteBuffered(provider, resource, bufferOrReadableOrStream instanceof VSBuffer ? bufferToReadable(bufferOrReadableOrStream) : bufferOrReadableOrStream);
338+
await this.doWriteBuffered(provider, resource, bufferOrReadableOrStream instanceof VSBuffer ? bufferToReadable(bufferOrReadableOrStream) : bufferOrReadableOrStream, options);
339339
}
340340
} catch (error) {
341341
throw new FileOperationError(localize('err.write', "Unable to write file '{0}' ({1})", this.resourceForError(resource), ensureFileSystemProviderError(error).toString()), toFileOperationResult(error), options);
@@ -953,7 +953,7 @@ export class FileService extends Disposable implements IFileService {
953953
return isPathCaseSensitive ? resource.toString() : resource.toString().toLowerCase();
954954
}
955955

956-
private async doWriteBuffered(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, resource: URI, readableOrStream: VSBufferReadable | VSBufferReadableStream): Promise<void> {
956+
private async doWriteBuffered(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, resource: URI, readableOrStream: VSBufferReadable | VSBufferReadableStream, options?: IWriteFileOptions): Promise<void> {
957957
return this.ensureWriteQueue(provider, resource).queue(async () => {
958958

959959
// open handle
@@ -962,9 +962,9 @@ export class FileService extends Disposable implements IFileService {
962962
// write into handle until all bytes from buffer have been written
963963
try {
964964
if (isReadableStream(readableOrStream)) {
965-
await this.doWriteStreamBufferedQueued(provider, handle, readableOrStream);
965+
await this.doWriteStreamBufferedQueued(provider, handle, readableOrStream, options);
966966
} else {
967-
await this.doWriteReadableBufferedQueued(provider, handle, readableOrStream);
967+
await this.doWriteReadableBufferedQueued(provider, handle, readableOrStream, options);
968968
}
969969
} catch (error) {
970970
throw ensureFileSystemProviderError(error);
@@ -976,7 +976,7 @@ export class FileService extends Disposable implements IFileService {
976976
});
977977
}
978978

979-
private doWriteStreamBufferedQueued(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, stream: VSBufferReadableStream): Promise<void> {
979+
private doWriteStreamBufferedQueued(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, stream: VSBufferReadableStream, options?: IWriteFileOptions): Promise<void> {
980980
return new Promise((resolve, reject) => {
981981
let posInFile = 0;
982982

@@ -986,7 +986,7 @@ export class FileService extends Disposable implements IFileService {
986986
stream.pause();
987987

988988
try {
989-
await this.doWriteBuffer(provider, handle, chunk, chunk.byteLength, posInFile, 0);
989+
await this.doWriteBuffer(provider, handle, chunk, chunk.byteLength, posInFile, 0, options);
990990
} catch (error) {
991991
return reject(error);
992992
}
@@ -1005,30 +1005,35 @@ export class FileService extends Disposable implements IFileService {
10051005
});
10061006
}
10071007

1008-
private async doWriteReadableBufferedQueued(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, readable: VSBufferReadable): Promise<void> {
1008+
private async doWriteReadableBufferedQueued(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, readable: VSBufferReadable, options?: IWriteFileOptions): Promise<void> {
10091009
let posInFile = 0;
10101010

10111011
let chunk: VSBuffer | null;
10121012
while ((chunk = readable.read()) !== null) {
1013-
await this.doWriteBuffer(provider, handle, chunk, chunk.byteLength, posInFile, 0);
1013+
await this.doWriteBuffer(provider, handle, chunk, chunk.byteLength, posInFile, 0, options);
10141014

10151015
posInFile += chunk.byteLength;
10161016
}
10171017
}
10181018

1019-
private async doWriteBuffer(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, buffer: VSBuffer, length: number, posInFile: number, posInBuffer: number): Promise<void> {
1019+
private async doWriteBuffer(provider: IFileSystemProviderWithOpenReadWriteCloseCapability, handle: number, buffer: VSBuffer, length: number, posInFile: number, posInBuffer: number, options?: IWriteFileOptions): Promise<void> {
10201020
let totalBytesWritten = 0;
10211021
while (totalBytesWritten < length) {
1022+
1023+
// Write through the provider
10221024
const bytesWritten = await provider.write(handle, posInFile + totalBytesWritten, buffer.buffer, posInBuffer + totalBytesWritten, length - totalBytesWritten);
10231025
totalBytesWritten += bytesWritten;
1026+
1027+
// report progress as needed
1028+
options?.progress?.(bytesWritten);
10241029
}
10251030
}
10261031

1027-
private async doWriteUnbuffered(provider: IFileSystemProviderWithFileReadWriteCapability, resource: URI, bufferOrReadableOrStream: VSBuffer | VSBufferReadable | VSBufferReadableStream): Promise<void> {
1028-
return this.ensureWriteQueue(provider, resource).queue(() => this.doWriteUnbufferedQueued(provider, resource, bufferOrReadableOrStream));
1032+
private async doWriteUnbuffered(provider: IFileSystemProviderWithFileReadWriteCapability, resource: URI, bufferOrReadableOrStream: VSBuffer | VSBufferReadable | VSBufferReadableStream, options?: IWriteFileOptions): Promise<void> {
1033+
return this.ensureWriteQueue(provider, resource).queue(() => this.doWriteUnbufferedQueued(provider, resource, bufferOrReadableOrStream, options));
10291034
}
10301035

1031-
private async doWriteUnbufferedQueued(provider: IFileSystemProviderWithFileReadWriteCapability, resource: URI, bufferOrReadableOrStream: VSBuffer | VSBufferReadable | VSBufferReadableStream): Promise<void> {
1036+
private async doWriteUnbufferedQueued(provider: IFileSystemProviderWithFileReadWriteCapability, resource: URI, bufferOrReadableOrStream: VSBuffer | VSBufferReadable | VSBufferReadableStream, options?: IWriteFileOptions): Promise<void> {
10321037
let buffer: VSBuffer;
10331038
if (bufferOrReadableOrStream instanceof VSBuffer) {
10341039
buffer = bufferOrReadableOrStream;
@@ -1038,7 +1043,11 @@ export class FileService extends Disposable implements IFileService {
10381043
buffer = readableToBuffer(bufferOrReadableOrStream);
10391044
}
10401045

1041-
return provider.writeFile(resource, buffer.buffer, { create: true, overwrite: true });
1046+
// Write through the provider
1047+
await provider.writeFile(resource, buffer.buffer, { create: true, overwrite: true });
1048+
1049+
// Report progress as needed
1050+
options?.progress?.(buffer.byteLength);
10421051
}
10431052

10441053
private async doPipeBuffered(sourceProvider: IFileSystemProviderWithOpenReadWriteCloseCapability, source: URI, targetProvider: IFileSystemProviderWithOpenReadWriteCloseCapability, target: URI): Promise<void> {

‎src/vs/platform/files/common/files.ts‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -731,6 +731,13 @@ export interface IWriteFileOptions {
731731
* The etag of the file. This can be used to prevent dirty writes.
732732
*/
733733
readonly etag?: string;
734+
735+
/**
736+
* The progress callback can be used to get accurate information how many
737+
* bytes have been written. Each call carries the length of bytes written
738+
* since the last call was made.
739+
*/
740+
readonly progress?: (byteLength: number) => void;
734741
}
735742

736743
export interface IResolveFileOptions {
@@ -865,3 +872,33 @@ export function whenProviderRegistered(file: URI, fileService: IFileService): Pr
865872
*/
866873
export const MIN_MAX_MEMORY_SIZE_MB = 2048;
867874
export const FALLBACK_MAX_MEMORY_SIZE_MB = 4096;
875+
876+
/**
877+
* Helper to format a raw byte size into a human readable label.
878+
*/
879+
export class BinarySize {
880+
static readonly KB = 1024;
881+
static readonly MB = BinarySize.KB * BinarySize.KB;
882+
static readonly GB = BinarySize.MB * BinarySize.KB;
883+
static readonly TB = BinarySize.GB * BinarySize.KB;
884+
885+
static formatSize(size: number): string {
886+
if (size < BinarySize.KB) {
887+
return localize('sizeB', "{0}B", size);
888+
}
889+
890+
if (size < BinarySize.MB) {
891+
return localize('sizeKB', "{0}KB", (size / BinarySize.KB).toFixed(2));
892+
}
893+
894+
if (size < BinarySize.GB) {
895+
return localize('sizeMB', "{0}MB", (size / BinarySize.MB).toFixed(2));
896+
}
897+
898+
if (size < BinarySize.TB) {
899+
return localize('sizeGB', "{0}GB", (size / BinarySize.GB).toFixed(2));
900+
}
901+
902+
return localize('sizeTB', "{0}TB", (size / BinarySize.TB).toFixed(2));
903+
}
904+
}

‎src/vs/platform/files/test/electron-browser/diskFileService.test.ts‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1681,9 +1681,11 @@ suite('Disk File Service', function () {
16811681
assert.equal(content, 'Small File');
16821682

16831683
const newContent = 'Updates to the small file';
1684-
await service.writeFile(resource, VSBuffer.fromString(newContent));
1684+
let totalBytes = 0;
1685+
await service.writeFile(resource, VSBuffer.fromString(newContent), { progress: byteLength => totalBytes += byteLength });
16851686

16861687
assert.equal(readFileSync(resource.fsPath), newContent);
1688+
assert.equal(totalBytes, newContent.length);
16871689
}
16881690

16891691
test('writeFile (large file) - default', async () => {
@@ -1708,10 +1710,12 @@ suite('Disk File Service', function () {
17081710
const content = readFileSync(resource.fsPath);
17091711
const newContent = content.toString() + content.toString();
17101712

1711-
const fileStat = await service.writeFile(resource, VSBuffer.fromString(newContent));
1713+
let totalBytes = 0;
1714+
const fileStat = await service.writeFile(resource, VSBuffer.fromString(newContent), { progress: byteLength => totalBytes += byteLength });
17121715
assert.equal(fileStat.name, 'lorem.txt');
17131716

17141717
assert.equal(readFileSync(resource.fsPath), newContent);
1718+
assert.equal(totalBytes, newContent.length);
17151719
}
17161720

17171721
test('writeFile - buffered - readonly throws', async () => {
@@ -1782,9 +1786,11 @@ suite('Disk File Service', function () {
17821786
assert.equal(content, 'Small File');
17831787

17841788
const newContent = 'Updates to the small file';
1785-
await service.writeFile(resource, toLineByLineReadable(newContent));
1789+
let totalBytes = 0;
1790+
await service.writeFile(resource, toLineByLineReadable(newContent), { progress: byteLength => totalBytes += byteLength });
17861791

17871792
assert.equal(readFileSync(resource.fsPath), newContent);
1793+
assert.equal(totalBytes, newContent.length);
17881794
}
17891795

17901796
test('writeFile (large file - readable) - default', async () => {
@@ -1809,10 +1815,12 @@ suite('Disk File Service', function () {
18091815
const content = readFileSync(resource.fsPath);
18101816
const newContent = content.toString() + content.toString();
18111817

1812-
const fileStat = await service.writeFile(resource, toLineByLineReadable(newContent));
1818+
let totalBytes = 0;
1819+
const fileStat = await service.writeFile(resource, toLineByLineReadable(newContent), { progress: byteLength => totalBytes += byteLength });
18131820
assert.equal(fileStat.name, 'lorem.txt');
18141821

18151822
assert.equal(readFileSync(resource.fsPath), newContent);
1823+
assert.equal(totalBytes, newContent.length);
18161824
}
18171825

18181826
test('writeFile (stream) - default', async () => {
@@ -1835,10 +1843,13 @@ suite('Disk File Service', function () {
18351843
const source = URI.file(join(testDir, 'small.txt'));
18361844
const target = URI.file(join(testDir, 'small-copy.txt'));
18371845

1838-
const fileStat = await service.writeFile(target, streamToBufferReadableStream(createReadStream(source.fsPath)));
1846+
let totalBytes = 0;
1847+
const fileStat = await service.writeFile(target, streamToBufferReadableStream(createReadStream(source.fsPath)), { progress: byteLength => totalBytes += byteLength });
18391848
assert.equal(fileStat.name, 'small-copy.txt');
18401849

1841-
assert.equal(readFileSync(source.fsPath).toString(), readFileSync(target.fsPath).toString());
1850+
const targetContents = readFileSync(target.fsPath).toString();
1851+
assert.equal(readFileSync(source.fsPath).toString(), targetContents);
1852+
assert.equal(totalBytes, targetContents.length);
18421853
}
18431854

18441855
test('writeFile (large file - stream) - default', async () => {
@@ -1861,10 +1872,13 @@ suite('Disk File Service', function () {
18611872
const source = URI.file(join(testDir, 'lorem.txt'));
18621873
const target = URI.file(join(testDir, 'lorem-copy.txt'));
18631874

1864-
const fileStat = await service.writeFile(target, streamToBufferReadableStream(createReadStream(source.fsPath)));
1875+
let totalBytes = 0;
1876+
const fileStat = await service.writeFile(target, streamToBufferReadableStream(createReadStream(source.fsPath)), { progress: byteLength => totalBytes += byteLength });
18651877
assert.equal(fileStat.name, 'lorem-copy.txt');
18661878

1867-
assert.equal(readFileSync(source.fsPath).toString(), readFileSync(target.fsPath).toString());
1879+
const targetContents = readFileSync(target.fsPath).toString();
1880+
assert.equal(readFileSync(source.fsPath).toString(), targetContents);
1881+
assert.equal(totalBytes, targetContents.length);
18681882
}
18691883

18701884
test('writeFile (file is created including parents)', async () => {

‎src/vs/workbench/browser/parts/editor/binaryEditor.ts‎

Lines changed: 1 addition & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import { dispose, IDisposable, Disposable, DisposableStore } from 'vs/base/commo
2020
import { IStorageService } from 'vs/platform/storage/common/storage';
2121
import { IWorkbenchEnvironmentService } from 'vs/workbench/services/environment/common/environmentService';
2222
import { assertIsDefined, assertAllDefined } from 'vs/base/common/types';
23+
import { BinarySize } from 'vs/platform/files/common/files';
2324

2425
export interface IOpenCallbacks {
2526
openInternal: (input: EditorInput, options: EditorOptions | undefined) => Promise<void>;
@@ -169,33 +170,6 @@ export interface IResourceDescriptor {
169170
readonly mime: string;
170171
}
171172

172-
class BinarySize {
173-
static readonly KB = 1024;
174-
static readonly MB = BinarySize.KB * BinarySize.KB;
175-
static readonly GB = BinarySize.MB * BinarySize.KB;
176-
static readonly TB = BinarySize.GB * BinarySize.KB;
177-
178-
static formatSize(size: number): string {
179-
if (size < BinarySize.KB) {
180-
return nls.localize('sizeB', "{0}B", size);
181-
}
182-
183-
if (size < BinarySize.MB) {
184-
return nls.localize('sizeKB', "{0}KB", (size / BinarySize.KB).toFixed(2));
185-
}
186-
187-
if (size < BinarySize.GB) {
188-
return nls.localize('sizeMB', "{0}MB", (size / BinarySize.MB).toFixed(2));
189-
}
190-
191-
if (size < BinarySize.TB) {
192-
return nls.localize('sizeGB', "{0}GB", (size / BinarySize.GB).toFixed(2));
193-
}
194-
195-
return nls.localize('sizeTB', "{0}TB", (size / BinarySize.TB).toFixed(2));
196-
}
197-
}
198-
199173
interface ResourceViewerContext extends IDisposable {
200174
layout?(dimension: Dimension): void;
201175
}

‎src/vs/workbench/contrib/files/browser/views/explorerViewer.ts‎

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import * as glob from 'vs/base/common/glob';
99
import { IListVirtualDelegate, ListDragOverEffect } from 'vs/base/browser/ui/list/list';
1010
import { IProgressService, ProgressLocation, IProgressStep, IProgress } from 'vs/platform/progress/common/progress';
1111
import { INotificationService, Severity } from 'vs/platform/notification/common/notification';
12-
import { IFileService, FileKind, FileOperationError, FileOperationResult, FileSystemProviderCapabilities } from 'vs/platform/files/common/files';
12+
import { IFileService, FileKind, FileOperationError, FileOperationResult, FileSystemProviderCapabilities, BinarySize } from 'vs/platform/files/common/files';
1313
import { IWorkbenchLayoutService } from 'vs/workbench/services/layout/browser/layoutService';
1414
import { IWorkspaceContextService, WorkbenchState } from 'vs/platform/workspace/common/workspace';
1515
import { IDisposable, Disposable, dispose, toDisposable, DisposableStore } from 'vs/base/common/lifecycle';
@@ -1023,12 +1023,25 @@ export class FileDragAndDrop implements ITreeDragAndDrop<ExplorerItem> {
10231023
}
10241024

10251025
// Report progress
1026+
let totalBytesUploaded = 0;
1027+
const reportProgress = (fileSize: number, bytesUploaded: number): void => {
1028+
totalBytesUploaded += bytesUploaded;
1029+
1030+
let message: string;
1031+
if (operation.total === 1 && entry.name) {
1032+
message = entry.name;
1033+
} else {
1034+
message = localize('uploadProgress', "{0} of {1} files", operation.worked, operation.total);
1035+
}
1036+
1037+
if (fileSize > BinarySize.MB) {
1038+
message = localize('uploadProgressDetail', "{0} ({1} of {2})", message, BinarySize.formatSize(totalBytesUploaded), BinarySize.formatSize(fileSize));
1039+
}
1040+
1041+
progress.report({ message });
1042+
};
10261043
operation.worked++;
1027-
if (operation.total === 1) {
1028-
progress.report({ message: entry.name });
1029-
} else {
1030-
progress.report({ message: localize('uploadProgress', "{0} of {1} files", operation.worked, operation.total) });
1031-
}
1044+
reportProgress(0, 0);
10321045

10331046
// Handle file upload
10341047
if (entry.isFile) {
@@ -1040,12 +1053,12 @@ export class FileDragAndDrop implements ITreeDragAndDrop<ExplorerItem> {
10401053

10411054
// Chrome/Edge/Firefox support stream method
10421055
if (typeof file.stream === 'function') {
1043-
await this.doUploadWebFileEntryBuffered(resource, file);
1056+
await this.doUploadWebFileEntryBuffered(resource, file, reportProgress);
10441057
}
10451058

10461059
// Fallback to unbuffered upload for other browsers
10471060
else {
1048-
await this.doUploadWebFileEntryUnbuffered(resource, file);
1061+
await this.doUploadWebFileEntryUnbuffered(resource, file, reportProgress);
10491062
}
10501063

10511064
return { isFile: true, resource };
@@ -1087,9 +1100,9 @@ export class FileDragAndDrop implements ITreeDragAndDrop<ExplorerItem> {
10871100
}
10881101
}
10891102

1090-
private async doUploadWebFileEntryBuffered(resource: URI, file: File): Promise<void> {
1103+
private async doUploadWebFileEntryBuffered(resource: URI, file: File, progressReporter: (fileSize: number, bytesUploaded: number) => void): Promise<void> {
10911104
const writeableStream = newWriteableBufferStream();
1092-
const writeFilePromise = this.fileService.writeFile(resource, writeableStream);
1105+
const writeFilePromise = this.fileService.writeFile(resource, writeableStream, { progress: byteLength => progressReporter(file.size, byteLength) });
10931106

10941107
// Read the file in chunks using File.stream() web APIs
10951108
try {
@@ -1110,13 +1123,13 @@ export class FileDragAndDrop implements ITreeDragAndDrop<ExplorerItem> {
11101123
await writeFilePromise;
11111124
}
11121125

1113-
private doUploadWebFileEntryUnbuffered(resource: URI, file: File): Promise<void> {
1126+
private doUploadWebFileEntryUnbuffered(resource: URI, file: File, progressReporter: (fileSize: number, bytesUploaded: number) => void): Promise<void> {
11141127
return new Promise<void>((resolve, reject) => {
11151128
const reader = new FileReader();
11161129
reader.onload = async event => {
11171130
try {
11181131
if (event.target?.result instanceof ArrayBuffer) {
1119-
await this.fileService.writeFile(resource, VSBuffer.wrap(new Uint8Array(event.target.result)));
1132+
await this.fileService.writeFile(resource, VSBuffer.wrap(new Uint8Array(event.target.result)), { progress: byteLength => progressReporter(file.size, byteLength) });
11201133
} else {
11211134
throw new Error('Could not read from dropped file.');
11221135
}

0 commit comments

Comments
 (0)