Skip to content

Commit 17a8e25

Browse files
author
Benjamin Pasero
authored
Creating files will add 2 times BOM (UTF-8, 16) microsoft#102103 (microsoft#102111)
1 parent ff4eb85 commit 17a8e25

2 files changed

Lines changed: 39 additions & 10 deletions

File tree

‎src/vs/workbench/services/textfile/common/encoding.ts‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ export async function toEncodeReadable(readable: Readable<string>, encoding: str
132132
const iconv = await import('iconv-lite-umd');
133133
const encoder = iconv.getEncoder(toNodeEncoding(encoding), options);
134134

135-
let bytesRead = 0;
135+
let bytesWritten = false;
136136
let done = false;
137137

138138
return {
@@ -146,9 +146,9 @@ export async function toEncodeReadable(readable: Readable<string>, encoding: str
146146
done = true;
147147

148148
// If we are instructed to add a BOM but we detect that no
149-
// bytes have been read, we must ensure to return the BOM
149+
// bytes have been written, we must ensure to return the BOM
150150
// ourselves so that we comply with the contract.
151-
if (bytesRead === 0 && options?.addBOM) {
151+
if (!bytesWritten && options?.addBOM) {
152152
switch (encoding) {
153153
case UTF8:
154154
case UTF8_with_bom:
@@ -162,13 +162,15 @@ export async function toEncodeReadable(readable: Readable<string>, encoding: str
162162

163163
const leftovers = encoder.end();
164164
if (leftovers && leftovers.length > 0) {
165+
bytesWritten = true;
166+
165167
return VSBuffer.wrap(leftovers);
166168
}
167169

168170
return null;
169171
}
170172

171-
bytesRead += chunk.length;
173+
bytesWritten = true;
172174

173175
return VSBuffer.wrap(encoder.write(chunk));
174176
}

‎src/vs/workbench/services/textfile/test/electron-browser/textFileService.io.test.ts‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import { DiskFileSystemProvider } from 'vs/platform/files/node/diskFileSystemPro
2020
import { generateUuid } from 'vs/base/common/uuid';
2121
import { join, basename } from 'vs/base/common/path';
2222
import { getPathFromAmdModule } from 'vs/base/common/amd';
23-
import { UTF16be, UTF16le, UTF8_with_bom, UTF8 } from 'vs/workbench/services/textfile/common/encoding';
23+
import { UTF16be, UTF16le, UTF8_with_bom, UTF8, UTF16le_BOM, UTF16be_BOM, UTF8_BOM } from 'vs/workbench/services/textfile/common/encoding';
2424
import { DefaultEndOfLine, ITextSnapshot } from 'vs/editor/common/model';
2525
import { createTextModel } from 'vs/editor/test/common/editorTestUtils';
2626
import { isWindows } from 'vs/base/common/platform';
@@ -84,25 +84,28 @@ suite('Files - TextFileService i/o', function () {
8484

8585
await service.create(resource);
8686

87-
assert.equal(await exists(resource.fsPath), true);
87+
const res = await readFile(resource.fsPath);
88+
assert.equal(res.byteLength, 0 /* no BOM */);
8889
});
8990

9091
test('create - no encoding - content provided (string)', async () => {
9192
const resource = URI.file(join(testDir, 'small_new.txt'));
9293

9394
await service.create(resource, 'Hello World');
9495

95-
assert.equal(await exists(resource.fsPath), true);
96-
assert.equal((await readFile(resource.fsPath)).toString(), 'Hello World');
96+
const res = await readFile(resource.fsPath);
97+
assert.equal(res.toString(), 'Hello World');
98+
assert.equal(res.byteLength, 'Hello World'.length);
9799
});
98100

99101
test('create - no encoding - content provided (snapshot)', async () => {
100102
const resource = URI.file(join(testDir, 'small_new.txt'));
101103

102104
await service.create(resource, stringToSnapshot('Hello World'));
103105

104-
assert.equal(await exists(resource.fsPath), true);
105-
assert.equal((await readFile(resource.fsPath)).toString(), 'Hello World');
106+
const res = await readFile(resource.fsPath);
107+
assert.equal(res.toString(), 'Hello World');
108+
assert.equal(res.byteLength, 'Hello World'.length);
106109
});
107110

108111
test('create - UTF 16 LE - no content', async () => {
@@ -114,6 +117,9 @@ suite('Files - TextFileService i/o', function () {
114117

115118
const detectedEncoding = await detectEncodingByBOM(resource.fsPath);
116119
assert.equal(detectedEncoding, UTF16le);
120+
121+
const res = await readFile(resource.fsPath);
122+
assert.equal(res.byteLength, UTF16le_BOM.length);
117123
});
118124

119125
test('create - UTF 16 LE - content provided', async () => {
@@ -125,6 +131,9 @@ suite('Files - TextFileService i/o', function () {
125131

126132
const detectedEncoding = await detectEncodingByBOM(resource.fsPath);
127133
assert.equal(detectedEncoding, UTF16le);
134+
135+
const res = await readFile(resource.fsPath);
136+
assert.equal(res.byteLength, 'Hello World'.length * 2 /* UTF16 2bytes per char */ + UTF16le_BOM.length);
128137
});
129138

130139
test('create - UTF 16 BE - no content', async () => {
@@ -136,6 +145,9 @@ suite('Files - TextFileService i/o', function () {
136145

137146
const detectedEncoding = await detectEncodingByBOM(resource.fsPath);
138147
assert.equal(detectedEncoding, UTF16be);
148+
149+
const res = await readFile(resource.fsPath);
150+
assert.equal(res.byteLength, UTF16le_BOM.length);
139151
});
140152

141153
test('create - UTF 16 BE - content provided', async () => {
@@ -147,6 +159,9 @@ suite('Files - TextFileService i/o', function () {
147159

148160
const detectedEncoding = await detectEncodingByBOM(resource.fsPath);
149161
assert.equal(detectedEncoding, UTF16be);
162+
163+
const res = await readFile(resource.fsPath);
164+
assert.equal(res.byteLength, 'Hello World'.length * 2 /* UTF16 2bytes per char */ + UTF16be_BOM.length);
150165
});
151166

152167
test('create - UTF 8 BOM - no content', async () => {
@@ -158,6 +173,9 @@ suite('Files - TextFileService i/o', function () {
158173

159174
const detectedEncoding = await detectEncodingByBOM(resource.fsPath);
160175
assert.equal(detectedEncoding, UTF8_with_bom);
176+
177+
const res = await readFile(resource.fsPath);
178+
assert.equal(res.byteLength, UTF8_BOM.length);
161179
});
162180

163181
test('create - UTF 8 BOM - content provided', async () => {
@@ -169,6 +187,9 @@ suite('Files - TextFileService i/o', function () {
169187

170188
const detectedEncoding = await detectEncodingByBOM(resource.fsPath);
171189
assert.equal(detectedEncoding, UTF8_with_bom);
190+
191+
const res = await readFile(resource.fsPath);
192+
assert.equal(res.byteLength, 'Hello World'.length + UTF8_BOM.length);
172193
});
173194

174195
test('create - UTF 8 BOM - empty content - snapshot', async () => {
@@ -180,6 +201,9 @@ suite('Files - TextFileService i/o', function () {
180201

181202
const detectedEncoding = await detectEncodingByBOM(resource.fsPath);
182203
assert.equal(detectedEncoding, UTF8_with_bom);
204+
205+
const res = await readFile(resource.fsPath);
206+
assert.equal(res.byteLength, UTF8_BOM.length);
183207
});
184208

185209
test('create - UTF 8 BOM - content provided - snapshot', async () => {
@@ -191,6 +215,9 @@ suite('Files - TextFileService i/o', function () {
191215

192216
const detectedEncoding = await detectEncodingByBOM(resource.fsPath);
193217
assert.equal(detectedEncoding, UTF8_with_bom);
218+
219+
const res = await readFile(resource.fsPath);
220+
assert.equal(res.byteLength, 'Hello World'.length + UTF8_BOM.length);
194221
});
195222

196223
test('write - use encoding (UTF 16 BE) - small content as string', async () => {

0 commit comments

Comments
 (0)