Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion doc/api/globals.md
Original file line number Diff line number Diff line change
Expand Up @@ -1298,7 +1298,8 @@ incomplete byte sequences occurring at the end of the `input` are buffered
internally and emitted after the next call to `textDecoder.decode()`.

If `textDecoder.fatal` is `true`, decoding errors that occur will result in a
`TypeError` being thrown.
`TypeError` being thrown, and the decoder is reset so that the next call starts
fresh.

### `textDecoder.encoding`

Expand Down
5 changes: 3 additions & 2 deletions lib/internal/encoding.js
Original file line number Diff line number Diff line change
Expand Up @@ -547,8 +547,9 @@ class TextDecoder {

return res;
} catch (e) {
this[kChunk] = null; // Reset unfinished chunk on errors
// The correct way per spec seems to be not destroying the decoder state (aka BOM here) in stream mode
// Reset the decoder state on errors
this[kChunk] = null;
this[kBOMSeen] = false;
throw e;
}
}
Expand Down
3 changes: 3 additions & 0 deletions src/node_i18n.cc
Original file line number Diff line number Diff line change
Expand Up @@ -523,6 +523,9 @@ void ConverterObject::Decode(const FunctionCallbackInfo<Value>& args) {
}
}

// Reset the converter state on errors.
converter->set_bom_seen(false);
converter->reset();
node::THROW_ERR_ENCODING_INVALID_ENCODED_DATA(
env->isolate(),
"The encoded data was not valid for encoding %s",
Expand Down
41 changes: 41 additions & 0 deletions test/parallel/test-whatwg-encoding-custom-fatal-streaming.js
Original file line number Diff line number Diff line change
Expand Up @@ -59,3 +59,44 @@ if (!common.hasIntl)
}
);
}

// Errors reset the decoder, so the next call starts fresh.
// Refs: https://github.com/whatwg/encoding/pull/369
{
const testCases = [
// Pending bytes are dropped.
{ encoding: 'utf-8', first: [0xE2, 0x82], bad: [0x41],
next: [0x41], expected: 'A' },
{ encoding: 'utf-16le', first: [0x3D, 0xD8], bad: [0x41, 0x00],
next: [0x41, 0x00], expected: 'A' },
{ encoding: 'utf-16be', first: [0xD8, 0x3D], bad: [0x00, 0x41],
next: [0x00, 0x41], expected: 'A' },
// The BOM is stripped again.
{ encoding: 'utf-8', first: [0xEF, 0xBB, 0xBF, 0x41], bad: [0xFF],
next: [0xEF, 0xBB, 0xBF, 0x42], expected: 'B' },
{ encoding: 'utf-16le', first: [0xFF, 0xFE, 0x41, 0x00],
bad: [0x00, 0xD8, 0x41, 0x00], next: [0xFF, 0xFE, 0x42, 0x00],
expected: 'B' },
];

// Non-Unicode encodings need full ICU.
if (common.hasFullICU) {
testCases.push(
{ encoding: 'gb18030', first: [0x81, 0x30], bad: [0x41],
next: [0x41], expected: 'A' },
// ESC $ B switches to JIS X 0208.
{ encoding: 'iso-2022-jp', first: [0x1B, 0x24, 0x42, 0x30, 0x21],
bad: [0x0E], next: [0x41, 0x42], expected: 'AB' },
);
}

testCases.forEach(({ encoding, first, bad, next, expected }) => {
const decoder = new TextDecoder(encoding, { fatal: true });
decoder.decode(new Uint8Array(first), { stream: true });
assert.throws(
() => decoder.decode(new Uint8Array(bad), { stream: true }),
{ code: 'ERR_ENCODING_INVALID_ENCODED_DATA', name: 'TypeError' }
);
assert.strictEqual(decoder.decode(new Uint8Array(next)), expected);
});
}