More lenient gzip decompression (node-fetch#239)

gregory90 · TimothyGu · commit a1e76b97e18f · 2017-02-27T10:02:45.000-08:00
Ref: nodejs/node#8701 (comment) Ref: request/request#2482 Ref: request/request#2492 Fixes: http://www.mantovanispa.it Fixes: node-fetch#139
diff --git a/src/index.js b/src/index.js
@@ -141,9 +141,18 @@ export default function fetch(url, opts) {
 				return;
 			}
 
+			// Be less strict when decoding compressed responses, since sometimes
+			// servers send slightly invalid responses that are still accepted
+			// by common browsers.
+			// Always using Z_SYNC_FLUSH is what cURL does.
+			const zlibOptions = {
+				flush: zlib.Z_SYNC_FLUSH,
+				finishFlush: zlib.Z_SYNC_FLUSH
+			};
+
 			// for gzip
 			if (codings == 'gzip' || codings == 'x-gzip') {
-				body = body.pipe(zlib.createGunzip());
+				body = body.pipe(zlib.createGunzip(zlibOptions));
 				resolve(new Response(body, response_options));
 				return;
 			}
@@ -156,9 +165,9 @@ export default function fetch(url, opts) {
 				raw.once('data', chunk => {
 					// see http://stackoverflow.com/questions/37519828
 					if ((chunk[0] & 0x0F) === 0x08) {
-						body = body.pipe(zlib.createInflate());
+						body = body.pipe(zlib.createInflate(zlibOptions));
 					} else {
-						body = body.pipe(zlib.createInflateRaw());
+						body = body.pipe(zlib.createInflateRaw(zlibOptions));
 					}
 					resolve(new Response(body, response_options));
 				});
diff --git a/test/server.js b/test/server.js
@@ -70,6 +70,16 @@ export default class TestServer {
 			});
 		}
 
+		if (p === '/gzip-truncated') {
+			res.statusCode = 200;
+			res.setHeader('Content-Type', 'text/plain');
+			res.setHeader('Content-Encoding', 'gzip');
+			zlib.gzip('hello world', function(err, buffer) {
+				// truncate the CRC checksum and size check at the end of the stream
+				res.end(buffer.slice(0, buffer.length - 8));
+			});
+		}
+
 		if (p === '/deflate') {
 			res.statusCode = 200;
 			res.setHeader('Content-Type', 'text/plain');
diff --git a/test/test.js b/test/test.js
@@ -513,6 +513,17 @@ describe('node-fetch', () => {
 		});
 	});
 
+	it('should decompress slightly invalid gzip response', function() {
+		url = `${base}gzip-truncated`;
+		return fetch(url).then(res => {
+			expect(res.headers.get('content-type')).to.equal('text/plain');
+			return res.text().then(result => {
+				expect(result).to.be.a('string');
+				expect(result).to.equal('hello world');
+			});
+		});
+	});
+
 	it('should decompress deflate response', function() {
 		url = `${base}deflate`;
 		return fetch(url).then(res => {
