Mercurial > p > roundup > code

comparison test/test_liveserver.py @ 6458:8f1b91756457

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
issue2551147 - Enable compression of http responses in roundup. gzip, (brotli/zstd with optional packages) on the fly compression/content-encoding enabled by default. Can serve pre-compressed static assets as well if the client can accept it. Docs updated. Also added example nginx config to installation.txt. The config allows nginx to compress data on the fly. If the config is used, dynamic compression in roundup can be disabled. Dedicating this checkin to my father Paul Hector Rouillard 1930-2021. I did much of the development in this changeset while sitting with him as he slept/transitioned. Without his encouragement and example, my desire to learn would not be what it is and I wouldn't be half the person I am.
author John Rouillard <rouilj@ieee.org>
date Sat, 24 Jul 2021 16:31:36 -0400
parents 2a2da73e1e26
children 0e86ea84e59d
comparison
equal deleted inserted replaced
6457:dc59051807b6 6458:8f1b91756457
1 import shutil, errno, pytest 1 import shutil, errno, pytest, json, gzip, os
2 2
3 from roundup.anypy.strings import b2s
3 from roundup.cgi.wsgi_handler import RequestDispatcher 4 from roundup.cgi.wsgi_handler import RequestDispatcher
4 from .wsgi_liveserver import LiveServerTestCase 5 from .wsgi_liveserver import LiveServerTestCase
5 from . import db_test_base 6 from . import db_test_base
6 7
7 try: 8 try:
10 except ImportError: 11 except ImportError:
11 from .pytest_patcher import mark_class 12 from .pytest_patcher import mark_class
12 skip_requests = mark_class(pytest.mark.skip( 13 skip_requests = mark_class(pytest.mark.skip(
13 reason='Skipping liveserver tests: requests library not available')) 14 reason='Skipping liveserver tests: requests library not available'))
14 15
15 16 try:
17 import brotli
18 skip_brotli = lambda func, *args, **kwargs: func
19 except ImportError:
20 from .pytest_patcher import mark_class
21 skip_brotli = mark_class(pytest.mark.skip(
22 reason='Skipping brotli tests: brotli library not available'))
23 brotli = None
24
25 try:
26 import zstd
27 skip_zstd = lambda func, *args, **kwargs: func
28 except ImportError:
29 from .pytest_patcher import mark_class
30 skip_zstd = mark_class(pytest.mark.skip(
31 reason='Skipping zstd tests: zstd library not available'))
32
16 @skip_requests 33 @skip_requests
17 class SimpleTest(LiveServerTestCase): 34 class SimpleTest(LiveServerTestCase):
18 # have chicken and egg issue here. Need to encode the base_url 35 # have chicken and egg issue here. Need to encode the base_url
19 # in the config file but we don't know it until after 36 # in the config file but we don't know it until after
20 # the server is started nd has read the config.ini. 37 # the server is started nd has read the config.ini.
38 # open the database 55 # open the database
39 cls.db = cls.instance.open('admin') 56 cls.db = cls.instance.open('admin')
40 57
41 # set the url the test instance will run at. 58 # set the url the test instance will run at.
42 cls.db.config['TRACKER_WEB'] = "http://localhost:9001/" 59 cls.db.config['TRACKER_WEB'] = "http://localhost:9001/"
60 # set up mailhost so errors get reported to debuging capture file
61 cls.db.config.MAILHOST = "localhost"
62 cls.db.config.MAIL_HOST = "localhost"
63 cls.db.config.MAIL_DEBUG = "../mail.log.t"
64
65 # enable static precompressed files
66 cls.db.config.WEB_USE_PRECOMPRESSED_FILES = 1
67
43 cls.db.config.save() 68 cls.db.config.save()
44 69
45 cls.db.commit() 70 cls.db.commit()
46 cls.db.close() 71 cls.db.close()
47 72
204 print(f.status_code) 229 print(f.status_code)
205 print(f.headers) 230 print(f.headers)
206 231
207 self.assertEqual(f.status_code, 404) 232 self.assertEqual(f.status_code, 404)
208 233
209 234 def test_ims(self):
235 ''' retreive the user_utils.js file with old and new
236 if-modified-since timestamps.
237 '''
238 from datetime import datetime
239
240 f = requests.get(self.url_base() + '/@@file/user_utils.js',
241 headers = { 'Accept-Encoding': 'gzip, foo',
242 'If-Modified-Since': 'Sun, 13 Jul 1986 01:20:00',
243 'Accept': '*/*'})
244 print(f.status_code)
245 print(f.headers)
246
247 self.assertEqual(f.status_code, 200)
248 expected = { 'Content-Type': 'application/javascript',
249 'Content-Encoding': 'gzip',
250 'Vary': 'Accept-Encoding',
251 }
252
253 # use dict comprehension to remove fields like date,
254 # etag etc. from f.headers.
255 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
256
257 # now use today's date
258 a_few_seconds_ago = datetime.now().strftime('%a, %d %b %Y %H:%M:%S GMT')
259 f = requests.get(self.url_base() + '/@@file/user_utils.js',
260 headers = { 'Accept-Encoding': 'gzip, foo',
261 'If-Modified-Since': a_few_seconds_ago,
262 'Accept': '*/*'})
263 print(f.status_code)
264 print(f.headers)
265
266 self.assertEqual(f.status_code, 304)
267 expected = { 'Content-Type': 'application/javascript',
268 'Vary': 'Accept-Encoding',
269 'Content-Length': '0',
270 }
271
272 # use dict comprehension to remove fields like date, etag
273 # etc. from f.headers.
274 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
275
276
277 def test_compression_gzipfile(self):
278 '''Get the compressed dummy file'''
279
280 # create a user_utils.js.gz file to test pre-compressed
281 # file serving code. Has custom contents to verify
282 # that I get the compressed one.
283 gzfile = "%s/html/user_utils.js.gzip"%self.dirname
284 test_text= b"Custom text for user_utils.js\n"
285
286 with gzip.open(gzfile, 'wb') as f:
287 bytes_written = f.write(test_text)
288
289 self.assertEqual(bytes_written, 30)
290
291 # test file x-fer
292 f = requests.get(self.url_base() + '/@@file/user_utils.js',
293 headers = { 'Accept-Encoding': 'gzip, foo',
294 'Accept': '*/*'})
295 print(f.status_code)
296 print(f.headers)
297
298 self.assertEqual(f.status_code, 200)
299 expected = { 'Content-Type': 'application/javascript',
300 'Content-Encoding': 'gzip',
301 'Vary': 'Accept-Encoding',
302 'Content-Length': '69',
303 }
304
305 # use dict comprehension to remove fields like date,
306 # content-length etc. from f.headers.
307 self.assertDictEqual({ key: value for (key, value) in
308 f.headers.items() if key in expected },
309 expected)
310
311
312 # check content - verify it's the .gz file not the real file.
313 self.assertEqual(f.content, test_text)
314
315 '''# verify that a different encoding request returns on the fly
316
317 # test file x-fer using br, so we get runtime compression
318 f = requests.get(self.url_base() + '/@@file/user_utils.js',
319 headers = { 'Accept-Encoding': 'br, foo',
320 'Accept': '*/*'})
321 print(f.status_code)
322 print(f.headers)
323
324 self.assertEqual(f.status_code, 200)
325 expected = { 'Content-Type': 'application/javascript',
326 'Content-Encoding': 'br',
327 'Vary': 'Accept-Encoding',
328 'Content-Length': '960',
329 }
330
331 # use dict comprehension to remove fields like date,
332 # content-length etc. from f.headers.
333 self.assertDictEqual({ key: value for (key, value) in
334 f.headers.items() if key in expected },
335 expected)
336
337 try:
338 from urllib3.response import BrotliDecoder
339 # requests has decoded br to text for me
340 data = f.content
341 except ImportError:
342 # I need to decode
343 data = brotli.decompress(f.content)
344
345 self.assertEqual(b2s(data)[0:25], '// User Editing Utilities')
346 '''
347
348 # re-request file, but now make .gzip out of date. So we get the
349 # real file compressed on the fly, not our test file.
350 os.utime(gzfile, (0,0)) # use 1970/01/01 or os base time
351
352 f = requests.get(self.url_base() + '/@@file/user_utils.js',
353 headers = { 'Accept-Encoding': 'gzip, foo',
354 'Accept': '*/*'})
355 print(f.status_code)
356 print(f.headers)
357
358 self.assertEqual(f.status_code, 200)
359 expected = { 'Content-Type': 'application/javascript',
360 'Content-Encoding': 'gzip',
361 'Vary': 'Accept-Encoding',
362 }
363
364 # use dict comprehension to remove fields like date,
365 # content-length etc. from f.headers.
366 self.assertDictEqual({ key: value for (key, value) in
367 f.headers.items() if key in expected },
368 expected)
369
370
371 # check content - verify it's the real file, not crafted .gz.
372 self.assertEqual(b2s(f.content)[0:25], '// User Editing Utilities')
373
374 # cleanup
375 os.remove(gzfile)
376
377 def test_compression_gzip(self):
378 # use basic auth for rest endpoint
379 f = requests.get(self.url_base() + '/rest/data/user/1/username',
380 auth=('admin', 'sekrit'),
381 headers = {'content-type': "",
382 'Accept-Encoding': 'gzip, foo',
383 'Accept': '*/*'})
384 print(f.status_code)
385 print(f.headers)
386
387 self.assertEqual(f.status_code, 200)
388 expected = { 'Content-Type': 'application/json',
389 'Access-Control-Allow-Origin': '*',
390 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-HTTP-Method-Override',
391 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH',
392 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, PUT, DELETE, PATCH',
393 'Content-Encoding': 'gzip',
394 'Vary': 'Accept-Encoding',
395 }
396
397 content_str = '''{ "data": {
398 "id": "1",
399 "link": "http://localhost:9001/rest/data/user/1/username",
400 "data": "admin"
401 }
402 }'''
403 content = json.loads(content_str)
404
405
406 if (type("") == type(f.content)):
407 json_dict = json.loads(f.content)
408 else:
409 json_dict = json.loads(b2s(f.content))
410
411 # etag wil not match, creation date different
412 del(json_dict['data']['@etag'])
413
414 # type is "class 'str'" under py3, "type 'str'" py2
415 # just skip comparing it.
416 del(json_dict['data']['type'])
417
418 self.assertDictEqual(json_dict, content)
419
420 # use dict comprehension to remove fields like date,
421 # content-length etc. from f.headers.
422 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
423
424
425
426 # use basic auth for rest endpoint, error case, bad attribute
427 f = requests.get(self.url_base() + '/rest/data/user/1/foo',
428 auth=('admin', 'sekrit'),
429 headers = {'content-type': "",
430 'Accept-Encoding': 'gzip, foo',
431 'Accept': '*/*'})
432 print(f.status_code)
433 print(f.headers)
434
435 # ERROR: attribute error turns into 405, not sure that's right.
436 # NOTE: not compressed payload too small
437 self.assertEqual(f.status_code, 405)
438 expected = { 'Content-Type': 'application/json',
439 'Access-Control-Allow-Origin': '*',
440 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-HTTP-Method-Override',
441 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH',
442 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, PUT, DELETE, PATCH',
443 }
444
445 content = { "error":
446 {
447 "status": 405,
448 "msg": "'foo'"
449 }
450 }
451
452 json_dict = json.loads(b2s(f.content))
453 self.assertDictEqual(json_dict, content)
454
455 # use dict comprehension to remove fields like date,
456 # content-length etc. from f.headers.
457 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
458
459 # test file x-fer
460 f = requests.get(self.url_base() + '/@@file/user_utils.js',
461 headers = { 'Accept-Encoding': 'gzip, foo',
462 'Accept': '*/*'})
463 print(f.status_code)
464 print(f.headers)
465
466 self.assertEqual(f.status_code, 200)
467 expected = { 'Content-Type': 'application/javascript',
468 'Content-Encoding': 'gzip',
469 'Vary': 'Accept-Encoding',
470 }
471
472 # check first few bytes.
473 self.assertEqual(b2s(f.content[0:25]), '// User Editing Utilities')
474
475 # use dict comprehension to remove fields like date,
476 # content-length etc. from f.headers.
477 self.assertDictEqual({ key: value for (key, value) in
478 f.headers.items() if key in expected },
479 expected)
480
481 # test file x-fer
482 f = requests.get(self.url_base() + '/user1',
483 headers = { 'Accept-Encoding': 'gzip, foo',
484 'Accept': '*/*'})
485 print(f.status_code)
486 print(f.headers)
487
488 self.assertEqual(f.status_code, 200)
489 expected = { 'Content-Type': 'text/html; charset=utf-8',
490 'Content-Encoding': 'gzip',
491 'Vary': 'Accept-Encoding',
492 }
493
494 # check first few bytes.
495 self.assertEqual(b2s(f.content[0:25]), '<!-- dollarId: user.item,')
496
497 # use dict comprehension to remove fields like date,
498 # content-length etc. from f.headers.
499 self.assertDictEqual({ key: value for (key, value) in
500 f.headers.items() if key in expected },
501 expected)
502
503 @skip_brotli
504 def test_compression_br(self):
505 # use basic auth for rest endpoint
506 f = requests.get(self.url_base() + '/rest/data/user/1/username',
507 auth=('admin', 'sekrit'),
508 headers = {'content-type': "",
509 'Accept-Encoding': 'br, foo',
510 'Accept': '*/*'})
511 print(f.status_code)
512 print(f.headers)
513
514 self.assertEqual(f.status_code, 200)
515 expected = { 'Content-Type': 'application/json',
516 'Access-Control-Allow-Origin': '*',
517 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-HTTP-Method-Override',
518 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH',
519 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, PUT, DELETE, PATCH',
520 'Content-Encoding': 'br',
521 'Vary': 'Accept-Encoding',
522 }
523
524 content_str = '''{ "data": {
525 "id": "1",
526 "link": "http://localhost:9001/rest/data/user/1/username",
527 "data": "admin"
528 }
529 }'''
530 content = json.loads(content_str)
531
532
533 if (type("") == type(f.content)):
534 json_dict = json.loads(f.content)
535 else:
536 json_dict = json.loads(b2s(brotli.decompress(f.content)))
537
538 # etag wil not match, creation date different
539 del(json_dict['data']['@etag'])
540
541 # type is "class 'str'" under py3, "type 'str'" py2
542 # just skip comparing it.
543 del(json_dict['data']['type'])
544
545 self.assertDictEqual(json_dict, content)
546
547 # use dict comprehension to remove fields like date,
548 # content-length etc. from f.headers.
549 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
550
551
552
553 # use basic auth for rest endpoint, error case, bad attribute
554 f = requests.get(self.url_base() + '/rest/data/user/1/foo',
555 auth=('admin', 'sekrit'),
556 headers = {'Accept-Encoding': 'br, foo',
557 'Accept': '*/*'})
558 print(f.status_code)
559 print(f.headers)
560 # ERROR: attribute error turns into 405, not sure that's right.
561 # Note: not compressed payload too small
562 self.assertEqual(f.status_code, 405)
563 expected = { 'Content-Type': 'application/json',
564 'Access-Control-Allow-Origin': '*',
565 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-HTTP-Method-Override',
566 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH',
567 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, PUT, DELETE, PATCH',
568 }
569
570 content = { "error":
571 {
572 "status": 405,
573 "msg": "'foo'"
574 }
575 }
576 json_dict = json.loads(b2s(f.content))
577
578 self.assertDictEqual(json_dict, content)
579
580 # use dict comprehension to remove fields like date,
581 # content-length etc. from f.headers.
582 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
583
584 # test file x-fer
585 f = requests.get(self.url_base() + '/@@file/user_utils.js',
586 headers = { 'Accept-Encoding': 'br, foo',
587 'Accept': '*/*'})
588 print(f.status_code)
589 print(f.headers)
590
591 self.assertEqual(f.status_code, 200)
592 expected = { 'Content-Type': 'application/javascript',
593 'Content-Encoding': 'br',
594 'Vary': 'Accept-Encoding',
595 }
596
597 try:
598 from urllib3.response import BrotliDecoder
599 # requests has decoded br to text for me
600 data = f.content
601 except ImportError:
602 # I need to decode
603 data = brotli.decompress(f.content)
604
605 # check first few bytes.
606 self.assertEqual(b2s(data)[0:25], '// User Editing Utilities')
607
608 # use dict comprehension to remove fields like date,
609 # content-length etc. from f.headers.
610 self.assertDictEqual({ key: value for (key, value) in
611 f.headers.items() if key in expected },
612 expected)
613
614 # test file x-fer
615 f = requests.get(self.url_base() + '/user1',
616 headers = { 'Accept-Encoding': 'br, foo',
617 'Accept': '*/*'})
618 print(f.status_code)
619 print(f.headers)
620
621 self.assertEqual(f.status_code, 200)
622 expected = { 'Content-Type': 'text/html; charset=utf-8',
623 'Content-Encoding': 'br',
624 'Vary': 'Accept-Encoding',
625 }
626
627 try:
628 from urllib3.response import BrotliDecoder
629 # requests has decoded br to text for me
630 data = f.content
631 except ImportError:
632 # I need to decode
633 data = brotli.decompress(f.content)
634
635 # check first few bytes.
636 self.assertEqual(b2s(data)[0:25],
637 '<!-- dollarId: user.item,')
638
639 # use dict comprehension to remove fields like date,
640 # content-length etc. from f.headers.
641 self.assertDictEqual({ key: value for (key, value) in
642 f.headers.items() if key in expected },
643 expected)
644
645
646 @skip_zstd
647 def test_compression_zstd(self):
648 # use basic auth for rest endpoint
649 f = requests.get(self.url_base() + '/rest/data/user/1/username',
650 auth=('admin', 'sekrit'),
651 headers = {'content-type': "",
652 'Accept-Encoding': 'zstd, foo',
653 'Accept': '*/*'})
654 print(f.status_code)
655 print(f.headers)
656
657 self.assertEqual(f.status_code, 200)
658 expected = { 'Content-Type': 'application/json',
659 'Access-Control-Allow-Origin': '*',
660 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-HTTP-Method-Override',
661 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH',
662 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, PUT, DELETE, PATCH',
663 'Content-Encoding': 'zstd',
664 'Vary': 'Accept-Encoding',
665 }
666
667 content_str = '''{ "data": {
668 "id": "1",
669 "link": "http://localhost:9001/rest/data/user/1/username",
670 "data": "admin"
671 }
672 }'''
673 content = json.loads(content_str)
674
675
676 if (type("") == type(f.content)):
677 json_dict = json.loads(f.content)
678 else:
679 json_dict = json.loads(b2s(zstd.decompress(f.content)))
680
681 # etag wil not match, creation date different
682 del(json_dict['data']['@etag'])
683
684 # type is "class 'str'" under py3, "type 'str'" py2
685 # just skip comparing it.
686 del(json_dict['data']['type'])
687
688 self.assertDictEqual(json_dict, content)
689
690 # use dict comprehension to remove fields like date,
691 # content-length etc. from f.headers.
692 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
693
694
695
696 # use basic auth for rest endpoint, error case, bad attribute
697 f = requests.get(self.url_base() + '/rest/data/user/1/foo',
698 auth=('admin', 'sekrit'),
699 headers = {'content-type': "",
700 'Accept-Encoding': 'zstd, foo',
701 'Accept': '*/*'})
702 print(f.status_code)
703 print(f.headers)
704
705 # ERROR: attribute error turns into 405, not sure that's right.
706 # Note: not compressed, payload too small
707 self.assertEqual(f.status_code, 405)
708 expected = { 'Content-Type': 'application/json',
709 'Access-Control-Allow-Origin': '*',
710 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-HTTP-Method-Override',
711 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH',
712 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, PUT, DELETE, PATCH',
713 }
714
715 content = { "error":
716 {
717 "status": 405,
718 "msg": "'foo'"
719 }
720 }
721
722 json_dict = json.loads(b2s(f.content))
723 self.assertDictEqual(json_dict, content)
724
725 # use dict comprehension to remove fields like date,
726 # content-length etc. from f.headers.
727 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
728
729 # test file x-fer
730 f = requests.get(self.url_base() + '/@@file/user_utils.js',
731 headers = { 'Accept-Encoding': 'zstd, foo',
732 'Accept': '*/*'})
733 print(f.status_code)
734 print(f.headers)
735
736 self.assertEqual(f.status_code, 200)
737 expected = { 'Content-Type': 'application/javascript',
738 'Content-Encoding': 'zstd',
739 'Vary': 'Accept-Encoding',
740 }
741
742 # check first few bytes.
743 self.assertEqual(b2s(zstd.decompress(f.content)[0:25]), '// User Editing Utilities')
744
745 # use dict comprehension to remove fields like date,
746 # content-length etc. from f.headers.
747 self.assertDictEqual({ key: value for (key, value) in
748 f.headers.items() if key in expected },
749 expected)
750
751 # test file x-fer
752 f = requests.get(self.url_base() + '/user1',
753 headers = { 'Accept-Encoding': 'zstd, foo',
754 'Accept': '*/*'})
755 print(f.status_code)
756 print(f.headers)
757
758 self.assertEqual(f.status_code, 200)
759 expected = { 'Content-Type': 'text/html; charset=utf-8',
760 'Content-Encoding': 'zstd',
761 'Vary': 'Accept-Encoding',
762 }
763
764 # check first few bytes.
765 self.assertEqual(b2s(zstd.decompress(f.content)[0:25]),
766 '<!-- dollarId: user.item,')
767
768 # use dict comprehension to remove fields like date,
769 # content-length etc. from f.headers.
770 self.assertDictEqual({ key: value for (key, value) in
771 f.headers.items() if key in expected },
772 expected)
Roundup Issue Tracker: http://roundup-tracker.org/