Mercurial > p > roundup > code

comparison test/test_liveserver.py @ 6693:9a1f5e496e6c

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
issue2551203 - Add support for CORS preflight request Add support for unauthenticated CORS preflight and fix headers for CORS. client.py: pass through unauthenticated CORS preflight to rest backend. Normal rest OPTION handlers (including tracker defined extensions) can see and handle the request. make some error cases return error json with crrect mime type rather than plain text tracebacks. create new functions to verify origin and referer that filter using allowed origins setting. remove tracker base url from error message is referer is not at an allowed origin. rest.py: fix up OPTION methods handlers to include Access-Control-Allow-Methods that are the same as the Allow header. set cache to one week for all Access-Control headers for CORS preflight only. remove self.client.setHeader("Access-Control-Allow-Origin", "*") and set Access-Control-Allow-Origin to the client supplied origin if it passes allowed origin checks. Required for CORS otherwise data isn't available to caller. Set for all responses. set Vary header now includes Origin as responses can differ based on Origin for all responses. set Access-Control-Allow-Credentials to true on all responses. test_liveserver.py: run server with setting to enforce origin csrf header check run server with setting to enforce x-requested-with csrf header check run server with setting for allowed_api_origins requests now set required csrf headers test preflight request on collections check new headers and Origin is no longer '*' rewrite all compression checks to use a single method with argument to use different compression methods. Reduce a lot of code duplication and makes updating for new headers easier. test_cgi: test new error messages in client.py account for new headers test preflight and new code paths
author John Rouillard <rouilj@ieee.org>
date Tue, 07 Jun 2022 09:39:35 -0400
parents a193653d6fa4
children d32d43e4a5ba
comparison
equal deleted inserted replaced
6692:1fbfb4a277d7 6693:9a1f5e496e6c
66 cls.db.config['TRACKER_WEB'] = "http://localhost:9001/" 66 cls.db.config['TRACKER_WEB'] = "http://localhost:9001/"
67 # set up mailhost so errors get reported to debuging capture file 67 # set up mailhost so errors get reported to debuging capture file
68 cls.db.config.MAILHOST = "localhost" 68 cls.db.config.MAILHOST = "localhost"
69 cls.db.config.MAIL_HOST = "localhost" 69 cls.db.config.MAIL_HOST = "localhost"
70 cls.db.config.MAIL_DEBUG = "../_test_tracker_mail.log" 70 cls.db.config.MAIL_DEBUG = "../_test_tracker_mail.log"
71 cls.db.config.WEB_CSRF_ENFORCE_HEADER_ORIGIN = "required"
72 cls.db.config.WEB_ALLOWED_API_ORIGINS = "https://client.com"
73
74 cls.db.config['WEB_CSRF_ENFORCE_HEADER_X-REQUESTED-WITH'] = "required"
71 75
72 # enable static precompressed files 76 # enable static precompressed files
73 cls.db.config.WEB_USE_PRECOMPRESSED_FILES = 1 77 cls.db.config.WEB_USE_PRECOMPRESSED_FILES = 1
74 78
75 cls.db.config.save() 79 cls.db.config.save()
235 f = requests.get(self.url_base() + "/@@file/style.css", headers=hdrs) 239 f = requests.get(self.url_base() + "/@@file/style.css", headers=hdrs)
236 self.assertEqual(f.status_code, 416) 240 self.assertEqual(f.status_code, 416)
237 self.assertEqual(f.headers['content-range'], 241 self.assertEqual(f.headers['content-range'],
238 "bytes */%s"%expected_length) 242 "bytes */%s"%expected_length)
239 243
244 def test_rest_preflight_collection(self):
245 # no auth for rest csrf preflight
246 f = requests.options(self.url_base() + '/rest/data/user',
247 headers = {'content-type': "",
248 'x-requested-with': "rest",
249 'Access-Control-Request-Headers':
250 "x-requested-with",
251 'Access-Control-Request-Method': "PUT",
252 'Origin': "https://client.com"})
253 print(f.status_code)
254 print(f.headers)
255 print(f.content)
256
257 self.assertEqual(f.status_code, 204)
258
259 expected = { 'Access-Control-Allow-Origin': 'https://client.com',
260 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override',
261 'Allow': 'OPTIONS, GET, POST',
262 'Access-Control-Allow-Methods': 'OPTIONS, GET, POST',
263 'Access-Control-Allow-Credentials': 'true',
264 }
265
266 # use dict comprehension to filter headers to the ones we want to check
267 self.assertEqual({ key: value for (key, value) in
268 f.headers.items() if key in expected },
269 expected)
270
271 # use invalid Origin
272 f = requests.options(self.url_base() + '/rest/data/user',
273 headers = {'content-type': "application/json",
274 'x-requested-with': "rest",
275 'Access-Control-Request-Headers':
276 "x-requested-with",
277 'Access-Control-Request-Method': "PUT",
278 'Origin': "ZZZ"})
279
280 self.assertEqual(f.status_code, 400)
281
282 expected = '{ "error": { "status": 400, "msg": "Client is not ' \
283 'allowed to use Rest Interface." } }'
284 self.assertEqual(b2s(f.content), expected)
285
286
240 def test_rest_invalid_method_collection(self): 287 def test_rest_invalid_method_collection(self):
241 # use basic auth for rest endpoint 288 # use basic auth for rest endpoint
242 f = requests.put(self.url_base() + '/rest/data/user', 289 f = requests.put(self.url_base() + '/rest/data/user',
243 auth=('admin', 'sekrit'), 290 auth=('admin', 'sekrit'),
244 headers = {'content-type': "", 291 headers = {'content-type': "",
245 'x-requested-with': "rest"}) 292 'X-Requested-With': "rest",
293 'Origin': "https://client.com"})
246 print(f.status_code) 294 print(f.status_code)
247 print(f.headers) 295 print(f.headers)
248 print(f.content) 296 print(f.content)
249 297
250 self.assertEqual(f.status_code, 405) 298 self.assertEqual(f.status_code, 405)
251 expected = { 'Access-Control-Allow-Origin': '*', 299 expected = { 'Access-Control-Allow-Origin': 'https://client.com',
252 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override', 300 'Access-Control-Allow-Credentials': 'true',
253 'Allow': 'DELETE, GET, OPTIONS, POST', 301 'Allow': 'DELETE, GET, OPTIONS, POST',
254 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, POST, PUT, DELETE, PATCH',
255 } 302 }
256 303
257 print(f.headers) 304 print(f.headers)
258 # use dict comprehension to remove fields like date, 305 # use dict comprehension to remove fields like date,
259 # content-length etc. from f.headers. 306 # content-length etc. from f.headers.
275 322
276 def test_rest_endpoint_root_options(self): 323 def test_rest_endpoint_root_options(self):
277 # use basic auth for rest endpoint 324 # use basic auth for rest endpoint
278 f = requests.options(self.url_base() + '/rest', 325 f = requests.options(self.url_base() + '/rest',
279 auth=('admin', 'sekrit'), 326 auth=('admin', 'sekrit'),
280 headers = {'content-type': ""}) 327 headers = {'content-type': "",
328 'Origin': "http://localhost:9001",
329 })
281 print(f.status_code) 330 print(f.status_code)
282 print(f.headers) 331 print(f.headers)
283 332
284 self.assertEqual(f.status_code, 204) 333 self.assertEqual(f.status_code, 204)
285 expected = { 'Access-Control-Allow-Origin': '*', 334 expected = { 'Access-Control-Allow-Origin': 'http://localhost:9001',
286 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override', 335 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override',
287 'Allow': 'OPTIONS, GET', 336 'Allow': 'OPTIONS, GET',
288 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, POST, PUT, DELETE, PATCH', 337 'Access-Control-Allow-Credentials': 'true',
338 'Access-Control-Allow-Methods': 'OPTIONS, GET',
339 'Access-Control-Allow-Credentials': 'true',
289 } 340 }
290 341
291 # use dict comprehension to remove fields like date, 342 # use dict comprehension to remove fields like date,
292 # content-length etc. from f.headers. 343 # content-length etc. from f.headers.
293 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected) 344 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
294 345
295 def test_rest_endpoint_data_options(self): 346 def test_rest_endpoint_data_options(self):
296 # use basic auth for rest endpoint 347 # use basic auth for rest endpoint
297 f = requests.options(self.url_base() + '/rest/data', 348 f = requests.options(self.url_base() + '/rest/data',
298 auth=('admin', 'sekrit'), 349 auth=('admin', 'sekrit'),
299 headers = {'content-type': ""} 350 headers = {'content-type': "",
300 ) 351 'Origin': "http://localhost:9001",
352 })
301 print(f.status_code) 353 print(f.status_code)
302 print(f.headers) 354 print(f.headers)
303 355
304 self.assertEqual(f.status_code, 204) 356 self.assertEqual(f.status_code, 204)
305 expected = { 'Access-Control-Allow-Origin': '*', 357 expected = { 'Access-Control-Allow-Origin': 'http://localhost:9001',
306 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override', 358 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override',
307 'Allow': 'OPTIONS, GET', 359 'Allow': 'OPTIONS, GET',
308 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, POST, PUT, DELETE, PATCH', 360 'Access-Control-Allow-Methods': 'OPTIONS, GET',
361 'Access-Control-Allow-Credentials': 'true',
309 } 362 }
310 363
311 # use dict comprehension to remove fields like date, 364 # use dict comprehension to remove fields like date,
312 # content-length etc. from f.headers. 365 # content-length etc. from f.headers.
313 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected) 366 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
314 367
315 def test_rest_endpoint_collection_options(self): 368 def test_rest_endpoint_collection_options(self):
316 # use basic auth for rest endpoint 369 # use basic auth for rest endpoint
317 f = requests.options(self.url_base() + '/rest/data/user', 370 f = requests.options(self.url_base() + '/rest/data/user',
318 auth=('admin', 'sekrit'), 371 auth=('admin', 'sekrit'),
319 headers = {'content-type': ""}) 372 headers = {'content-type': "",
373 'Origin': "http://localhost:9001",
374 })
320 print(f.status_code) 375 print(f.status_code)
321 print(f.headers) 376 print(f.headers)
322 377
323 self.assertEqual(f.status_code, 204) 378 self.assertEqual(f.status_code, 204)
324 expected = { 'Access-Control-Allow-Origin': '*', 379 expected = { 'Access-Control-Allow-Origin': 'http://localhost:9001',
325 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override', 380 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override',
326 'Allow': 'OPTIONS, GET, POST', 381 'Allow': 'OPTIONS, GET, POST',
327 'Access-Control-Allow-Methods': 'OPTIONS, GET, POST', 382 'Access-Control-Allow-Methods': 'OPTIONS, GET, POST',
383 'Access-Control-Allow-Credentials': 'true',
328 } 384 }
329 385
330 # use dict comprehension to remove fields like date, 386 # use dict comprehension to remove fields like date,
331 # content-length etc. from f.headers. 387 # content-length etc. from f.headers.
332 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected) 388 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
334 390
335 def test_rest_endpoint_item_options(self): 391 def test_rest_endpoint_item_options(self):
336 392
337 f = requests.options(self.url_base() + '/rest/data/user/1', 393 f = requests.options(self.url_base() + '/rest/data/user/1',
338 auth=('admin', 'sekrit'), 394 auth=('admin', 'sekrit'),
339 headers = {'content-type': ""}) 395 headers = {'content-type': "",
396 'Origin': "http://localhost:9001",
397 })
340 print(f.status_code) 398 print(f.status_code)
341 print(f.headers) 399 print(f.headers)
342 400
343 self.assertEqual(f.status_code, 204) 401 self.assertEqual(f.status_code, 204)
344 expected = { 'Access-Control-Allow-Origin': '*', 402 expected = { 'Access-Control-Allow-Origin': 'http://localhost:9001',
345 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override', 403 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override',
346 'Allow': 'OPTIONS, GET, PUT, DELETE, PATCH', 404 'Allow': 'OPTIONS, GET, PUT, DELETE, PATCH',
347 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, POST, PUT, DELETE, PATCH', 405 'Access-Control-Allow-Methods': 'OPTIONS, GET, PUT, DELETE, PATCH',
406 'Access-Control-Allow-Credentials': 'true',
348 } 407 }
349 408
350 # use dict comprehension to remove fields like date, 409 # use dict comprehension to remove fields like date,
351 # content-length etc. from f.headers. 410 # content-length etc. from f.headers.
352 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected) 411 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
353 412
354 def test_rest_endpoint_attribute_options(self): 413 def test_rest_endpoint_attribute_options(self):
355 # use basic auth for rest endpoint 414 # use basic auth for rest endpoint
356 f = requests.options(self.url_base() + '/rest/data/user/1/username', 415 f = requests.options(self.url_base() + '/rest/data/user/1/username',
357 auth=('admin', 'sekrit'), 416 auth=('admin', 'sekrit'),
358 headers = {'content-type': ""}) 417 headers = {'content-type': "",
418 'Origin': "http://localhost:9001",
419 })
359 print(f.status_code) 420 print(f.status_code)
360 print(f.headers) 421 print(f.headers)
361 422
362 self.assertEqual(f.status_code, 204) 423 self.assertEqual(f.status_code, 204)
363 expected = { 'Access-Control-Allow-Origin': '*', 424 expected = { 'Access-Control-Allow-Origin': 'http://localhost:9001',
364 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override', 425 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override',
365 'Allow': 'OPTIONS, GET, PUT, DELETE, PATCH', 426 'Allow': 'OPTIONS, GET, PUT, DELETE, PATCH',
366 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, POST, PUT, DELETE, PATCH', 427 'Access-Control-Allow-Methods': 'OPTIONS, GET, PUT, DELETE, PATCH',
428 'Access-Control-Allow-Credentials': 'true',
367 } 429 }
368 430
369 # use dict comprehension to remove fields like date, 431 # use dict comprehension to remove fields like date,
370 # content-length etc. from f.headers. 432 # content-length etc. from f.headers.
371 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected) 433 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
372 434
373 ## test a read only property. 435 ## test a read only property.
374 436
375 f = requests.options(self.url_base() + '/rest/data/user/1/creator', 437 f = requests.options(self.url_base() + '/rest/data/user/1/creator',
376 auth=('admin', 'sekrit'), 438 auth=('admin', 'sekrit'),
377 headers = {'content-type': ""}) 439 headers = {'content-type': "",
440 'Origin': "http://localhost:9001",
441 })
378 print(f.status_code) 442 print(f.status_code)
379 print(f.headers) 443 print(f.headers)
380 444
381 self.assertEqual(f.status_code, 204) 445 self.assertEqual(f.status_code, 204)
382 expected1 = dict(expected) 446 expected1 = dict(expected)
383 expected1['Allow'] = 'OPTIONS, GET' 447 expected1['Allow'] = 'OPTIONS, GET'
448 expected1['Access-Control-Allow-Methods'] = 'OPTIONS, GET'
384 449
385 # use dict comprehension to remove fields like date, 450 # use dict comprehension to remove fields like date,
386 # content-length etc. from f.headers. 451 # content-length etc. from f.headers.
387 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected1) 452 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected1)
388 453
414 self.assertEqual(f.status_code, 204) 479 self.assertEqual(f.status_code, 204)
415 expected = { 'Access-Control-Allow-Origin': '*', 480 expected = { 'Access-Control-Allow-Origin': '*',
416 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override', 481 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override',
417 'Allow': 'OPTIONS, GET', 482 'Allow': 'OPTIONS, GET',
418 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, POST, PUT, DELETE, PATCH', 483 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, POST, PUT, DELETE, PATCH',
484 'Access-Control-Allow-Credentials': 'true',
419 } 485 }
420 486
421 for i in range(10): 487 for i in range(10):
422 # use basic auth for rest endpoint 488 # use basic auth for rest endpoint
423 489
583 print(f.status_code) 649 print(f.status_code)
584 print(f.headers) 650 print(f.headers)
585 651
586 self.assertEqual(f.status_code, 200) 652 self.assertEqual(f.status_code, 200)
587 expected = { 'Content-Type': 'application/json', 653 expected = { 'Content-Type': 'application/json',
588 'Access-Control-Allow-Origin': '*', 654 'Access-Control-Allow-Credentials': 'true',
589 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override',
590 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH', 655 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH',
591 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, POST, PUT, DELETE, PATCH'
592 } 656 }
593 657
594 content_str = '''{ "data": { 658 content_str = '''{ "data": {
595 "id": "1", 659 "id": "1",
596 "link": "http://localhost:9001/rest/data/user/1/username", 660 "link": "http://localhost:9001/rest/data/user/1/username",
622 # use dict comprehension to remove fields like date, 686 # use dict comprehension to remove fields like date,
623 # content-length etc. from f.headers. 687 # content-length etc. from f.headers.
624 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected) 688 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
625 689
626 690
627 def test_compression_gzip(self): 691 def test_compression_gzip(self, method='gzip'):
692 if method == 'gzip':
693 decompressor = None
694 elif method == 'br':
695 decompressor = brotli.decompress
696 elif method == 'zstd':
697 decompressor = zstd.decompress
698
628 # use basic auth for rest endpoint 699 # use basic auth for rest endpoint
629 f = requests.get(self.url_base() + '/rest/data/user/1/username', 700 f = requests.get(self.url_base() + '/rest/data/user/1/username',
630 auth=('admin', 'sekrit'), 701 auth=('admin', 'sekrit'),
631 headers = {'content-type': "", 702 headers = {'content-type': "",
632 'Accept-Encoding': 'gzip, foo', 703 'Accept-Encoding': '%s, foo'%method,
633 'Accept': '*/*'}) 704 'Accept': '*/*'})
634 print(f.status_code) 705 print(f.status_code)
635 print(f.headers) 706 print(f.headers)
636 707
637 self.assertEqual(f.status_code, 200) 708 self.assertEqual(f.status_code, 200)
638 expected = { 'Content-Type': 'application/json', 709 expected = { 'Content-Type': 'application/json',
639 'Access-Control-Allow-Origin': '*', 710 'Access-Control-Allow-Credentials': 'true',
640 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override',
641 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH', 711 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH',
642 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, POST, PUT, DELETE, PATCH', 712 'Content-Encoding': method,
643 'Content-Encoding': 'gzip', 713 'Vary': 'Origin, Accept-Encoding',
644 'Vary': 'Accept-Encoding',
645 } 714 }
646 715
647 content_str = '''{ "data": { 716 content_str = '''{ "data": {
648 "id": "1", 717 "id": "1",
649 "link": "http://localhost:9001/rest/data/user/1/username", 718 "link": "http://localhost:9001/rest/data/user/1/username",
650 "data": "admin" 719 "data": "admin"
651 } 720 }
652 }''' 721 }'''
653 content = json.loads(content_str) 722 content = json.loads(content_str)
654 723
655 724 print(f.content)
656 if (type("") == type(f.content)): 725 print(type(f.content))
657 json_dict = json.loads(f.content) 726
658 else: 727 try:
659 json_dict = json.loads(b2s(f.content)) 728 if (type("") == type(f.content)):
660 729 json_dict = json.loads(f.content)
661 # etag wil not match, creation date different 730 else:
731 json_dict = json.loads(b2s(f.content))
732 except (ValueError, UnicodeDecodeError):
733 # Handle error from trying to load compressed data as only
734 # gzip gets decompressed automatically
735 # ValueError - raised by loads on compressed content python2
736 # UnicodeDecodeError - raised by loads on compressed content
737 # python3
738 json_dict = json.loads(b2s(decompressor(f.content)))
739
740 # etag will not match, creation date different
662 del(json_dict['data']['@etag']) 741 del(json_dict['data']['@etag'])
663 742
664 # type is "class 'str'" under py3, "type 'str'" py2 743 # type is "class 'str'" under py3, "type 'str'" py2
665 # just skip comparing it. 744 # just skip comparing it.
666 del(json_dict['data']['type']) 745 del(json_dict['data']['type'])
667 746
668 self.assertDictEqual(json_dict, content) 747 self.assertDictEqual(json_dict, content)
669 748
670 # verify that ETag header ends with -gzip 749 # verify that ETag header ends with -<method>
671 try: 750 try:
672 self.assertRegex(f.headers['ETag'], r'^"[0-9a-f]{32}-gzip"$') 751 self.assertRegex(f.headers['ETag'], r'^"[0-9a-f]{32}-%s"$'%method)
673 except AttributeError: 752 except AttributeError:
674 # python2 no assertRegex so try substring match 753 # python2 no assertRegex so try substring match
675 self.assertEqual(33, f.headers['ETag'].rindex('-gzip"')) 754 self.assertEqual(33, f.headers['ETag'].rindex('-' + method))
676 755
677 # use dict comprehension to remove fields like date, 756 # use dict comprehension to remove fields like date,
678 # content-length etc. from f.headers. 757 # content-length etc. from f.headers.
679 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected) 758 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
680
681 759
682 760
683 # use basic auth for rest endpoint, error case, bad attribute 761 # use basic auth for rest endpoint, error case, bad attribute
684 f = requests.get(self.url_base() + '/rest/data/user/1/foo', 762 f = requests.get(self.url_base() + '/rest/data/user/1/foo',
685 auth=('admin', 'sekrit'), 763 auth=('admin', 'sekrit'),
686 headers = {'content-type': "", 764 headers = {'content-type': "",
687 'Accept-Encoding': 'gzip, foo', 765 'Accept-Encoding': '%s, foo'%method,
688 'Accept': '*/*'}) 766 'Accept': '*/*',
767 'Origin': 'ZZZZ'})
689 print(f.status_code) 768 print(f.status_code)
690 print(f.headers) 769 print(f.headers)
691 770
692 # NOTE: not compressed payload too small 771 # NOTE: not compressed payload too small
693 self.assertEqual(f.status_code, 400) 772 self.assertEqual(f.status_code, 400)
694 expected = { 'Content-Type': 'application/json', 773 expected = { 'Content-Type': 'application/json',
695 'Access-Control-Allow-Origin': '*', 774 'Access-Control-Allow-Credentials': 'true',
696 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override', 775 'Access-Control-Allow-Origin': 'ZZZZ',
697 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH', 776 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH',
698 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, POST, PUT, DELETE, PATCH', 777 'Vary': 'Origin'
699 } 778 }
700 779
701 content = { "error": 780 content = { "error":
702 { 781 {
703 "status": 400, 782 "status": 400,
712 # content-length etc. from f.headers. 791 # content-length etc. from f.headers.
713 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected) 792 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
714 793
715 # test file x-fer 794 # test file x-fer
716 f = requests.get(self.url_base() + '/@@file/user_utils.js', 795 f = requests.get(self.url_base() + '/@@file/user_utils.js',
717 headers = { 'Accept-Encoding': 'gzip, foo', 796 headers = { 'Accept-Encoding': '%s, foo'%method,
718 'Accept': '*/*'}) 797 'Accept': '*/*'})
719 print(f.status_code) 798 print(f.status_code)
720 print(f.headers) 799 print(f.headers)
721 800
722 self.assertEqual(f.status_code, 200) 801 self.assertEqual(f.status_code, 200)
723 expected = { 'Content-Type': 'application/javascript', 802 expected = { 'Content-Type': 'application/javascript',
724 'Content-Encoding': 'gzip', 803 'Content-Encoding': method,
725 'Vary': 'Accept-Encoding', 804 'Vary': 'Accept-Encoding',
726 } 805 }
727 806
807 # compare to byte string as f.content may be compressed.
808 # so running b2s on it will throw a UnicodeError
809 if f.content[0:25] == b'// User Editing Utilities':
810 # no need to decompress, urlib3.response did it for gzip and br
811 data = f.content
812 else:
813 # I need to decode
814 data = decompressor(f.content)
815
728 # check first few bytes. 816 # check first few bytes.
729 self.assertEqual(b2s(f.content[0:25]), '// User Editing Utilities') 817 self.assertEqual(b2s(data)[0:25], '// User Editing Utilities')
730 818
731 # use dict comprehension to remove fields like date, 819 # use dict comprehension to remove fields like date,
732 # content-length etc. from f.headers. 820 # content-length etc. from f.headers.
733 self.assertDictEqual({ key: value for (key, value) in 821 self.assertDictEqual({ key: value for (key, value) in
734 f.headers.items() if key in expected }, 822 f.headers.items() if key in expected },
735 expected) 823 expected)
736 824
737 # test file x-fer 825 # test file x-fer
738 f = requests.get(self.url_base() + '/user1', 826 f = requests.get(self.url_base() + '/user1',
739 headers = { 'Accept-Encoding': 'gzip, foo', 827 headers = { 'Accept-Encoding': '%s, foo'%method,
740 'Accept': '*/*'}) 828 'Accept': '*/*'})
741 print(f.status_code) 829 print(f.status_code)
742 print(f.headers) 830 print(f.headers)
743 831
744 self.assertEqual(f.status_code, 200) 832 self.assertEqual(f.status_code, 200)
745 expected = { 'Content-Type': 'text/html; charset=utf-8', 833 expected = { 'Content-Type': 'text/html; charset=utf-8',
746 'Content-Encoding': 'gzip', 834 'Content-Encoding': method,
747 'Vary': 'Accept-Encoding', 835 'Vary': 'Accept-Encoding',
748 } 836 }
749 837
838 if f.content[0:25] == b'<!-- dollarId: user.item,':
839 # no need to decompress, urlib3.response did it for gzip and br
840 data = f.content
841 else:
842 # I need to decode
843 data = decompressor(f.content)
844
750 # check first few bytes. 845 # check first few bytes.
751 self.assertEqual(b2s(f.content[0:25]), '<!-- dollarId: user.item,') 846 self.assertEqual(b2s(data[0:25]), '<!-- dollarId: user.item,')
752 847
753 # use dict comprehension to remove fields like date, 848 # use dict comprehension to remove fields like date,
754 # content-length etc. from f.headers. 849 # content-length etc. from f.headers.
755 self.assertDictEqual({ key: value for (key, value) in 850 self.assertDictEqual({ key: value for (key, value) in
756 f.headers.items() if key in expected }, 851 f.headers.items() if key in expected },
757 expected) 852 expected)
758 853
759 @skip_brotli 854 @skip_brotli
760 def test_compression_br(self): 855 def test_compression_br(self):
761 # use basic auth for rest endpoint 856 self.test_compression_gzip(method="br")
762 f = requests.get(self.url_base() + '/rest/data/user/1/username',
763 auth=('admin', 'sekrit'),
764 headers = {'content-type': "",
765 'Accept-Encoding': 'br, foo',
766 'Accept': '*/*'})
767 print(f.status_code)
768 print(f.headers)
769
770 self.assertEqual(f.status_code, 200)
771 expected = { 'Content-Type': 'application/json',
772 'Access-Control-Allow-Origin': '*',
773 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override',
774 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH',
775 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, POST, PUT, DELETE, PATCH',
776 'Content-Encoding': 'br',
777 'Vary': 'Accept-Encoding',
778 }
779
780 content_str = '''{ "data": {
781 "id": "1",
782 "link": "http://localhost:9001/rest/data/user/1/username",
783 "data": "admin"
784 }
785 }'''
786 content = json.loads(content_str)
787
788 print(f.content)
789 print(type(f.content))
790
791 try:
792 json_dict = json.loads(f.content)
793 except (ValueError, TypeError):
794 # Handle error from trying to load compressed data
795 json_dict = json.loads(b2s(brotli.decompress(f.content)))
796
797 # etag wil not match, creation date different
798 del(json_dict['data']['@etag'])
799
800 # type is "class 'str'" under py3, "type 'str'" py2
801 # just skip comparing it.
802 del(json_dict['data']['type'])
803
804 self.assertDictEqual(json_dict, content)
805
806 # verify that ETag header ends with -br
807 try:
808 self.assertRegex(f.headers['ETag'], r'^"[0-9a-f]{32}-br"$')
809 except AttributeError:
810 # python2 no assertRegex so try substring match
811 self.assertEqual(33, f.headers['ETag'].rindex('-br"'))
812
813 # use dict comprehension to remove fields like date,
814 # content-length etc. from f.headers.
815 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
816
817
818
819 # use basic auth for rest endpoint, error case, bad attribute
820 f = requests.get(self.url_base() + '/rest/data/user/1/foo',
821 auth=('admin', 'sekrit'),
822 headers = {'Accept-Encoding': 'br, foo',
823 'Accept': '*/*'})
824 print(f.status_code)
825 print(f.headers)
826
827 # Note: not compressed payload too small
828 self.assertEqual(f.status_code, 400)
829 expected = { 'Content-Type': 'application/json',
830 'Access-Control-Allow-Origin': '*',
831 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override',
832 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH',
833 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, POST, PUT, DELETE, PATCH',
834 }
835
836 content = { "error":
837 {
838 "status": 400,
839 "msg": "Invalid attribute foo"
840 }
841 }
842 json_dict = json.loads(b2s(f.content))
843
844 self.assertDictEqual(json_dict, content)
845
846 # use dict comprehension to remove fields like date,
847 # content-length etc. from f.headers.
848 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
849
850 # test file x-fer
851 f = requests.get(self.url_base() + '/@@file/user_utils.js',
852 headers = { 'Accept-Encoding': 'br, foo',
853 'Accept': '*/*'})
854 print(f.status_code)
855 print(f.headers)
856
857 self.assertEqual(f.status_code, 200)
858 expected = { 'Content-Type': 'application/javascript',
859 'Content-Encoding': 'br',
860 'Vary': 'Accept-Encoding',
861 }
862
863 try:
864 from urllib3.response import BrotliDecoder
865 # requests has decoded br to text for me
866 data = f.content
867 except ImportError:
868 # I need to decode
869 data = brotli.decompress(f.content)
870
871 # check first few bytes.
872 self.assertEqual(b2s(data)[0:25], '// User Editing Utilities')
873
874 # use dict comprehension to remove fields like date,
875 # content-length etc. from f.headers.
876 self.assertDictEqual({ key: value for (key, value) in
877 f.headers.items() if key in expected },
878 expected)
879
880 # test file x-fer
881 f = requests.get(self.url_base() + '/user1',
882 headers = { 'Accept-Encoding': 'br, foo',
883 'Accept': '*/*'})
884 print(f.status_code)
885 print(f.headers)
886
887 self.assertEqual(f.status_code, 200)
888 expected = { 'Content-Type': 'text/html; charset=utf-8',
889 'Content-Encoding': 'br',
890 'Vary': 'Accept-Encoding',
891 }
892
893 try:
894 from urllib3.response import BrotliDecoder
895 # requests has decoded br to text for me
896 data = f.content
897 except ImportError:
898 # I need to decode
899 data = brotli.decompress(f.content)
900
901 # check first few bytes.
902 self.assertEqual(b2s(data)[0:25],
903 '<!-- dollarId: user.item,')
904
905 # use dict comprehension to remove fields like date,
906 # content-length etc. from f.headers.
907 self.assertDictEqual({ key: value for (key, value) in
908 f.headers.items() if key in expected },
909 expected)
910
911 857
912 @skip_zstd 858 @skip_zstd
913 def test_compression_zstd(self): 859 def test_compression_zstd(self):
914 # use basic auth for rest endpoint 860 self.test_compression_gzip(method="zstd")
915 f = requests.get(self.url_base() + '/rest/data/user/1/username',
916 auth=('admin', 'sekrit'),
917 headers = {'content-type': "",
918 'Accept-Encoding': 'zstd, foo',
919 'Accept': '*/*'})
920 print(f.status_code)
921 print(f.headers)
922
923 self.assertEqual(f.status_code, 200)
924 expected = { 'Content-Type': 'application/json',
925 'Access-Control-Allow-Origin': '*',
926 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override',
927 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH',
928 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, POST, PUT, DELETE, PATCH',
929 'Content-Encoding': 'zstd',
930 'Vary': 'Accept-Encoding',
931 }
932
933 content_str = '''{ "data": {
934 "id": "1",
935 "link": "http://localhost:9001/rest/data/user/1/username",
936 "data": "admin"
937 }
938 }'''
939 content = json.loads(content_str)
940
941
942 try:
943 json_dict = json.loads(f.content)
944 except (ValueError, UnicodeDecodeError, TypeError):
945 # ValueError - raised by loads on compressed content python2
946 # UnicodeDecodeError - raised by loads on compressed content
947 # python3
948 json_dict = json.loads(b2s(zstd.decompress(f.content)))
949
950 # etag wil not match, creation date different
951 del(json_dict['data']['@etag'])
952
953 # type is "class 'str'" under py3, "type 'str'" py2
954 # just skip comparing it.
955 del(json_dict['data']['type'])
956
957 self.assertDictEqual(json_dict, content)
958
959 # verify that ETag header ends with -zstd
960 try:
961 self.assertRegex(f.headers['ETag'], r'^"[0-9a-f]{32}-zstd"$')
962 except AttributeError:
963 # python2 no assertRegex so try substring match
964 self.assertEqual(33, f.headers['ETag'].rindex('-zstd"'))
965
966 # use dict comprehension to remove fields like date,
967 # content-length etc. from f.headers.
968 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
969
970
971
972 # use basic auth for rest endpoint, error case, bad attribute
973 f = requests.get(self.url_base() + '/rest/data/user/1/foo',
974 auth=('admin', 'sekrit'),
975 headers = {'content-type': "",
976 'Accept-Encoding': 'zstd, foo',
977 'Accept': '*/*'})
978 print(f.status_code)
979 print(f.headers)
980
981 # Note: not compressed, payload too small
982 self.assertEqual(f.status_code, 400)
983 expected = { 'Content-Type': 'application/json',
984 'Access-Control-Allow-Origin': '*',
985 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override',
986 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH',
987 'Access-Control-Allow-Methods': 'HEAD, OPTIONS, GET, POST, PUT, DELETE, PATCH',
988 }
989
990 content = { "error":
991 {
992 "status": 400,
993 "msg": "Invalid attribute foo"
994 }
995 }
996
997 json_dict = json.loads(b2s(f.content))
998 self.assertDictEqual(json_dict, content)
999
1000 # use dict comprehension to remove fields like date,
1001 # content-length etc. from f.headers.
1002 self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected)
1003
1004 # test file x-fer
1005 f = requests.get(self.url_base() + '/@@file/user_utils.js',
1006 headers = { 'Accept-Encoding': 'zstd, foo',
1007 'Accept': '*/*'})
1008 print(f.status_code)
1009 print(f.headers)
1010
1011 self.assertEqual(f.status_code, 200)
1012 expected = { 'Content-Type': 'application/javascript',
1013 'Content-Encoding': 'zstd',
1014 'Vary': 'Accept-Encoding',
1015 }
1016
1017 # check first few bytes.
1018 self.assertEqual(b2s(zstd.decompress(f.content)[0:25]), '// User Editing Utilities')
1019
1020 # use dict comprehension to remove fields like date,
1021 # content-length etc. from f.headers.
1022 self.assertDictEqual({ key: value for (key, value) in
1023 f.headers.items() if key in expected },
1024 expected)
1025
1026 # test file x-fer
1027 f = requests.get(self.url_base() + '/user1',
1028 headers = { 'Accept-Encoding': 'zstd, foo',
1029 'Accept': '*/*'})
1030 print(f.status_code)
1031 print(f.headers)
1032
1033 self.assertEqual(f.status_code, 200)
1034 expected = { 'Content-Type': 'text/html; charset=utf-8',
1035 'Content-Encoding': 'zstd',
1036 'Vary': 'Accept-Encoding',
1037 }
1038
1039 # check first few bytes.
1040 self.assertEqual(b2s(zstd.decompress(f.content)[0:25]),
1041 '<!-- dollarId: user.item,')
1042
1043 # use dict comprehension to remove fields like date,
1044 # content-length etc. from f.headers.
1045 self.assertDictEqual({ key: value for (key, value) in
1046 f.headers.items() if key in expected },
1047 expected)
1048 861
1049 def test_cache_control_css(self): 862 def test_cache_control_css(self):
1050 f = requests.get(self.url_base() + '/@@file/style.css', 863 f = requests.get(self.url_base() + '/@@file/style.css',
1051 headers = {'content-type': "", 864 headers = {'content-type': "",
1052 'Accept': '*/*'}) 865 'Accept': '*/*'})
1067 self.assertEqual(f.headers['Cache-Control'], 'public, max-age=1209600') 880 self.assertEqual(f.headers['Cache-Control'], 'public, max-age=1209600')
1068 881
1069 def test_new_issue_with_file_upload(self): 882 def test_new_issue_with_file_upload(self):
1070 # Set up session to manage cookies <insert blue monster here> 883 # Set up session to manage cookies <insert blue monster here>
1071 session = requests.Session() 884 session = requests.Session()
885 session.headers.update({'Origin': 'http://localhost:9001'})
1072 886
1073 # login using form 887 # login using form
1074 login = {"__login_name": 'admin', '__login_password': 'sekrit', 888 login = {"__login_name": 'admin', '__login_password': 'sekrit',
1075 "@action": "login"} 889 "@action": "login"}
1076 f = session.post(self.url_base()+'/', data=login) 890 f = session.post(self.url_base()+'/', data=login)
1110 url = self.url_base() + '/rest/data/' 924 url = self.url_base() + '/rest/data/'
1111 fname = 'a-bigger-testfile' 925 fname = 'a-bigger-testfile'
1112 d = dict(name = fname, type='application/octet-stream') 926 d = dict(name = fname, type='application/octet-stream')
1113 c = dict (content = r'xyzzy') 927 c = dict (content = r'xyzzy')
1114 r = session.post(url + 'file', files = c, data = d, 928 r = session.post(url + 'file', files = c, data = d,
1115 headers = {'x-requested-with': "rest"} 929 headers = {'x-requested-with': "rest",
930 'Origin': "http://localhost:9001"}
1116 ) 931 )
1117 932
1118 # was a 500 before fix for issue2551178 933 # was a 500 before fix for issue2551178
1119 self.assertEqual(r.status_code, 201) 934 self.assertEqual(r.status_code, 201)
1120 # just compare the path leave off the number 935 # just compare the path leave off the number
1122 r.headers["location"]) 937 r.headers["location"])
1123 json_dict = json.loads(r.text) 938 json_dict = json.loads(r.text)
1124 self.assertEqual(json_dict["data"]["link"], r.headers["location"]) 939 self.assertEqual(json_dict["data"]["link"], r.headers["location"])
1125 940
1126 # download file and verify content 941 # download file and verify content
1127 r = session.get(r.headers["location"] +'/content') 942 r = session.get(r.headers["location"] +'/content',
943 headers = {'x-requested-with': "rest",
944 'Origin': "http://localhost:9001"}
945 )
1128 json_dict = json.loads(r.text) 946 json_dict = json.loads(r.text)
1129 self.assertEqual(json_dict['data']['data'], c["content"]) 947 self.assertEqual(json_dict['data']['data'], c["content"])
1130 print(r.text) 948 print(r.text)
1131 949
1132 # Upload a file via rest interface - no auth 950 # Upload a file via rest interface - no auth
1133 session.auth = None 951 session.auth = None
1134 r = session.post(url + 'file', files = c, data = d, 952 r = session.post(url + 'file', files = c, data = d,
1135 headers = {'x-requested-with': "rest"} 953 headers = {'x-requested-with': "rest",
954 'Origin': "http://localhost:9001"}
1136 ) 955 )
1137 self.assertEqual(r.status_code, 403) 956 self.assertEqual(r.status_code, 403)
1138 957
1139 # get session variable from web form login 958 # get session variable from web form login
1140 # and use it to upload file 959 # and use it to upload file
1141 # login using form 960 # login using form
1142 login = {"__login_name": 'admin', '__login_password': 'sekrit', 961 login = {"__login_name": 'admin', '__login_password': 'sekrit',
1143 "@action": "login"} 962 "@action": "login"}
1144 f = session.post(self.url_base()+'/', data=login) 963 f = session.post(self.url_base()+'/', data=login,
964 headers = {'Origin': "http://localhost:9001"}
965 )
1145 # look for change in text in sidebar post login 966 # look for change in text in sidebar post login
1146 self.assertIn('Hello, admin', f.text) 967 self.assertIn('Hello, admin', f.text)
1147 968
1148 r = session.post(url + 'file', files = c, data = d, 969 r = session.post(url + 'file', files = c, data = d,
1149 headers = {'x-requested-with': "rest"} 970 headers = {'x-requested-with': "rest",
971 'Origin': "http://localhost:9001"}
1150 ) 972 )
1151 self.assertEqual(r.status_code, 201) 973 self.assertEqual(r.status_code, 201)
1152 print(r.status_code) 974 print(r.status_code)
1153 975
1154 976
Roundup Issue Tracker: http://roundup-tracker.org/