Mercurial > p > roundup > code
comparison test/rest_common.py @ 5732:0e6ed3d72f92
Rest rate limiting code first commit. It is a bit rough and turned off
by default.
The current code is lossy. If client connections are fast enough, the
rate limiting code doesn't count every connection. So the client can
get more connections than configured if they are fast enough.
5-20% of the connections are not recorded.
| author | John Rouillard <rouilj@ieee.org> |
|---|---|
| date | Sat, 25 May 2019 16:50:25 -0400 |
| parents | 9ea2ce9d10cf |
| children | 62bdcb874433 |
comparison
equal
deleted
inserted
replaced
| 5731:058ef18af5fd | 5732:0e6ed3d72f92 |
|---|---|
| 1 import unittest | 1 import unittest |
| 2 import os | 2 import os |
| 3 import shutil | 3 import shutil |
| 4 import errno | 4 import errno |
| 5 | |
| 6 from time import sleep | |
| 7 from datetime import datetime | |
| 5 | 8 |
| 6 from roundup.cgi.exceptions import * | 9 from roundup.cgi.exceptions import * |
| 7 from roundup.hyperdb import HyperdbValueError | 10 from roundup.hyperdb import HyperdbValueError |
| 8 from roundup.exceptions import * | 11 from roundup.exceptions import * |
| 9 from roundup import password, hyperdb | 12 from roundup import password, hyperdb |
| 619 # FIXME add tests for out of range once we decide what response | 622 # FIXME add tests for out of range once we decide what response |
| 620 # is needed to: | 623 # is needed to: |
| 621 # page_size < 0 | 624 # page_size < 0 |
| 622 # page_index < 0 | 625 # page_index < 0 |
| 623 | 626 |
| 627 def testRestRateLimit(self): | |
| 628 | |
| 629 self.db.config['WEB_API_CALLS_PER_INTERVAL'] = 20 | |
| 630 self.db.config['WEB_API_INTERVAL_IN_SEC'] = 60 | |
| 631 | |
| 632 print("Now realtime start:", datetime.utcnow()) | |
| 633 # don't set an accept header; json should be the default | |
| 634 # use up all our allowed api calls | |
| 635 for i in range(20): | |
| 636 # i is 0 ... 19 | |
| 637 self.client_error_message = [] | |
| 638 results = self.server.dispatch('GET', | |
| 639 "/rest/data/user/%s/realname"%self.joeid, | |
| 640 self.empty_form) | |
| 641 | |
| 642 # is successful | |
| 643 self.assertEqual(self.server.client.response_code, 200) | |
| 644 # does not have Retry-After header as we have | |
| 645 # suceeded with this query | |
| 646 self.assertFalse("Retry-After" in | |
| 647 self.server.client.additional_headers) | |
| 648 # remaining count is correct | |
| 649 self.assertEqual( | |
| 650 self.server.client.additional_headers["X-RateLimit-Remaining"], | |
| 651 self.db.config['WEB_API_CALLS_PER_INTERVAL'] -1 - i | |
| 652 ) | |
| 653 | |
| 654 # trip limit | |
| 655 self.server.client.additional_headers.clear() | |
| 656 results = self.server.dispatch('GET', | |
| 657 "/rest/data/user/%s/realname"%self.joeid, | |
| 658 self.empty_form) | |
| 659 print(results) | |
| 660 self.assertEqual(self.server.client.response_code, 429) | |
| 661 | |
| 662 self.assertEqual( | |
| 663 self.server.client.additional_headers["X-RateLimit-Limit"], | |
| 664 self.db.config['WEB_API_CALLS_PER_INTERVAL']) | |
| 665 self.assertEqual( | |
| 666 self.server.client.additional_headers["X-RateLimit-Limit-Period"], | |
| 667 self.db.config['WEB_API_INTERVAL_IN_SEC']) | |
| 668 self.assertEqual( | |
| 669 self.server.client.additional_headers["X-RateLimit-Remaining"], | |
| 670 0) | |
| 671 # value will be almost 60. Allow 1-2 seconds for all 20 rounds. | |
| 672 self.assertAlmostEqual( | |
| 673 self.server.client.additional_headers["X-RateLimit-Reset"], | |
| 674 59, delta=1) | |
| 675 self.assertEqual( | |
| 676 str(self.server.client.additional_headers["Retry-After"]), | |
| 677 "3.0") # check as string | |
| 678 | |
| 679 print("Reset:", self.server.client.additional_headers["X-RateLimit-Reset"]) | |
| 680 print("Now realtime pre-sleep:", datetime.utcnow()) | |
| 681 sleep(3.1) # sleep as requested so we can do another login | |
| 682 print("Now realtime post-sleep:", datetime.utcnow()) | |
| 683 | |
| 684 # this should succeed | |
| 685 self.server.client.additional_headers.clear() | |
| 686 results = self.server.dispatch('GET', | |
| 687 "/rest/data/user/%s/realname"%self.joeid, | |
| 688 self.empty_form) | |
| 689 print(results) | |
| 690 print("Reset:", self.server.client.additional_headers["X-RateLimit-Reset-date"]) | |
| 691 print("Now realtime:", datetime.utcnow()) | |
| 692 print("Now ts header:", self.server.client.additional_headers["Now"]) | |
| 693 print("Now date header:", self.server.client.additional_headers["Now-date"]) | |
| 694 | |
| 695 self.assertEqual(self.server.client.response_code, 200) | |
| 696 | |
| 697 self.assertEqual( | |
| 698 self.server.client.additional_headers["X-RateLimit-Limit"], | |
| 699 self.db.config['WEB_API_CALLS_PER_INTERVAL']) | |
| 700 self.assertEqual( | |
| 701 self.server.client.additional_headers["X-RateLimit-Limit-Period"], | |
| 702 self.db.config['WEB_API_INTERVAL_IN_SEC']) | |
| 703 self.assertEqual( | |
| 704 self.server.client.additional_headers["X-RateLimit-Remaining"], | |
| 705 0) | |
| 706 self.assertFalse("Retry-After" in | |
| 707 self.server.client.additional_headers) | |
| 708 # we still need to wait a minute for everything to clear | |
| 709 self.assertAlmostEqual( | |
| 710 self.server.client.additional_headers["X-RateLimit-Reset"], | |
| 711 59, delta=1) | |
| 712 | |
| 713 # and make sure we need to wait another three seconds | |
| 714 # as we consumed the last api call | |
| 715 results = self.server.dispatch('GET', | |
| 716 "/rest/data/user/%s/realname"%self.joeid, | |
| 717 self.empty_form) | |
| 718 | |
| 719 self.assertEqual(self.server.client.response_code, 429) | |
| 720 self.assertEqual( | |
| 721 str(self.server.client.additional_headers["Retry-After"]), | |
| 722 "3.0") # check as string | |
| 723 | |
| 724 json_dict = json.loads(b2s(results)) | |
| 725 self.assertEqual(json_dict['error']['msg'], | |
| 726 "Api rate limits exceeded. Please wait: 3 seconds.") | |
| 727 | |
| 728 # reset rest params | |
| 729 self.db.config['WEB_API_CALLS_PER_INTERVAL'] = 0 | |
| 730 self.db.config['WEB_API_INTERVAL_IN_SEC'] = 3600 | |
| 731 | |
| 624 def testEtagGeneration(self): | 732 def testEtagGeneration(self): |
| 625 ''' Make sure etag generation is stable | 733 ''' Make sure etag generation is stable |
| 626 | 734 |
| 627 This mocks date.Date() when creating the target to be | 735 This mocks date.Date() when creating the target to be |
| 628 etagged. Differing dates make this test impossible. | 736 etagged. Differing dates make this test impossible. |