Mercurial > p > roundup > code
comparison roundup/configuration.py @ 6814:3f60a71b0812
Summary: Support selecion session/otk data store. Add redis as data store.
Allow admin to select the backend data store. Compatibility matrix:
main\/ session>| anydbm | sqlite | redis | mysql | postgresql |
anydbm | D | | X | | |
sqlite | X | D | X | | |
mysql | | | | D | |
postgresql | | | | | D |
--------------------------------------------------------------+
D - default if unconfigured, X - compatible choice
DETAILS
roundup/configuration.py:
add config.ini section sessiondb with settings: backend and redis_url.
CHANGES.txt, doc/admin_guide.txt, doc/installation.txt, doc/upgrading.txt:
doc on config of session db and redis. Plus some other fixes:
admin - clarified why we do not drop __words and __testids
table in native-fts conversion. TYpo fix.
upgrading - doc how you can keep using anydbm for session data with
sqlite. Fix dupe sentence in an upgrading config.ini
section.
roundup/backends/back_anydbm.py, roundup/backends/back_sqlite.py:
code to support redis, redis/anydbm backends respectively.
roundup/backends/sessions_redis.py
new storage backend for redis.
roundup/rest.py, roundup/cgi/actions.py, roundup/cgi/templating.py
redis uses a different way of calculating lifetime/timestamp.
Since expiration of an item occurred if its timestamp was more
than 1 week old, code would calculate:
now - 1 week + lifetime.
But this results in faster expiration in redis if used for
lifetime/timestamp.
Convert code to use the lifetime() method in BasicDatabase
that generates the right timestamp for each backend.
test/session_common.py:
added tests for more cases, get without default, getall non-existing
key etc. timestamp test changed to use new self.get_ts which is
overridden in other tests. Test that datatypes survive storage.
test/test_redis_session.py:
test redis session store with sqlite and anydbm primary databases
test/test_anydbm.py, test/test_sqlite.py
add test to make sure the databases are properly set up
sqlite - add test cases where anydbm is used as datastore
anydbm - remove updateTimestamp override add get_ts().
test/test_config.py
tests on redis_url and compatibility on choice of sessiondb backend
.travis.yml:
add redis db and redis-py
| author | John Rouillard <rouilj@ieee.org> |
|---|---|
| date | Thu, 04 Aug 2022 14:41:58 -0400 |
| parents | 031996eb9bb5 |
| children | 9ff091537f43 |
comparison
equal
deleted
inserted
replaced
| 6813:6b636fb29740 | 6814:3f60a71b0812 |
|---|---|
| 799 | 799 |
| 800 class SecretNullableOption(NullableOption, SecretOption): | 800 class SecretNullableOption(NullableOption, SecretOption): |
| 801 # use get from SecretOption and rest from NullableOption | 801 # use get from SecretOption and rest from NullableOption |
| 802 get = SecretOption.get | 802 get = SecretOption.get |
| 803 class_description = SecretOption.class_description | 803 class_description = SecretOption.class_description |
| 804 | |
| 805 class RedisUrlOption(SecretNullableOption): | |
| 806 """Do required check to make sure known bad parameters are not | |
| 807 put in the url. | |
| 808 | |
| 809 Should I do more URL validation? Validate schema: | |
| 810 redis, rediss, unix? How many cycles to invest | |
| 811 to keep users from their own mistakes? | |
| 812 """ | |
| 813 | |
| 814 class_description = SecretNullableOption.class_description | |
| 815 | |
| 816 def str2value(self, value): | |
| 817 if value and value.find("decode_responses") != -1: | |
| 818 raise OptionValueError(self, value, "URL must not include " | |
| 819 "decode_responses. Please remove " | |
| 820 "the option.") | |
| 821 return value | |
| 822 | |
| 823 class SessiondbBackendOption(Option): | |
| 824 """Make sure that sessiondb is compatile with the primary db. | |
| 825 Fail with error and suggestions if they are incompatible. | |
| 826 """ | |
| 827 | |
| 828 compatibility_matrix = [ | |
| 829 ('anydbm', 'anydbm'), | |
| 830 ('anydbm', 'redis'), | |
| 831 ('sqlite', 'anydbm'), | |
| 832 ('sqlite', 'sqlite'), | |
| 833 ('sqlite', 'redis'), | |
| 834 ('mysql', 'mysql'), | |
| 835 ('postgresql', 'postgresql'), | |
| 836 ] | |
| 837 | |
| 838 def validate(self, options): | |
| 839 ''' make sure session db is compatible with primary db. | |
| 840 also if redis is specified make sure it's available. | |
| 841 suggest valid session db backends include redis if | |
| 842 available. | |
| 843 ''' | |
| 844 | |
| 845 if self.name == 'SESSIONDB_BACKEND': | |
| 846 rdbms_backend = options['RDBMS_BACKEND']._value | |
| 847 sessiondb_backend = self._value | |
| 848 | |
| 849 if not sessiondb_backend: | |
| 850 # unset will choose default | |
| 851 return | |
| 852 | |
| 853 redis_available = False | |
| 854 try: | |
| 855 import redis | |
| 856 redis_available = True | |
| 857 except ImportError: | |
| 858 if sessiondb_backend == 'redis': | |
| 859 valid_session_backends = ', '.join(sorted(list( | |
| 860 [ x[1] for x in self.compatibility_matrix | |
| 861 if x[0] == rdbms_backend and x[1] != 'redis']) | |
| 862 )) | |
| 863 raise OptionValueError(self, sessiondb_backend, | |
| 864 "Unable to load redis module. Please install " | |
| 865 "a redis library or choose\n an alternate " | |
| 866 "session db: %(valid_session_backends)s"%locals()) | |
| 867 | |
| 868 if ( (rdbms_backend, sessiondb_backend) not in | |
| 869 self.compatibility_matrix ): | |
| 870 | |
| 871 valid_session_backends = ', '.join(sorted(list( | |
| 872 set([ x[1] for x in self.compatibility_matrix | |
| 873 if x[0] == rdbms_backend and | |
| 874 ( redis_available or x[1] != 'redis')]) | |
| 875 ))) | |
| 876 | |
| 877 raise OptionValueError(self, sessiondb_backend, | |
| 878 "You can not use session db type: %(sessiondb_backend)s " | |
| 879 "with %(rdbms_backend)s.\n Valid session db types: " | |
| 880 "%(valid_session_backends)s."%locals()) | |
| 804 | 881 |
| 805 | 882 |
| 806 class TimezoneOption(Option): | 883 class TimezoneOption(Option): |
| 807 | 884 |
| 808 class_description = \ | 885 class_description = \ |
| 1365 "cursor, this avoids caching large amounts of data in the\n" | 1442 "cursor, this avoids caching large amounts of data in the\n" |
| 1366 "client. This option only applies for the postgresql backend."), | 1443 "client. This option only applies for the postgresql backend."), |
| 1367 ), "Settings in this section (except for backend) are used" | 1444 ), "Settings in this section (except for backend) are used" |
| 1368 " by RDBMS backends only." | 1445 " by RDBMS backends only." |
| 1369 ), | 1446 ), |
| 1447 ("sessiondb", ( | |
| 1448 (SessiondbBackendOption, "backend", "", | |
| 1449 "Set backend for storing one time key (otk) and session data.\n" | |
| 1450 "Values have to be compatible with main backend.\n" | |
| 1451 "main\\/ session>| anydbm | sqlite | redis | mysql | postgresql |\n" | |
| 1452 " anydbm | D | | X | | |\n" | |
| 1453 " sqlite | X | D | X | | |\n" | |
| 1454 " mysql | | | | D | |\n" | |
| 1455 " postgresql | | | | | D |\n" | |
| 1456 " -------------------------------------------------------------+\n" | |
| 1457 " D - default if unset, X - compatible choice"), | |
| 1458 (RedisUrlOption, "redis_url", | |
| 1459 "redis://localhost:6379/0?health_check_interval=2", | |
| 1460 "URL used to connect to redis. Default uses unauthenticated\n" | |
| 1461 "redis database 0 running on localhost with default port.\n"), | |
| 1462 ), "Choose configuration for session and one time key storage."), | |
| 1370 ("logging", ( | 1463 ("logging", ( |
| 1371 (FilePathOption, "config", "", | 1464 (FilePathOption, "config", "", |
| 1372 "Path to configuration file for standard Python logging module.\n" | 1465 "Path to configuration file for standard Python logging module.\n" |
| 1373 "If this option is set, logging configuration is loaded\n" | 1466 "If this option is set, logging configuration is loaded\n" |
| 1374 "from specified file; options 'filename' and 'level'\n" | 1467 "from specified file; options 'filename' and 'level'\n" |