Mercurial > p > roundup > code
view test/test_hyperdbvals.py @ 7853:03c1b7ae3a68
issue2551328/issue2551264 unneeded next link and total_count incorrect
Fix: issue2551328 - REST results show next link if number of
results is a multiple of page size. (Found by members of
team 3 in the UMass-Boston CS682 Spring 2024 class.)
issue2551264 - REST X-Total-Count header and @total_size
count incorrect when paginated
These issues arose because we retrieved the exact number of rows
from the database as requested by the user using the @page_size
parameter. With this changeset, we retrieve up to 10 million + 1
rows from the database. If the total number of rows exceeds 10
million, we set the total_count indicators to -1 as an invalid
size. (The max number of requested rows (default 10 million +1)
can be modified by the admin through interfaces.py.)
By retrieving more data than necessary, we can calculate the
total count by adding @page_index*@page_size to the number of
rows returned by the query.
Furthermore, since we return more than @page_size rows, we can
determine the existence of a row at @page_size+1 and use that
information to determine if a next link should be
provided. Previously, a next link was returned if @page_size rows
were retrieved.
This change does not guarantee that the user will get @page_size
rows returned. Access policy filtering occurs after the rows are
returned, and discards rows inaccessible by the user.
Using the current @page_index/@page_size it would be difficult to
have the roundup code refetch data and make sure that a full
@page_size set of rows is returned. E.G. @page_size=100 and 5 of
them are dropped due to access restrictions. We then fetch 10
items and add items 1-4 and 6 (5 is inaccessible). There is no
way to calculate the new database offset at:
@page_index*@page_size + 6 from the URL. We would need to add an
@page_offset=6 or something.
This could work since the client isn't adding 1 to @page_index to
get the next page. Thanks to HATEOAS, the client just uses the
'next' url. But I am not going to cross that bridge without a
concrete use case.
This can also be handled client side by merging a short response
with the next response and re-paginating client side.
Also added extra index markers to the docs to highlight use of
interfaces.py.
| author | John Rouillard <rouilj@ieee.org> |
|---|---|
| date | Mon, 01 Apr 2024 09:57:16 -0400 |
| parents | 69e6bcb39f90 |
| children |
line wrap: on
line source
# # Copyright (c) 2003 Richard Jones, richard@commonground.com.au # This module is free software, and you may redistribute it and/or modify # under the same terms as Python, so long as this copyright message and # disclaimer are retained in their original form. # # This module is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. import unittest, os, shutil, errno, sys, difflib, re from hashlib import sha1 try: from unittest.mock import Mock except ImportError: # python 2.7 from mock import Mock from roundup import init, instance, password, hyperdb, date class TestClass: def getprops(self): return { 'string': hyperdb.String(), 'number': hyperdb.Number(), 'integer': hyperdb.Integer(), 'boolean': hyperdb.Boolean(), 'password': hyperdb.Password(), 'date': hyperdb.Date(), 'interval': hyperdb.Interval(), 'link': hyperdb.Link('test'), 'linkkeyonly': hyperdb.Link('test', try_id_parsing='no'), 'link2': hyperdb.Link('test2'), 'multilink': hyperdb.Multilink('test'), 'multilink2': hyperdb.Multilink('test2'), 'multilink3': hyperdb.Multilink('test', try_id_parsing='no'), } def getkey(self): return 'string' def lookup(self, value): if value == 'valid': return '1' if value == '2valid': return '2' raise KeyError def get(self, nodeid, propname): assert propname.startswith('multilink') assert nodeid is not None return ['2', '3'] class TestClass2: def properties(self): return { 'string': hyperdb.String(), } def getkey(self): return None def labelprop(self, default_to_id=1): return 'id' class TestDatabase: classes = {'test': TestClass(), 'test2': TestClass2()} def getUserTimezone(self): return 0 config = Mock(PASSWORD_PBKDF2_DEFAULT_ROUNDS = 1000) class RawToHyperdbTest(unittest.TestCase): def _test(self, propname, value, itemid=None): return hyperdb.rawToHyperdb(TestDatabase(), TestClass(), itemid, propname, value) def testString(self): self.assertEqual(self._test('password', ''), None) self.assertEqual(self._test('string', ' a string '), 'a string') def testNumber(self): self.assertEqual(self._test('password', ''), None) self.assertEqual(self._test('number', ' 10 '), 10) self.assertEqual(self._test('number', ' 1.5 '), 1.5) self.assertEqual(self._test('number', ' -1022.5 '), -1022.5) def testInteger(self): self.assertEqual(self._test('integer', ' 100 '), 100) self.assertEqual(self._test('integer', ' 0 '), 0) self.assertEqual(self._test('integer', ' -100 '), -100) # make sure error raised on string self.assertRaises(hyperdb.HyperdbValueError, self._test, 'integer', 'a string', 'a string') # make sure error raised on real number self.assertRaises(hyperdb.HyperdbValueError, self._test, 'integer', ' -100.2 ') def testBoolean(self): self.assertEqual(self._test('password', ''), None) for true in 'yes true on 1'.split(): self.assertEqual(self._test('boolean', ' %s '%true), 1) for false in 'no false off 0'.split(): self.assertEqual(self._test('boolean', ' %s '%false), 0) def testPassword(self): self.assertEqual(self._test('password', ''), None) self.assertEqual(self._test('password', ' a string '), 'a string') val = self._test('password', ' a string ') self.assertTrue(isinstance(val, password.Password)) val = self._test('password', '{plaintext}a string') self.assertTrue(isinstance(val, password.Password)) try: import crypt val = self._test('password', '{crypt}a string') self.assertTrue(isinstance(val, password.Password)) except ImportError: pass s = sha1(b'a string').hexdigest() val = self._test('password', '{SHA}'+s) self.assertTrue(isinstance(val, password.Password)) self.assertEqual(val, 'a string') self.assertRaises(hyperdb.HyperdbValueError, self._test, 'password', '{fubar}a string') def testDate(self): self.assertEqual(self._test('password', ''), None) val = self._test('date', ' 2003-01-01 ') self.assertTrue(isinstance(val, date.Date)) val = self._test('date', ' 2003/01/01 ') self.assertTrue(isinstance(val, date.Date)) val = self._test('date', ' 2003/1/1 ') self.assertTrue(isinstance(val, date.Date)) val = self._test('date', ' 2003-1-1 ') self.assertTrue(isinstance(val, date.Date)) self.assertRaises(hyperdb.HyperdbValueError, self._test, 'date', 'fubar') def testInterval(self): self.assertEqual(self._test('password', ''), None) val = self._test('interval', ' +1d ') self.assertTrue(isinstance(val, date.Interval)) self.assertRaises(hyperdb.HyperdbValueError, self._test, 'interval', 'fubar') def testLink(self): self.assertEqual(self._test('link', '1'), '1') self.assertEqual(self._test('link', 'valid'), '1') self.assertEqual(self._test('linkkeyonly', 'valid'), '1') self.assertRaises(hyperdb.HyperdbValueError, self._test, 'link', 'invalid') self.assertRaises(hyperdb.HyperdbValueError, self._test, 'linkkeyonly', '1') self.assertRaises(hyperdb.HyperdbValueError, self._test, 'linkkeyonly', 'invalid') def testMultilink(self): self.assertEqual(self._test('multilink', '', '1'), []) self.assertEqual(self._test('multilink', '1', '1'), ['1']) self.assertEqual(self._test('multilink', 'valid', '1'), ['1']) self.assertRaises(hyperdb.HyperdbValueError, self._test, 'multilink', 'invalid', '1') self.assertEqual(self._test('multilink', '+1', '1'), ['1', '2', '3']) self.assertEqual(self._test('multilink', '+valid', '1'), ['1', '2', '3']) self.assertEqual(self._test('multilink', '+1,-2', '1'), ['1', '3']) self.assertEqual(self._test('multilink', '+valid,-3', '1'), ['1', '2']) self.assertEqual(self._test('multilink', '+1', None), ['1']) self.assertEqual(self._test('multilink', '+valid', None), ['1']) self.assertEqual(self._test('multilink', '', None), []) def testMultilink3(self): # note that all +1, -2 type references will fail with exceptions # '+1' is an id and try_id_parsing is set to no for multilink3 # and the 'name/key' 1 or 2 doesn't exist. self.assertEqual(self._test('multilink3', '', '1'), []) with self.assertRaises(hyperdb.HyperdbValueError) as cm: self._test('multilink3', '1', '1') self.assertEqual(cm.exception.args, ("property multilink3: '1' is not a test.",)) self.assertEqual(self._test('multilink3', 'valid', '1'), ['1']) self.assertRaises(hyperdb.HyperdbValueError, self._test, 'multilink3', 'invalid', '1') with self.assertRaises(hyperdb.HyperdbValueError) as cm: self._test('multilink3', '+1', '1') self.assertEqual(cm.exception.args, ("property multilink3: '1' is not a test.",)) self.assertEqual(self._test('multilink3', '+valid', '1'), ['1', '2', '3']) with self.assertRaises(hyperdb.HyperdbValueError) as cm: self._test('multilink3', '+1,-2', '1') self.assertEqual(cm.exception.args, ("property multilink3: '1' is not a test.",)) with self.assertRaises(hyperdb.HyperdbValueError) as cm: self._test('multilink3', '+valid,-2', '1') self.assertEqual(cm.exception.args, ("property multilink3: '2' is not a test.",)) self.assertEqual(self._test('multilink3', '+valid,-2valid', '1'), ['1', '3']) self.assertEqual(self._test('multilink3', '+valid', None), ['1']) self.assertEqual(self._test('multilink3', '', None), []) self.assertEqual(self._test('multilink3', '-valid', None), []) # vim: set filetype=python ts=4 sw=4 et si
