Mercurial > p > roundup > code
diff test/test_cgi.py @ 7906:470616e64414
issue2551334 - get test suite running under windows
Fix test_cgi under windows. Few classes of errors:
1) comparing paths with a/b (unix) to a\b (windows). Use normpath to
fix it. Also change regexps used for path matching to use either
\ or / for directory separators.
2) database not being closed preventing deletion of test case
directory. Windows doesn't allow deletion of open files.
In some cases replaced calling client.inner_mail() with main()
because main() makes sure to close the database. In other cases
assigned self.db = client.db beacuse client reopens the database
and closes the original self.db. Reassigning allows tearDown to
close the last opened handle to a db.
3) commit the admin password to the database. If it's not commited
calling the code sometimes comes up with a different admin
password. Not sure why we don't see this on linux.
4) run commit() on database so that sqlite databases can be closed
and deleted. Unit tests don't call the main entry points that
have finally clauses to close the databases properly, so do it in
the test.
5) split tests that try to resolve symbolic links in the template
directory to a separate method. Under windows user needs special
permissions to creae symbolic links, so I report the method is
skipped if creating a link fails.
| author | John Rouillard <rouilj@ieee.org> |
|---|---|
| date | Sat, 27 Apr 2024 23:19:51 -0400 |
| parents | a430339f55e6 |
| children | 28aa76443f58 |
line wrap: on
line diff
--- a/test/test_cgi.py Sat Apr 27 21:03:20 2024 -0400 +++ b/test/test_cgi.py Sat Apr 27 23:19:51 2024 -0400 @@ -14,6 +14,8 @@ import pytest import copy +from os.path import normpath + from roundup.anypy.cgi_ import cgi from roundup.cgi import client, actions, exceptions from roundup.cgi.exceptions import FormError, NotFound, Redirect @@ -933,6 +935,9 @@ e2 = HTMLProperty.is_edit_ok HTMLProperty.is_edit_ok = lambda x : True cl.inner_main() + # The original self.db has been changed. Assign the new + # cl.db to self.db so it gets closed at the end of the test. + self.db = cl.db _HTMLItem.is_edit_ok = e1 HTMLProperty.is_edit_ok = e2 self.assertEqual(len(out), 1) @@ -969,6 +974,7 @@ out.append(s) cl.write_html = wh cl.main() + self.db = cl.db # to close new db handle from main() at tearDown self.assertFalse('HTTP_PROXY' in cl.env) self.assertFalse('HTTP_PROXY' in os.environ) @@ -1036,7 +1042,7 @@ # test with no headers. Default config requires that 1 header # is present and passes checks. - cl.inner_main() + cl.main() match_at=out[0].find('Unable to verify sufficient headers') print("result of subtest 1:", out[0]) self.assertNotEqual(match_at, -1) @@ -1045,7 +1051,7 @@ # all the rest of these allow at least one header to pass # and the edit happens with a redirect back to issue 1 cl.env['HTTP_REFERER'] = 'http://whoami.com/path/' - cl.inner_main() + cl.main() match_at=out[0].find('Redirecting to <a href="http://whoami.com/path/issue1?@ok_message') print("result of subtest 2:", out[0]) self.assertEqual(match_at, 0) @@ -1053,7 +1059,7 @@ del(out[0]) cl.env['HTTP_ORIGIN'] = 'http://whoami.com' - cl.inner_main() + cl.main() match_at=out[0].find('Redirecting to <a href="http://whoami.com/path/issue1?@ok_message') print("result of subtest 3:", out[0]) self.assertEqual(match_at, 0) @@ -1067,7 +1073,7 @@ # the proxy's name for the web server and not the name # thatis exposed to the world. cl.env['HTTP_HOST'] = 'frontend1.whoami.net' - cl.inner_main() + cl.main() match_at=out[0].find('Redirecting to <a href="http://whoami.com/path/issue1?@ok_message') print("result of subtest 4:", out[0]) self.assertNotEqual(match_at, -1) @@ -1076,7 +1082,7 @@ del(out[0]) cl.env['HTTP_HOST'] = 'whoami.com' - cl.inner_main() + cl.main() match_at=out[0].find('Redirecting to <a href="http://whoami.com/path/issue1?@ok_message') print("result of subtest 5:", out[0]) self.assertEqual(match_at, 0) @@ -1087,7 +1093,7 @@ cl.env['HTTP_X_FORWARDED_HOST'] = 'whoami.net' # this raises an error as the header check passes and # it did the edit and tries to send mail. - cl.inner_main() + cl.main() match_at=out[0].find('Invalid X-FORWARDED-HOST whoami.net') print("result of subtest 6:", out[0]) self.assertNotEqual(match_at, -1) @@ -1100,7 +1106,7 @@ # roundup will report a missing token. cl.db.config['WEB_CSRF_ENFORCE_TOKEN'] = 'required' - cl.inner_main() + cl.main() match_at=out[0].find("<p>We can't validate your session (csrf failure). Re-enter any unsaved data and try again.</p>") print("result of subtest 6a:", out[0], match_at) self.assertEqual(match_at, 33) @@ -1109,10 +1115,10 @@ form2 = copy.copy(form) form2.update({'@csrf': 'booogus'}) - # add a bogus csrf field to the form and rerun the inner_main + # add a bogus csrf field to the form and rerun main cl.form = db_test_base.makeForm(form2) - cl.inner_main() + cl.main() match_at=out[0].find("We can't validate your session (csrf failure). Re-enter any unsaved data and try again.") print("result of subtest 7:", out[0]) self.assertEqual(match_at, 36) @@ -1129,9 +1135,9 @@ self.assertEqual(isitthere, True) form2.update({'@csrf': nonce}) - # add a real csrf field to the form and rerun the inner_main + # add a real csrf field to the form and rerun main cl.form = db_test_base.makeForm(form2) - cl.inner_main() + cl.main() # csrf passes and redirects to the new issue. match_at=out[0].find('Redirecting to <a href="http://whoami.com/path/issue1?@ok_message') print("result of subtest 9:", out[0]) @@ -1139,7 +1145,7 @@ del(out[0]) # try a replay attack - cl.inner_main() + cl.main() # This should fail as token was wiped by last run. match_at=out[0].find("We can't validate your session (csrf failure). Re-enter any unsaved data and try again.") print("replay of csrf after post use", out[0]) @@ -1153,9 +1159,9 @@ form2 = copy.copy(form) nonce = anti_csrf_nonce(cl) form2.update({'@csrf': nonce}) - # add a real csrf field to the form and rerun the inner_main + # add a real csrf field to the form and rerun main cl.form = db_test_base.makeForm(form2) - cl.inner_main() + cl.main() # csrf passes but fail creating new issue because not a post match_at=out[0].find('<p>Invalid request</p>') print("result of subtest 11:", out[0]) @@ -1171,7 +1177,7 @@ # since get deleted the token. cl.env.update({'REQUEST_METHOD': 'POST'}) print(cl.env) - cl.inner_main() + cl.main() match_at=out[0].find("We can't validate your session (csrf failure). Re-enter any unsaved data and try again.") print("post failure after get", out[0]) print("result of subtest 13:", out[0]) @@ -1184,7 +1190,7 @@ # this should not redirect as it is not an API call. cl.db.config.WEB_ALLOWED_API_ORIGINS = " * " cl.env['HTTP_ORIGIN'] = 'https://baz.edu' - cl.inner_main() + cl.main() match_at=out[0].find('Invalid Origin https://baz.edu') print("result of subtest invalid origin:", out[0]) self.assertEqual(match_at, 36) @@ -1197,7 +1203,7 @@ cl.db.config.WEB_ALLOWED_API_ORIGINS = " * " cl.env['HTTP_ORIGIN'] = 'http://whoami.com' cl.env['HTTP_REFERER'] = 'https://baz.edu/path/' - cl.inner_main() + cl.main() match_at=out[0].find('Invalid Referer: https://baz.edu/path/') print("result of subtest invalid referer:", out[0]) self.assertEqual(match_at, 36) @@ -1216,6 +1222,7 @@ # set the password for admin so we can log in. passwd=password.Password('admin') self.db.user.set('1', password=passwd) + self.db.commit() out = [] def wh(s): @@ -1248,6 +1255,7 @@ cl.write = wh # capture output cl.handle_rest() + self.db = cl.db # to close new db handle from main() at tearDown print(b2s(out[0])) expected=""" { @@ -1297,6 +1305,7 @@ # Should return explanation because content type is text/plain # and not text/xml cl.handle_rest() + self.db = cl.db # to close new db handle from main() at tearDown self.assertIn('Access-Control-Allow-Credentials', cl.additional_headers) @@ -1327,6 +1336,7 @@ # Should return explanation because content type is text/plain # and not text/xml cl.handle_rest() + self.db = cl.db # to close new db handle from main() at tearDown self.assertNotIn('Access-Control-Allow-Credentials', cl.additional_headers) @@ -1353,6 +1363,7 @@ 'HTTP_X_REQUESTED_WITH': 'rest', }, form) cl.db = self.db + self.db = cl.db # to close new db handle from main() at tearDown cl.base = 'http://whoami.com/path/' cl._socket_op = lambda *x : True cl._error_message = [] @@ -1366,6 +1377,7 @@ cl.write = wh # capture output cl.handle_rest() + self.db = cl.db # to close new db handle from main() at tearDown self.assertEqual(json.loads(b2s(out[0])), json.loads(expected) ) @@ -1405,6 +1417,7 @@ # Should return explanation because content type is text/plain # and not text/xml cl.handle_rest() + self.db = cl.db # to close new db handle from main() at tearDown self.assertIn('Access-Control-Allow-Credentials', cl.additional_headers) @@ -1437,6 +1450,7 @@ # Should return explanation because content type is text/plain # and not text/xml cl.handle_rest() + self.db = cl.db # to close new db handle from main() at tearDown self.assertNotIn('Access-Control-Allow-Credentials', cl.additional_headers) self.assertEqual(json.loads(b2s(out[0])),json.loads(expected)) @@ -1474,6 +1488,7 @@ cl.write = wh # capture output cl.handle_rest() + self.db = cl.db # to close new db handle from handle_rest at tearDown _py3 = sys.version_info[0] > 2 @@ -1534,6 +1549,7 @@ cl.write = wh # capture output cl.handle_rest() + self.db = cl.db # to close new db handle from handle_rest at tearDown self.assertEqual(out[0], '') # 204 options returns no data expected_headers = { @@ -1588,6 +1604,7 @@ cl.write = wh # capture output cl.handle_rest() + self.db = cl.db # to close new db handle from handle_rest at tearDown self.assertEqual(cl.response_code, 400) @@ -2369,7 +2386,8 @@ # Check the returned values. cl.serve_static_file("issue.index.html") self.assertEqual(output[0][1], "text/html") - self.assertEqual(output[0][3], "_test_cgi_form/html/issue.index.html") + self.assertEqual(output[0][3], + normpath('_test_cgi_form/html/issue.index.html')) del output[0] # reset output buffer # stop searching TEMPLATES for the files. @@ -2382,7 +2400,8 @@ cl.instance.config['STATIC_FILES'] = 'html -' cl.serve_static_file("issue.index.html") self.assertEqual(output[0][1], "text/html") - self.assertEqual(output[0][3], "_test_cgi_form/html/issue.index.html") + self.assertEqual(output[0][3], + normpath('_test_cgi_form/html/issue.index.html')) del output[0] # reset output buffer # set the list of files and do not look at the templates directory @@ -2391,13 +2410,15 @@ # find file in first directory cl.serve_static_file("messagesummary.py") self.assertEqual(output[0][1], "text/x-python") - self.assertEqual(output[0][3], "_test_cgi_form/detectors/messagesummary.py") + self.assertEqual(output[0][3], + normpath( "_test_cgi_form/detectors/messagesummary.py")) del output[0] # reset output buffer # find file in second directory cl.serve_static_file("README.txt") self.assertEqual(output[0][1], "text/plain") - self.assertEqual(output[0][3], "_test_cgi_form/extensions/README.txt") + self.assertEqual(output[0][3], + normpath("_test_cgi_form/extensions/README.txt")) del output[0] # reset output buffer # make sure an embedded - ends the searching. @@ -2412,14 +2433,16 @@ # find file now in first directory cl.serve_static_file("README.txt") self.assertEqual(output[0][1], "text/plain") - self.assertEqual(output[0][3], "_test_cgi_form/detectors/README.txt") + self.assertEqual(output[0][3], + normpath("_test_cgi_form/detectors/README.txt")) del output[0] # reset output buffer cl.instance.config['STATIC_FILES'] = ' detectors extensions ' # make sure lack of trailing - allows searching TEMPLATES cl.serve_static_file("issue.index.html") self.assertEqual(output[0][1], "text/html") - self.assertEqual(output[0][3], "_test_cgi_form/html/issue.index.html") + self.assertEqual(output[0][3], + normpath("_test_cgi_form/html/issue.index.html")) del output[0] # reset output buffer # Make STATIC_FILES a single element. @@ -2427,7 +2450,8 @@ # find file now in first directory cl.serve_static_file("messagesummary.py") self.assertEqual(output[0][1], "text/x-python") - self.assertEqual(output[0][3], "_test_cgi_form/detectors/messagesummary.py") + self.assertEqual(output[0][3], + normpath("_test_cgi_form/detectors/messagesummary.py")) del output[0] # reset output buffer # make sure files found in subdirectory @@ -2436,7 +2460,8 @@ # use subdir in filename cl.serve_static_file("css/README.css") self.assertEqual(output[0][1], "text/css") - self.assertEqual(output[0][3], "_test_cgi_form/detectors/css/README.css") + self.assertEqual(output[0][3], + normpath("_test_cgi_form/detectors/css/README.css")) del output[0] # reset output buffer cl.Cache_Control['text/css'] = 'public, max-age=3600' @@ -2446,7 +2471,8 @@ f = open('_test_cgi_form/html/css/README1.css', 'a').close() cl.serve_static_file("README1.css") self.assertEqual(output[0][1], "text/css") - self.assertEqual(output[0][3], "_test_cgi_form/html/css/README1.css") + self.assertEqual(output[0][3], + normpath("_test_cgi_form/html/css/README1.css")) self.assertTrue( "Cache-Control" in cl.additional_headers ) self.assertEqual( cl.additional_headers, {'Cache-Control': 'public, max-age=3600'} ) @@ -2455,7 +2481,8 @@ cl.Cache_Control['README1.css'] = 'public, max-age=60' cl.serve_static_file("README1.css") self.assertEqual(output[0][1], "text/css") - self.assertEqual(output[0][3], "_test_cgi_form/html/css/README1.css") + self.assertEqual(output[0][3], + normpath("_test_cgi_form/html/css/README1.css")) self.assertTrue( "Cache-Control" in cl.additional_headers ) self.assertEqual( cl.additional_headers, {'Cache-Control': 'public, max-age=60'} ) @@ -2943,11 +2970,6 @@ shutil.copyfile(self.dirname + "/html/user.item.html", self.dirname + "/user.item.html") - # create link outside the html subdir. This should fail due to - # path traversal check. - os.symlink("../../user.item.html", subdir + "/user.item.html") - # it will be removed and replaced by a later test - # make sure a simple non-subdir template works. # user.item.html exists so this works. # note that the extension is not included just the basename @@ -2977,6 +2999,34 @@ r = t.selectTemplate("issue", "subdir/item") self.assertEqual("subdir/issue.item", r) + def testTemplateSubdirectory_symlink(self): + # test for templates in subdirectories using symlinks. + # this doesn't work under windows unless you have special + # permissions + + # make the directory + subdir = self.dirname + "/html/subdir" + os.mkdir(subdir) + + # get the client instance The form is needed to initialize, + # but not used since I call selectTemplate directly. + t = client.Client(self.instance, "user", + {'PATH_INFO':'/user', 'REQUEST_METHOD':'POST'}, + form=db_test_base.makeForm({"@template": "item"})) + + # create link outside the html subdir. This should fail due to + # path traversal check. + try: + os.symlink("../../user.item.html", subdir + "/user.item.html") + except OSError as e: + # windows requires special privs for symbolic links + allowed_error = 'A required privilege is not held by the client' + if not e.args[1] == allowed_error: + raise + pytest.skip("User does not have permission to create symbolic links under Windows") + + # it will be removed and replaced by a later test + # there is a self.directory + /html/subdir/user.item.html file, # but it is a link to self.dir /user.item.html which is outside # the html subdir so is rejected by the path traversal check. @@ -3197,7 +3247,7 @@ self.assertRegex(self._caplog.record_tuples[0][2], ( r"^Found an incorrect token when expandfile applied " r"string subsitution on " - r"'[^']*/_test_template/html/file_with_broken_expand_type.js'. " + r"'[^']*[\\/]_test_template[\\/]html[\\/]file_with_broken_expand_type.js'. " r"ValueError\('incomplete format'\) was raised. Check the format " r"of your named conversion specifiers.")) self._caplog.clear() @@ -3240,7 +3290,7 @@ # match the changable filename directory self.assertRegex(self._caplog.record_tuples[0][2], ( r"When running " - r"expandfile\('[^']*/_test_template/html/file_with_missing.js'\) " + r"expandfile\('[^']*[\\/]_test_template[\\/]html[\\/]file_with_missing.js'\) " r"in 'home' there was no value for token: 'idontexist'.")) self._caplog.clear() r = None @@ -3275,7 +3325,7 @@ self.assertRegex(self._caplog.record_tuples[0][2], ( r"^Found an incorrect token when expandfile applied " r"string subsitution on " - r"'[^']*/_test_template/html/file_with_bare_%.js'. " + r"'[^']*[\\/]_test_template[\\/]html[\\/]file_with_bare_%.js'. " r"ValueError\(" r"'unsupported format character ' ' \(0x20\) at index 12'\) was " r"raised. Check the format " @@ -3365,6 +3415,10 @@ self.assertEqual(result, expected) self.assertEqual(self.client.response_code, 400) + # handle outstanding commits since we are not using the + # standard entry points. + self.db.commit() + # # SECURITY # @@ -3410,6 +3464,11 @@ with self.assertRaises(NotFound) as cm: actions.ExportCSVWithIdAction(cl).handle() + # commit changes so db can be properly closed on windows. + # because we are testing the backend method and not using + # cl.main() that handles db commit/close, we need to do this. + cl.db.commit() + class SqliteNativeCgiTest(unittest.TestCase, testFtsQuery): """All of the rest of the tests use anydbm as the backend. This class tests renderContext for fulltext search.