|
30 | 30 |
|
31 | 31 | from acme import challenges |
32 | 32 |
|
33 | | -import requests |
34 | | - |
35 | 33 | def run_client_tests(): |
36 | 34 | root = os.environ.get("CERTBOT_PATH") |
37 | 35 | assert root is not None, ( |
@@ -83,6 +81,94 @@ def expect(target_time, num, table): |
83 | 81 | expect(now, 0, "authz") |
84 | 82 | expect(after_grace_period, 1, "authz") |
85 | 83 |
|
| 84 | +def run_janitor(): |
| 85 | + # Set the fake clock to a year in the future such that all of the database |
| 86 | + # rows created during the integration tests are older than the grace period. |
| 87 | + now = datetime.datetime.utcnow() |
| 88 | + target_time = now+datetime.timedelta(days=+365) |
| 89 | + |
| 90 | + e = os.environ.copy() |
| 91 | + e.setdefault("GORACE", "halt_on_error=1") |
| 92 | + e.setdefault("FAKECLOCK", fakeclock(target_time)) |
| 93 | + |
| 94 | + # Note: Must use exec here so that killing this process kills the command. |
| 95 | + cmdline = "exec ./bin/boulder-janitor --config test/config-next/janitor.json" |
| 96 | + p = subprocess.Popen(cmdline, shell=True, env=e) |
| 97 | + |
| 98 | + # Wait for the janitor to come up |
| 99 | + waitport(8014, "boulder-janitor", None) |
| 100 | + |
| 101 | + def statline(statname, table): |
| 102 | + # NOTE: we omit the trailing "}}" to make this match general enough to |
| 103 | + # permit new labels in the future. |
| 104 | + return "janitor_{0}{{table=\"{1}\"".format(statname, table) |
| 105 | + |
| 106 | + def get_stat_line(port, stat): |
| 107 | + url = "http://localhost:%d/metrics" % port |
| 108 | + response = requests.get(url) |
| 109 | + for l in response.content.split("\n"): |
| 110 | + if l.strip().startswith(stat): |
| 111 | + return l |
| 112 | + return None |
| 113 | + |
| 114 | + def stat_value(line): |
| 115 | + parts = line.split(" ") |
| 116 | + if len(parts) != 2: |
| 117 | + raise Exception("stat line {0} was missing required parts".format(line)) |
| 118 | + return parts[1] |
| 119 | + |
| 120 | + # Wait for the janitor to report it isn't finding new work |
| 121 | + print("waiting for boulder-janitor work to complete...\n") |
| 122 | + workDone = False |
| 123 | + for i in range(10): |
| 124 | + certStatusWorkbatch = get_stat_line(8014, statline("workbatch", "certificateStatus")) |
| 125 | + certsWorkBatch = get_stat_line(8014, statline("workbatch", "certificates")) |
| 126 | + certsPerNameWorkBatch = get_stat_line(8014, statline("workbatch", "certificatesPerName")) |
| 127 | + |
| 128 | + allReady = True |
| 129 | + for line in [certStatusWorkbatch, certsWorkBatch, certsPerNameWorkBatch]: |
| 130 | + if stat_value(line) != "0": |
| 131 | + allReady = False |
| 132 | + |
| 133 | + if allReady is False: |
| 134 | + print("not done after check {0}. Sleeping".format(i)) |
| 135 | + time.sleep(2) |
| 136 | + else: |
| 137 | + workDone = True |
| 138 | + break |
| 139 | + |
| 140 | + if workDone is False: |
| 141 | + raise Exception("Timed out waiting for janitor to report all work completed\n") |
| 142 | + |
| 143 | + # Check deletion stats are not empty/zero |
| 144 | + for i in range(10): |
| 145 | + certStatusDeletes = get_stat_line(8014, statline("deletions", "certificateStatus")) |
| 146 | + certsDeletes = get_stat_line(8014, statline("deletions", "certificates")) |
| 147 | + certsPerNameDeletes = get_stat_line(8014, statline("deletions", "certificatesPerName")) |
| 148 | + |
| 149 | + if certStatusDeletes is None or certsDeletes is None or certsPerNameDeletes is None: |
| 150 | + print("delete stats not present after check {0}. Sleeping".format(i)) |
| 151 | + time.sleep(2) |
| 152 | + continue |
| 153 | + |
| 154 | + for l in [certStatusDeletes, certsDeletes, certsPerNameDeletes]: |
| 155 | + if stat_value(l) == "0": |
| 156 | + raise Exception("Expected a non-zero number of deletes to be performed. Found {0}".format(l)) |
| 157 | + |
| 158 | + # Check that all error stats are empty |
| 159 | + errorStats = [ |
| 160 | + statline("errors", "certificateStatus"), |
| 161 | + statline("errors", "certificates"), |
| 162 | + statline("errors", "certificatesPerName"), |
| 163 | + ] |
| 164 | + for eStat in errorStats: |
| 165 | + actual = get_stat_line(8014, eStat) |
| 166 | + if actual is not None: |
| 167 | + raise Exception("Expected to find no error stat lines but found {0}\n".format(eStat)) |
| 168 | + |
| 169 | + # Terminate the janitor |
| 170 | + p.terminate() |
| 171 | + |
86 | 172 | def test_single_ocsp(): |
87 | 173 | """Run the single-ocsp command, which is used to generate OCSP responses for |
88 | 174 | intermediate certificates on a manual basis. Then start up an |
@@ -187,6 +273,12 @@ def main(): |
187 | 273 | check_balance() |
188 | 274 | if not CONFIG_NEXT: |
189 | 275 | run_expired_authz_purger() |
| 276 | + |
| 277 | + # Run the boulder-janitor. This should happen after all other tests because |
| 278 | + # it runs with the fake clock set to the future and deletes rows that may |
| 279 | + # otherwise be referenced by tests. |
| 280 | + run_janitor() |
| 281 | + |
190 | 282 | # Run the load-generator last. run_loadtest will stop the |
191 | 283 | # pebble-challtestsrv before running the load-generator and will not restart |
192 | 284 | # it. |
|
0 commit comments