Skip to content

Commit 0998eda

Browse files
majeskiCommit Bot
majeski
authored and
Commit Bot
committed
[test] Disable reduce result on the main process
Since we're not winning anything by changing the result between processors on the main process, reduce is noop there and result is immutable. Bug: v8:6917 Change-Id: Ieb282e7abd4ab31162aee6b52493a6e1b6a25109 Cq-Include-Trybots: luci.v8.try:v8_linux64_fyi_rel_ng Reviewed-on: https://chromium-review.googlesource.com/878239 Commit-Queue: Michał Majewski <majeski@google.com> Reviewed-by: Michael Achenbach <machenbach@chromium.org> Cr-Commit-Position: refs/heads/master@{#50784}
1 parent a262b54 commit 0998eda

6 files changed

Lines changed: 54 additions & 50 deletions

File tree

‎tools/testrunner/objects/output.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,11 @@
2727

2828

2929
import signal
30+
import copy
3031

3132
from ..local import utils
3233

34+
3335
class Output(object):
3436

3537
def __init__(self, exit_code, timed_out, stdout, stderr, pid, duration):
@@ -40,6 +42,13 @@ def __init__(self, exit_code, timed_out, stdout, stderr, pid, duration):
4042
self.pid = pid
4143
self.duration = duration
4244

45+
def without_text(self):
46+
"""Returns copy of the output without stdout and stderr."""
47+
other = copy.copy(self)
48+
other.stdout = None
49+
other.stderr = None
50+
return other
51+
4352
def HasCrashed(self):
4453
if utils.IsWindows():
4554
return 0x80000000 & self.exit_code and not (0x3FFFFF00 & self.exit_code)

‎tools/testrunner/objects/predictable.py‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,6 @@ def __init__(self, _outproc):
3131
super(OutProc, self).__init__()
3232
self._outproc = _outproc
3333

34-
def process(self, output):
35-
return Result(self.has_unexpected_output(output), output)
36-
3734
def has_unexpected_output(self, output):
3835
return output.exit_code != 0
3936

‎tools/testrunner/outproc/base.py‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,10 @@
22
# Use of this source code is governed by a BSD-style license that can be
33
# found in the LICENSE file.
44

5-
import collections
65
import itertools
76

7+
from ..testproc.base import (
8+
DROP_RESULT, DROP_OUTPUT, DROP_PASS_OUTPUT, DROP_PASS_STDOUT)
89
from ..local import statusfile
910
from ..testproc.result import Result
1011

@@ -14,12 +15,30 @@
1415

1516

1617
class BaseOutProc(object):
17-
def process(self, output):
18-
return Result(self.has_unexpected_output(output), output)
18+
def process(self, output, reduction=None):
19+
has_unexpected_output = self.has_unexpected_output(output)
20+
return self._create_result(has_unexpected_output, output, reduction)
1921

2022
def has_unexpected_output(self, output):
2123
return self.get_outcome(output) not in self.expected_outcomes
2224

25+
def _create_result(self, has_unexpected_output, output, reduction):
26+
"""Creates Result instance. When reduction is passed it tries to drop some
27+
parts of the result to save memory and time needed to send the result
28+
across process boundary. None disables reduction and full result is created.
29+
"""
30+
if reduction == DROP_RESULT:
31+
return None
32+
if reduction == DROP_OUTPUT:
33+
return Result(has_unexpected_output, None)
34+
if not has_unexpected_output:
35+
if reduction == DROP_PASS_OUTPUT:
36+
return Result(has_unexpected_output, None)
37+
if reduction == DROP_PASS_STDOUT:
38+
return Result(has_unexpected_output, output.without_text())
39+
40+
return Result(has_unexpected_output, output)
41+
2342
def get_outcome(self, output):
2443
if output.HasCrashed():
2544
return statusfile.CRASH

‎tools/testrunner/testproc/base.py‎

Lines changed: 8 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -37,31 +37,6 @@
3737
DROP_PASS_OUTPUT = 2
3838
DROP_PASS_STDOUT = 3
3939

40-
def get_reduce_result_function(requirement):
41-
if requirement == DROP_RESULT:
42-
return lambda _: None
43-
44-
if requirement == DROP_OUTPUT:
45-
def f(result):
46-
result.output = None
47-
return result
48-
return f
49-
50-
if requirement == DROP_PASS_OUTPUT:
51-
def f(result):
52-
if not result.has_unexpected_output:
53-
result.output = None
54-
return result
55-
return f
56-
57-
if requirement == DROP_PASS_STDOUT:
58-
def f(result):
59-
if not result.has_unexpected_output:
60-
result.output.stdout = None
61-
result.output.stderr = None
62-
return result
63-
return f
64-
6540

6641
class TestProc(object):
6742
def __init__(self):
@@ -90,8 +65,14 @@ def setup(self, requirement=DROP_RESULT):
9065
self._prev_requirement = requirement
9166
if self._next_proc:
9267
self._next_proc.setup(max(requirement, self._requirement))
93-
if self._prev_requirement < self._requirement:
94-
self._reduce_result = get_reduce_result_function(self._prev_requirement)
68+
69+
# Since we're not winning anything by droping part of the result we are
70+
# dropping the whole result or pass it as it is. The real reduction happens
71+
# during result creation (in the output processor), so the result is
72+
# immutable.
73+
if (self._prev_requirement < self._requirement and
74+
self._prev_requirement == DROP_RESULT):
75+
self._reduce_result = lambda _: None
9576

9677
def next_test(self, test):
9778
"""

‎tools/testrunner/testproc/execution.py‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,12 @@ def run_job(job, process_context):
1515
return job.run(process_context)
1616

1717

18-
def create_process_context(requirement):
19-
return ProcessContext(base.get_reduce_result_function(requirement))
18+
def create_process_context(result_reduction):
19+
return ProcessContext(result_reduction)
2020

2121

2222
JobResult = collections.namedtuple('JobResult', ['id', 'result'])
23-
ProcessContext = collections.namedtuple('ProcessContext', ['reduce_result_f'])
23+
ProcessContext = collections.namedtuple('ProcessContext', ['result_reduction'])
2424

2525

2626
class Job(object):
@@ -32,9 +32,8 @@ def __init__(self, test_id, cmd, outproc, keep_output):
3232

3333
def run(self, process_ctx):
3434
output = self.cmd.execute()
35-
result = self.outproc.process(output)
36-
if not self.keep_output:
37-
result = process_ctx.reduce_result_f(result)
35+
reduction = process_ctx.result_reduction if not self.keep_output else None
36+
result = self.outproc.process(output, reduction)
3837
return JobResult(self.test_id, result)
3938

4039

‎tools/testrunner/testproc/progress.py‎

Lines changed: 10 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22
# Use of this source code is governed by a BSD-style license that can be
33
# found in the LICENSE file.
44

5-
import copy
65
import json
76
import os
87
import sys
@@ -70,25 +69,25 @@ def _on_next_test(self, test):
7069
def _on_result_for(self, test, result):
7170
# TODO(majeski): Support for dummy/grouped results
7271
if result.has_unexpected_output:
73-
self._failed.append((test, result.cmd, copy.copy(result.output)))
72+
self._failed.append((test, result))
7473

7574
def finished(self):
7675
crashed = 0
7776
print
78-
for test, cmd, output in self._failed:
77+
for test, result in self._failed:
7978
print_failure_header(test)
80-
if output.stderr:
79+
if result.output.stderr:
8180
print "--- stderr ---"
82-
print output.stderr.strip()
83-
if output.stdout:
81+
print result.output.stderr.strip()
82+
if result.output.stdout:
8483
print "--- stdout ---"
85-
print output.stdout.strip()
86-
print "Command: %s" % cmd.to_string()
87-
if output.HasCrashed():
88-
print "exit code: %d" % output.exit_code
84+
print result.output.stdout.strip()
85+
print "Command: %s" % result.cmd.to_string()
86+
if result.output.HasCrashed():
87+
print "exit code: %d" % result.output.exit_code
8988
print "--- CRASHED ---"
9089
crashed += 1
91-
if output.HasTimedOut():
90+
if result.output.HasTimedOut():
9291
print "--- TIMEOUT ---"
9392
if len(self._failed) == 0:
9493
print "==="

0 commit comments

Comments
 (0)