phpbench · dantleech · Jul 31, 2021 · Jun 22, 2024
diff --git a/docs/executors.rst b/docs/executors.rst
@@ -21,6 +21,23 @@ Options:
 
 .. include:: ./executors/options/_remote.rst
 
+.. _executor_opcode:
+
+``opcode``
+----------
+
+As with remote but enables opcache debug mode and counts the number of
+opcodes:
+
+This benchmark records:
+
+- Same as `remote`
+- The number of opcodes (`result_opcode_count`)
+
+Options:
+
+.. include:: ./executors/options/_opcode.rst
+
 .. _executor_local:
 
 ``local``

diff --git a/docs/executors/options/_opcode.rst b/docs/executors/options/_opcode.rst
@@ -0,0 +1,28 @@
+
+.. _executor_opcode_option_php_config:
+
+**php_config**:
+  Type(s): ``array``, Default: ``[]``
+
+  Key value array of ini settings, e.g. ``{"max_execution_time":100}``
+
+.. _executor_opcode_option_safe_parameters:
+
+**safe_parameters**:
+  Type(s): ``bool``, Default: ``true``
+
+  INTERNAL: Use process process-safe parameters, this option exists for backwards-compatibility and will be removed in PHPBench 2.0
+
+.. _executor_opcode_option_optimisation_stage:
+
+**optimisation_stage**:
+  Type(s): ``string``, Default: ``pre``
+
+  If the count should be `pre` or `post` optimisation
+
+.. _executor_opcode_option_dump_path:
+
+**dump_path**:
+  Type(s): ``[null, string]``, Default: ``NULL``
+
+  If specified, dump the opcode debug output to this file on each run
diff --git a/docs/report-generators/options/_expression.rst b/docs/report-generators/options/_expression.rst
@@ -53,4 +53,11 @@
 **include_baseline**:
   Type(s): ``bool``, Default: ``false``
 
-  If the baseline should be included as additional rows, or if it should be inlined
+  If the baseline should be included as additional rows, or if it should be inlined
+
+.. _generator_expression_option_hide_cols_with_missing_vars:
+
+**hide_cols_with_missing_vars**:
+  Type(s): ``bool``, Default: ``true``
+
+  Hide columns which would have produced a "variable not found" error
diff --git a/lib/Compat/SymfonyOptionsResolverCompat.php b/lib/Compat/SymfonyOptionsResolverCompat.php
@@ -6,6 +6,10 @@
 
 use function method_exists;
 
+/**
+ * @deprecated can be removed in 2.0 as we don't support a non-compatible
+ * version of option-resolver
+ */
 class SymfonyOptionsResolverCompat
 {
     /**

diff --git a/lib/Executor/Benchmark/OpcodeExecutor.php b/lib/Executor/Benchmark/OpcodeExecutor.php
@@ -0,0 +1,106 @@
+<?php
+
+namespace PhpBench\Executor\Benchmark;
+
+use PhpBench\Executor\ExecutionContext;
+use PhpBench\Executor\ExecutionResults;
+use PhpBench\Model\Result\MemoryResult;
+use PhpBench\Model\Result\OpcodeResult;
+use PhpBench\Model\Result\TimeResult;
+use PhpBench\Opcache\OpcodeDebugParser;
+use PhpBench\Registry\Config;
+use PhpBench\Remote\Launcher;
+use RuntimeException;
+use Symfony\Component\Filesystem\Filesystem;
+use Symfony\Component\OptionsResolver\OptionsResolver;
+
+final class OpcodeExecutor extends TemplateExecutor
+{
+    public const OPTION_OPTIMISATION_STAGE = 'optimisation_stage';
+    public const OPTION_DUMP_PATH = 'dump_path';
+
+    public const OPCACHE_OPTIMISATION_PRE = 'pre';
+    public const OPCACHE_OPTIMISATION_POST = 'post';
+    public const PHP_OPTION_OPCACHE_ENABLE_CLI = 'opcache.enable_cli';
+    public const PHP_OPTION_OPCACHE_DEBUG_LEVEL = 'opcache.opt_debug_level';
+
+    public function __construct(
+        Launcher $launcher,
+        private OpcodeDebugParser $parser,
+        private Filesystem $filesystem,
+    ) {
+        parent::__construct(
+            $launcher,
+            __DIR__ . '/template/remote.template',
+        );
+    }
+
+    public function execute(ExecutionContext $context, Config $config): ExecutionResults
+    {
+        $opcacheSettings = [
+            self::PHP_OPTION_MAX_EXECUTION_TIME => 0,
+            self::PHP_OPTION_OPCACHE_ENABLE_CLI => 1,
+            self::PHP_OPTION_OPCACHE_DEBUG_LEVEL => $this->resolveDebugLevel($config[self::OPTION_OPTIMISATION_STAGE]),
+        ];
+        $config->offsetSet(self::OPTION_PHP_CONFIG, array_merge(
+            $config[self::OPTION_PHP_CONFIG] ?? [],
+            $opcacheSettings,
+        ));
+
+        $result = $this->launch($context, $config);
+
+        $this->dump($config->offsetGet(self::OPTION_DUMP_PATH), $result->stderr());
+
+        $data = $result->unserializeResult();
+
+        return ExecutionResults::fromResults(
+            new OpcodeResult($this->parser->countOpcodes($result->stderr())),
+            TimeResult::fromArray($data['time']),
+            MemoryResult::fromArray($data['mem'])
+        );
+    }
+
+    /**
+     * {@inheritdoc}
+     */
+    public function configure(OptionsResolver $options): void
+    {
+        parent::configure($options);
+        $options->setDefaults([
+            self::OPTION_SAFE_PARAMETERS => true,
+            self::OPTION_OPTIMISATION_STAGE => self::OPCACHE_OPTIMISATION_PRE,
+            self::OPTION_DUMP_PATH => null,
+        ]);
+        $options->setAllowedValues(self::OPTION_OPTIMISATION_STAGE, [self::OPCACHE_OPTIMISATION_PRE, self::OPCACHE_OPTIMISATION_POST]);
+        $options->setAllowedTypes(self::OPTION_OPTIMISATION_STAGE, ['string']);
+        $options->setAllowedTypes(self::OPTION_DUMP_PATH, ['null', 'string']);
+        $options->setInfo(self::OPTION_OPTIMISATION_STAGE, 'If the count should be `pre` or `post` optimisation');
+        $options->setInfo(self::OPTION_DUMP_PATH, 'If specified, dump the opcode debug output to this file on each run');
+    }
+
+    private function resolveDebugLevel(string $stage): string
+    {
+        return $stage === self::OPCACHE_OPTIMISATION_PRE ? '0x10000' : '0x20000';
+    }
+
+    private function dump(?string $path, string $dump): void
+    {
+        if (null === $path) {
+            return;
+        }
+
+        if (!file_exists(dirname($path))) {
+            $this->filesystem->mkdir(dirname($path));
+        }
+        $written = file_put_contents($path, $dump);
+
+        if (false !== $written) {
+            return;
+        }
+
+        throw new RuntimeException(sprintf(
+            'Could not write opcode dump file to: %s',
+            $path
+        ));
+    }
+}
diff --git a/lib/Executor/Benchmark/TemplateExecutor.php b/lib/Executor/Benchmark/TemplateExecutor.php
@@ -12,6 +12,7 @@
 
 namespace PhpBench\Executor\Benchmark;
 
+use PhpBench\Remote\LaunchResult;
 use RuntimeException;
 use PhpBench\Compat\SymfonyOptionsResolverCompat;
 use PhpBench\Executor\BenchmarkExecutorInterface;
@@ -28,34 +29,23 @@
 class TemplateExecutor implements BenchmarkExecutorInterface
 {
     final public const OPTION_PHP_CONFIG = 'php_config';
+
+    /**
+     * @deprecated This will be enabled unconditionally and removed in PHPBench 2.0
+     */
     final public const OPTION_SAFE_PARAMETERS = 'safe_parameters';
 
-    private const PHP_OPTION_MAX_EXECUTION_TIME = 'max_execution_time';
+    protected const PHP_OPTION_MAX_EXECUTION_TIME = 'max_execution_time';
 
-    public function __construct(private readonly Launcher $launcher, private readonly string $templatePath)
+    public function __construct(protected readonly Launcher $launcher, private readonly string $templatePath)
     {
     }
 
     public function execute(ExecutionContext $context, Config $config): ExecutionResults
     {
-        $tokens = $this->createTokens($context, $config);
-        $payload = $this->launcher->payload($this->templatePath, $tokens, $context->getTimeout());
-        $payload->mergePhpConfig(array_merge(
-            [
-                self::PHP_OPTION_MAX_EXECUTION_TIME => 0,
-            ],
-            $config[self::OPTION_PHP_CONFIG] ?? []
-        ));
+        $result = $this->launch($context, $config);
 
-        try {
-            $result = $payload->launch();
-        } catch (ScriptErrorException $error) {
-            throw new ExecutionError(sprintf(
-                "Benchmarking script exited with code %s\n\n%s",
-                $error->getExitCode() ?? 'unknown',
-                $error->getMessage()
-            ));
-        }
+        $result = $result->unserializeResult();
 
         if (isset($result['buffer']) && $result['buffer']) {
             throw new RuntimeException(sprintf(
@@ -116,4 +106,28 @@ private function resolveParameterSet(ExecutionContext $context, Config $config):
 
         return $context->getParameterSet()->toUnserializedParameters();
     }
+
+    protected function launch(ExecutionContext $context, Config $config): LaunchResult
+    {
+        $tokens = $this->createTokens($context, $config);
+        $payload = $this->launcher->payload($this->templatePath, $tokens, $context->getTimeout());
+        $payload->mergePhpConfig(array_merge(
+            [
+                self::PHP_OPTION_MAX_EXECUTION_TIME => 0,
+            ],
+            $config[self::OPTION_PHP_CONFIG] ?? []
+        ));
+
+        try {
+            $result = $payload->launchResult();
+        } catch (ScriptErrorException $error) {
+            throw new ExecutionError(sprintf(
+                "Benchmarking script exited with code %s\n\n%s",
+                $error->getExitCode() ?? 'unknown',
+                $error->getMessage()
+            ));
+        }
+
+        return $result;
+    }
 }
diff --git a/lib/Expression/Evaluator/PrettyErrorEvaluator.php b/lib/Expression/Evaluator/PrettyErrorEvaluator.php
@@ -45,12 +45,12 @@ public function evaluate(Node $node, array $params): Node
     private function prettyError(Node $rootNode, EvaluationError $error, array $params): EvaluationError
     {
         try {
-            return new EvaluationError($error->node(), implode(PHP_EOL, [
+            return $error->withMessage(implode(PHP_EOL, [
                 sprintf('%s:', $error->getMessage()),
                 '',
                 '    ' . $this->printer->print($rootNode),
                 '    ' . $this->underlineFactory->underline($error->node())->print($rootNode),
-            ]), $error);
+            ]));
         } catch (PrinterError) {
             throw $error;
         }

diff --git a/lib/Expression/Exception/EvaluationError.php b/lib/Expression/Exception/EvaluationError.php
@@ -17,4 +17,11 @@ public function node(): Node
     {
         return $this->node;
     }
+
+    public function withMessage(string $message): self
+    {
+        $this->message = $message;
+
+        return $this;
+    }
 }
diff --git a/lib/Expression/Exception/KeyDoesNotExist.php → ...Expression/Exception/VariableNotFound.php b/lib/Expression/Exception/KeyDoesNotExist.php → ...Expression/Exception/VariableNotFound.php
@@ -2,6 +2,6 @@
 
 namespace PhpBench\Expression\Exception;
 
-class KeyDoesNotExist extends EvaluationError
+final class VariableNotFound extends EvaluationError
 {
 }
diff --git a/lib/Expression/Func/StDevFunction.php b/lib/Expression/Func/StDevFunction.php
@@ -11,6 +11,8 @@ final class StDevFunction
 {
     public function __invoke(ListNode $values, ?BooleanNode $sample = null): FloatNode
     {
-        return new FloatNode(Statistics::stdev($values->nonNullPhpValues(), $sample ? $sample->value() : false));
+        return new FloatNode(
+            Statistics::stdev($values->nonNullPhpValues(), $sample ? $sample->value() : false)
+        );
     }
 }
diff --git a/lib/Expression/NodeEvaluator/FunctionEvaluator.php b/lib/Expression/NodeEvaluator/FunctionEvaluator.php
@@ -34,6 +34,8 @@ public function evaluate(Evaluator $evaluator, Node $node, array $params): ?Node
                     return $evaluator->evaluateType($node, PhpValue::class, $params);
                 }, $this->args($node->args()))
             );
+        } catch (EvaluationError $error) {
+            throw $error;
         } catch (Throwable $throwable) {
             throw new EvaluationError($node, $throwable->getMessage(), $throwable);
         }

diff --git a/lib/Expression/NodeEvaluator/VariableEvaluator.php b/lib/Expression/NodeEvaluator/VariableEvaluator.php
@@ -6,7 +6,7 @@
 use PhpBench\Expression\Ast\PhpValueFactory;
 use PhpBench\Expression\Ast\VariableNode;
 use PhpBench\Expression\Evaluator;
-use PhpBench\Expression\Exception\EvaluationError;
+use PhpBench\Expression\Exception\VariableNotFound;
 use PhpBench\Expression\NodeEvaluator;
 
 class VariableEvaluator implements NodeEvaluator
@@ -31,7 +31,7 @@ public function evaluate(Evaluator $evaluator, Node $node, array $params): ?Node
     private function resolveFromParameters(string $key, array $params, VariableNode $node)
     {
         if (!isset($params[$key])) {
-            throw new EvaluationError(
+            throw new VariableNotFound(
                 $node,
                 sprintf(
                     'Variable "%s" not found, known variables: "%s"',

diff --git a/lib/Extension/RunnerExtension.php b/lib/Extension/RunnerExtension.php
@@ -33,6 +33,7 @@
 use PhpBench\Executor\Benchmark\DebugExecutor;
 use PhpBench\Executor\Benchmark\LocalExecutor;
 use PhpBench\Executor\Benchmark\MemoryCentricMicrotimeExecutor;
+use PhpBench\Executor\Benchmark\OpcodeExecutor;
 use PhpBench\Executor\Benchmark\RemoteExecutor;
 use PhpBench\Executor\CompositeExecutor;
 use PhpBench\Executor\Method\ErrorHandlingExecutorDecorator;
@@ -43,7 +44,9 @@
 use PhpBench\Expression\Printer;
 use PhpBench\Expression\Printer\EvaluatingPrinter;
 use PhpBench\Json\JsonDecoder;
+use Symfony\Component\Filesystem\Filesystem;
 use Symfony\Component\Filesystem\Path;
+use PhpBench\Opcache\OpcodeDebugParser;
 use PhpBench\Progress\Logger\BlinkenLogger;
 use PhpBench\Progress\Logger\DotsLogger;
 use PhpBench\Progress\Logger\HistogramLogger;
@@ -376,6 +379,16 @@ private function registerBenchmark(Container $container): void
             self::TAG_EXECUTOR => ['name' => 'debug']
         ]);
 
+        $container->register(OpcodeExecutor::class, function (Container $container) {
+            return new OpcodeExecutor(
+                $container->get(Launcher::class),
+                new OpcodeDebugParser(),
+                new Filesystem(),
+            );
+        }, [
+            self::TAG_EXECUTOR => ['name' => 'opcode']
+        ]);
+
         $container->register(Finder::class, function (Container $container) {
             return new Finder();
         });

diff --git a/lib/Extension/config/report/generators.php b/lib/Extension/config/report/generators.php
@@ -19,6 +19,8 @@
             'mem_peak',
             'mode',
             'rstdev',
+            'opcodes',
+
         ]
     ],
     'bar_chart_time' => [
@@ -41,6 +43,7 @@
             'time_avg' => 'display_as_time(first(result_time_avg), first(subject_time_unit))',
             'comp_z_value' => 'format("%+.2fσ", first(result_comp_z_value))',
             'comp_deviation' => 'format("%+.2f%%", first(result_comp_deviation))',
+            'opcodes' => 'first(result_opcode_count)',
         ],
         'aggregate' => ['benchmark_class', 'subject_name', 'variant_index', 'iteration_index'],
     ],