-
Notifications
You must be signed in to change notification settings - Fork 8
Expand file tree
/
Copy pathMultithread.php
More file actions
226 lines (192 loc) · 7.46 KB
/
Copy pathMultithread.php
File metadata and controls
226 lines (192 loc) · 7.46 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
<?php
/*
* Copyright (c) Default Value LLC.
* This source file is subject to the License https://github.com/DefaultValue/dockerizer_for_php/LICENSE.txt
* Do not change this file if you want to upgrade the tool to the newer versions in the future
* Please, contact us at https://default-value.com/#contact if you wish to customize this tool
* according to you business needs
*/
declare(strict_types=1);
namespace DefaultValue\Dockerizer\Process;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\SignalRegistry\SignalRegistry;
/**
* Currently linux hosts only, because we check available CPU and memory. Need implementation for other OSes
*/
class Multithread
{
/**
* @var array<int, float>
*/
private array $childProcessPIDs = [];
private bool $terminate = false;
/**
* @param \DefaultValue\Dockerizer\Shell\Shell $shell
*/
public function __construct(
private \DefaultValue\Dockerizer\Shell\Shell $shell
) {
}
/**
* @param callable[] $callbacks
* @param OutputInterface $output
* @param SignalRegistry $signalRegistry
* @param float $memoryRequirementsInGB
* @param int $maxThreads
* @param int $startDelay - delay starting new processes to eliminate shock from to many threads started at once
* @return void
*/
public function run(
array $callbacks,
OutputInterface $output,
SignalRegistry $signalRegistry,
float $memoryRequirementsInGB = 0.5,
int $maxThreads = 4,
int $startDelay = 10,
): void {
$maxThreads = $this->getMaxThreads($maxThreads, $memoryRequirementsInGB);
$totalCallbacks = count($callbacks);
$output->writeln(sprintf(
// phpcs:disable Generic.Files.LineLength.TooLong
'Processing %d callbacks in max %d threads (%.2fGB RAM per thread) with %ds delay before starting a new thread',
// phpcs:enable
$totalCallbacks,
$maxThreads,
$memoryRequirementsInGB,
$startDelay
));
// Send kill signal to all child processes for proper tier down
// Need to check Process::doSignal() for more info about this and `enable-sigchild`
$signalRegistry->register(
SIGINT,
function () use ($output) {
$this->terminate = true;
if (!count($this->childProcessPIDs)) {
return;
}
foreach (array_keys($this->childProcessPIDs) as $pid) {
$output->writeln("Sending SIGINT to the process <info>#$pid</info>...");
posix_kill($pid, SIGINT);
}
$output->writeln('Please, wait for the child processes to complete...');
}
);
$callbackNumber = 0;
// Handle callbacks, stop if SIGINT was received
while ($callbacks && !$this->terminate) {
$callback = array_shift($callbacks);
$lastStart = microtime(true);
$pid = pcntl_fork();
if ($pid === -1) {
throw new \RuntimeException('Error forking process');
}
// If there is no PID then this is a child process, and we can do the stuff
if (!$pid) {
try {
// Child process MUST NOT know about other process PIDs and do not handle their shutdown
$this->childProcessPIDs = [];
$callback();
} catch (\Throwable) {
exit(1);
}
exit(0);
}
// If there is PID then we're in the parent process
$this->childProcessPIDs[$pid] = microtime(true);
++$callbackNumber;
$message = sprintf(
'%s: Started new process with ID #<fg=blue>%d</fg=blue> (%d/%d)',
$this->getDateTime(),
$pid,
$callbackNumber,
$totalCallbacks
);
$output->writeln($message);
if (
$startDelay
&& (microtime(true) - $lastStart < $startDelay)
) {
$sleepTime = (int) ceil($startDelay - (microtime(true) - $lastStart));
while ($callbacks && $sleepTime--) {
$this->checkChildProcesses($output);
sleep(1);
}
$this->checkChildProcesses($output);
}
$this->waitForAtLeastOneChildToComplete($maxThreads, $output);
}
$this->waitForAtLeastOneChildToComplete(0, $output);
}
/**
* @TODO: For Docker Desktop we need to check allocated CPU and memory instead of host
*
* @param int $maxThreads
* @param float $memoryRequirementsInGB
* @return int
*/
private function getMaxThreads(int $maxThreads, float $memoryRequirementsInGB): int
{
// Get CPU cores count or at least available threads count
$process = PHP_OS_FAMILY === 'Darwin'
? $this->shell->mustRun('sysctl -n hw.physicalcpu')
: $this->shell->mustRun('grep -c ^processor /proc/cpuinfo');
$output = trim($process->getOutput());
$coresCount = (int) $output;
// Get available memory in GB
$process = PHP_OS_FAMILY === 'Darwin'
? $this->shell->mustRun('sysctl -n hw.memsize')
: $this->shell->mustRun('grep MemAvailable /proc/meminfo');
$output = trim($process->getOutput());
$availableMemoryInGb = preg_match('/^MemAvailable:\s+(\d+)\skB$/', $output, $pieces)
? ((int) $pieces[1]) / 1024 ** 2
: ((int) $output) / 1024 ** 3;
if (!$coresCount || !$availableMemoryInGb) {
throw new \RuntimeException('Can\'t analyze memory or CPU params on this host ');
}
// Leave at least one core for other tasks
return min($maxThreads, $coresCount - 1, (int) floor($availableMemoryInGb / $memoryRequirementsInGB));
}
/**
* @param int $maxThreads
* @param OutputInterface $output
* @return void
*/
private function waitForAtLeastOneChildToComplete(int $maxThreads, OutputInterface $output): void
{
while (count($this->childProcessPIDs) && count($this->childProcessPIDs) >= $maxThreads) {
$this->checkChildProcesses($output);
sleep(1);
}
}
/**
* @param OutputInterface $output
* @return void
*/
private function checkChildProcesses(OutputInterface $output): void
{
foreach (array_keys($this->childProcessPIDs) as $pid) {
$result = pcntl_waitpid($pid, $status, WNOHANG);
// If the process has already exited
if ($result === -1 || $result > 0) {
$message = $status === 0
? '%s: PID #<fg=blue>%d</fg=blue> completed in %ds'
: '%s: PID #<fg=blue>%d</fg=blue> <fg=red>failed</fg=red> in %ds with status <fg=red>%s</fg=red>! Check log file.';
$output->writeln(sprintf(
$message,
$this->getDateTime(),
$pid,
microtime(true) - $this->childProcessPIDs[$pid],
$status
));
unset($this->childProcessPIDs[$pid]);
}
}
}
/**
* @return string
*/
private function getDateTime(): string
{
return date('Y-m-d_H:i:s');
}
}