Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/Analyser/Analyser.php
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
use function array_fill_keys;
use function array_merge;
use function count;
use function memory_get_peak_usage;

/**
* @phpstan-import-type CollectorData from CollectedData
Expand Down Expand Up @@ -147,7 +146,8 @@ public function analyse(
packageDependencies: $internalErrorsCount === 0 ? $packageDependencies : null,
exportedNodes: $exportedNodes,
reachedInternalErrorsCountLimit: $reachedInternalErrorsCountLimit,
peakMemoryUsageBytes: memory_get_peak_usage(true),
// analysis ran in this process - its peak is read where the number is printed
peakMemoryUsageBytes: 0,
processedFiles: $allProcessedFiles,
);
}
Expand Down
10 changes: 10 additions & 0 deletions src/Analyser/AnalyserResult.php
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,20 @@ public function __construct(
private bool $reachedInternalErrorsCountLimit,
private int $peakMemoryUsageBytes,
private array $processedFiles,
private int $workerCount = 0,
)
{
}

/**
* How many parallel workers produced this result; 0 when the analysis ran in
* the main process.
*/
public function getWorkerCount(): int
{
return $this->workerCount;
}

/**
* @return list<Error>
*/
Expand Down
3 changes: 3 additions & 0 deletions src/Analyser/AnalyserResultFinalizer.php
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ public function finalize(AnalyserResult $analyserResult, bool $onlyFiles, bool $
reachedInternalErrorsCountLimit: $analyserResult->hasReachedInternalErrorsCountLimit(),
peakMemoryUsageBytes: $analyserResult->getPeakMemoryUsageBytes(),
processedFiles: $analyserResult->getProcessedFiles(),
workerCount: $analyserResult->getWorkerCount(),
), $collectorErrors, $locallyIgnoredCollectorErrors);
}

Expand All @@ -177,6 +178,7 @@ private function mergeFilteredPhpErrors(AnalyserResult $analyserResult): Analyse
reachedInternalErrorsCountLimit: $analyserResult->hasReachedInternalErrorsCountLimit(),
peakMemoryUsageBytes: $analyserResult->getPeakMemoryUsageBytes(),
processedFiles: $analyserResult->getProcessedFiles(),
workerCount: $analyserResult->getWorkerCount(),
);
}

Expand Down Expand Up @@ -243,6 +245,7 @@ private function addUnmatchedIgnoredErrors(
reachedInternalErrorsCountLimit: $analyserResult->hasReachedInternalErrorsCountLimit(),
peakMemoryUsageBytes: $analyserResult->getPeakMemoryUsageBytes(),
processedFiles: $analyserResult->getProcessedFiles(),
workerCount: $analyserResult->getWorkerCount(),
),
$collectorErrors,
$locallyIgnoredCollectorErrors,
Expand Down
8 changes: 6 additions & 2 deletions src/Command/AnalyseApplication.php
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,8 @@ public function analyse(
$internalErrors = [];
$collectedData = [];
$savedResultCache = false;
$memoryUsageBytes = memory_get_peak_usage(true);
$memoryUsageBytes = 0;
$workerCount = 0;
$processedFiles = [];
if ($errorOutput->isVeryVerbose()) {
$errorOutput->writeLineFormatted('Result cache was not saved because of ignoredErrorHelperResult errors.');
Expand Down Expand Up @@ -127,6 +128,7 @@ public function analyse(
reachedInternalErrorsCountLimit: $intermediateAnalyserResult->hasReachedInternalErrorsCountLimit(),
peakMemoryUsageBytes: $intermediateAnalyserResult->getPeakMemoryUsageBytes(),
processedFiles: $intermediateAnalyserResult->getProcessedFiles(),
workerCount: $intermediateAnalyserResult->getWorkerCount(),
);
}

Expand All @@ -145,6 +147,7 @@ public function analyse(
);
$hasInternalErrors = count($internalErrors) > 0 || $analyserResult->hasReachedInternalErrorsCountLimit();
$memoryUsageBytes = $analyserResult->getPeakMemoryUsageBytes();
$workerCount = $analyserResult->getWorkerCount();
$isResultCacheUsed = !$resultCache->isFullAnalysis();

$changedProjectExtensionFilesOutsideOfAnalysedPaths = [];
Expand Down Expand Up @@ -194,6 +197,7 @@ public function analyse(
$changedProjectExtensionFilesOutsideOfAnalysedPaths,
$processedFiles,
$resultCacheExisted,
$workerCount,
);
}

Expand Down Expand Up @@ -255,7 +259,7 @@ private function runAnalyser(
packageDependencies: [],
exportedNodes: [],
reachedInternalErrorsCountLimit: false,
peakMemoryUsageBytes: memory_get_peak_usage(true),
peakMemoryUsageBytes: 0,
processedFiles: [],
);
}
Expand Down
14 changes: 8 additions & 6 deletions src/Command/AnalyseCommand.php
Original file line number Diff line number Diff line change
Expand Up @@ -496,7 +496,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
count($internalErrors) === 1 ? 'An internal error' : 'Internal errors',
));

return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime);
return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, $analysisResult->getWorkerCount());
}

return $this->generateBaseline($generateBaselineFile, $inceptionResult, $analysisResult, $output, $allowEmptyBaseline, $baselineExtension, $failWithoutResultCache);
Expand Down Expand Up @@ -535,6 +535,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
$exitCode,
$analysisResult->getPeakMemoryUsageBytes(),
$this->analysisStartTime,
$analysisResult->getWorkerCount(),
);
}

Expand Down Expand Up @@ -672,7 +673,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int

$errorOutput->writeLineFormatted('');

return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime);
return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, $analysisResult->getWorkerCount());
}
}

Expand All @@ -684,6 +685,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
$exitCode,
$analysisResult->getPeakMemoryUsageBytes(),
$this->analysisStartTime,
$analysisResult->getWorkerCount(),
);
}

Expand Down Expand Up @@ -798,7 +800,7 @@ private function generateBaseline(string $generateBaselineFile, InceptionResult
$inceptionResult->getStdOutput()->getStyle()->error('No errors were found during the analysis. Baseline could not be generated.');
$inceptionResult->getStdOutput()->writeLineFormatted('To allow generating empty baselines, pass <fg=cyan>--allow-empty-baseline</> option.');

return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime);
return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, $analysisResult->getWorkerCount());
}

$streamOutput = $this->createStreamOutput();
Expand Down Expand Up @@ -826,15 +828,15 @@ private function generateBaseline(string $generateBaselineFile, InceptionResult
} catch (DirectoryCreatorException $e) {
$inceptionResult->getStdOutput()->writeLineFormatted($e->getMessage());

return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime);
return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, $analysisResult->getWorkerCount());
}

try {
FileWriter::write($generateBaselineFile, $baselineContents);
} catch (CouldNotWriteFileException $e) {
$inceptionResult->getStdOutput()->writeLineFormatted($e->getMessage());

return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime);
return $inceptionResult->handleReturn(1, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, $analysisResult->getWorkerCount());
}

$errorsCount = 0;
Expand Down Expand Up @@ -874,7 +876,7 @@ private function generateBaseline(string $generateBaselineFile, InceptionResult
$exitCode = 2;
}

return $inceptionResult->handleReturn($exitCode, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime);
return $inceptionResult->handleReturn($exitCode, $analysisResult->getPeakMemoryUsageBytes(), $this->analysisStartTime, $analysisResult->getWorkerCount());
}

/**
Expand Down
3 changes: 1 addition & 2 deletions src/Command/AnalyserRunner.php
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
use function filesize;
use function function_exists;
use function is_file;
use function memory_get_peak_usage;

#[AutowiredService]
final class AnalyserRunner
Expand Down Expand Up @@ -72,7 +71,7 @@ public function runAnalyser(
packageDependencies: [],
exportedNodes: [],
reachedInternalErrorsCountLimit: false,
peakMemoryUsageBytes: memory_get_peak_usage(true),
peakMemoryUsageBytes: 0,
processedFiles: [],
);
}
Expand Down
10 changes: 10 additions & 0 deletions src/Command/AnalysisResult.php
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ public function __construct(
private array $changedProjectExtensionFilesOutsideOfAnalysedPaths,
private array $processedFiles = [],
private bool $resultCacheExisted = true,
private int $workerCount = 0,
)
{
usort(
Expand Down Expand Up @@ -138,6 +139,15 @@ public function getPeakMemoryUsageBytes(): int
return $this->peakMemoryUsageBytes;
}

/**
* How many parallel workers produced this result; 0 when the analysis ran in
* the main process.
*/
public function getWorkerCount(): int
{
return $this->workerCount;
}

public function isResultCacheUsed(): bool
{
return $this->isResultCacheUsed;
Expand Down
33 changes: 27 additions & 6 deletions src/Command/InceptionResult.php
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
use PHPStan\DependencyInjection\Container;
use PHPStan\File\PathNotFoundException;
use PHPStan\Internal\BytesHelper;
use PHPStan\Parallel\ForkParallelChecker;
use function floor;
use function implode;
use function max;
use function memory_get_peak_usage;
use function microtime;
use function round;
Expand Down Expand Up @@ -100,7 +100,11 @@ public function getEditorModeInsteadOfFile(): ?string
return $this->editorModeInsteadOfFile;
}

public function handleReturn(int $exitCode, ?int $peakMemoryUsageBytes, float $analysisStartTime): int
/**
* @param int|null $peakMemoryUsageBytes the heaviest parallel worker's peak, 0 when the
* analysis ran in this process
*/
public function handleReturn(int $exitCode, ?int $peakMemoryUsageBytes, float $analysisStartTime, int $workerCount = 0): int
{
if ($this->getErrorOutput()->isVerbose()) {
$elapsedTime = round(microtime(true) - $analysisStartTime, 2);
Expand All @@ -116,10 +120,27 @@ public function handleReturn(int $exitCode, ?int $peakMemoryUsageBytes, float $a
}

if ($peakMemoryUsageBytes !== null && $this->getErrorOutput()->isVerbose()) {
$this->getErrorOutput()->writeLineFormatted(sprintf(
'Used memory: %s',
BytesHelper::bytes(max(memory_get_peak_usage(true), $peakMemoryUsageBytes)),
));
// This process's peak is read here, at the very end, so it covers collecting
// the workers' results and saving the result cache - the part of a parallel
// run where the main process is at its largest. Both numbers are per process,
// which is also how memory_limit applies.
$mainProcessPeak = memory_get_peak_usage(true);
if ($peakMemoryUsageBytes === 0 || $workerCount === 0) {
$this->getErrorOutput()->writeLineFormatted(sprintf(
'Peak memory: %s',
BytesHelper::bytes($mainProcessPeak),
));
} else {
$mechanism = $this->container->getByType(ForkParallelChecker::class)->isSupported() ? 'forked' : 'spawned';
$this->getErrorOutput()->writeLineFormatted(sprintf(
'Peak memory: %s (main process), %s (%s)',
BytesHelper::bytes($mainProcessPeak),
BytesHelper::bytes($peakMemoryUsageBytes),
$workerCount === 1
? sprintf('the %s worker', $mechanism)
: sprintf('largest of %d %s workers', $workerCount, $mechanism),
));
}
}

return $exitCode;
Expand Down
12 changes: 12 additions & 0 deletions src/Parallel/ForkedProcess.php
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
use Symfony\Component\Console\Output\StreamOutput;
use Throwable;
use function fclose;
use function function_exists;
use function pcntl_fork;
use function pcntl_waitpid;
use function pcntl_wexitstatus;
Expand Down Expand Up @@ -97,6 +98,17 @@ public function start(callable $onData, callable $onError, callable $onExit): vo
// Child: drop the inherited listening socket immediately, then run
// the worker on its own fresh event loop and never return.
$this->server->close();
// memory_get_peak_usage() carries over into the child, so without this a
// worker would report the main process's peak instead of its own - on an
// incremental run, the spike taken while loading the result cache, which
// every worker would then repeat. Restarting the peak here keeps the
// reported number the worker's own high-water usage, inherited memory it
// still holds included.
/** phpcs:disable SlevomatCodingStandard.Namespaces.ReferenceUsedNamesOnly.ReferenceViaFullyQualifiedName */
if (function_exists('memory_reset_peak_usage')) {
\memory_reset_peak_usage();
}
/** phpcs:enable */
ForkedChildCrashReporter::install($tmpStdErr);
$output = new StreamOutput($tmpStdOut);
try {
Expand Down
17 changes: 10 additions & 7 deletions src/Parallel/ParallelAnalyser.php
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,12 @@
use function array_map;
use function array_pop;
use function array_reverse;
use function array_sum;
use function count;
use function defined;
use function escapeshellarg;
use function getenv;
use function ini_get;
use function max;
use function memory_get_usage;
use function parse_url;
use function sprintf;
use function str_contains;
Expand Down Expand Up @@ -110,6 +108,7 @@ public function analyse(
$locallyIgnoredErrors = [];
$linesToIgnore = [];
$unmatchedLineIgnores = [];
/** @var array<string, int> $peakMemoryUsages */
$peakMemoryUsages = [];
$internalErrors = [];
$internalErrorsCount = 0;
Expand Down Expand Up @@ -156,8 +155,12 @@ public function analyse(
packageDependencies: $internalErrorsCount === 0 ? $packageDependencies : null,
exportedNodes: $exportedNodes,
reachedInternalErrorsCountLimit: $reachedInternalErrorsCountLimit,
peakMemoryUsageBytes: array_sum($peakMemoryUsages), // not 100% correct as the peak usages of workers might not have met
// The heaviest single worker. Summing the workers' peaks would describe a
// moment that never happens - they do not peak at the same time - while
// each worker's own peak is what its memory_limit is measured against.
peakMemoryUsageBytes: $peakMemoryUsages === [] ? 0 : max($peakMemoryUsages),
processedFiles: $allProcessedFiles,
workerCount: count($peakMemoryUsages),
));
});
$server->on('connection', function (ConnectionInterface $connection) use (&$jobs, $arenaName, $expectedWorkerCount, &$helloCount): void {
Expand Down Expand Up @@ -372,10 +375,10 @@ public function analyse(

$job = array_pop($jobs);
$process->request(['action' => 'analyse', 'files' => $job]);
}, $handleError, function ($exitCode, string $output) use (&$someChildEnded, &$peakMemoryUsages, &$internalErrors, &$internalErrorsCount, $processIdentifier): void {
if ($someChildEnded === false) {
$peakMemoryUsages['main'] = memory_get_usage(true);
}
}, $handleError, function ($exitCode, string $output) use (&$someChildEnded, &$internalErrors, &$internalErrorsCount, $processIdentifier): void {
// The main process is not sampled here any more: its own peak comes
// later (collecting the workers' results, saving the result cache) and
// is read where the number is printed. Only worker peaks are summed.
$someChildEnded = true;

if ($exitCode === 0) {
Expand Down
Loading