diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index bd7fe19..8df6478 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -43,7 +43,7 @@ jobs: with: php-version: ${{ inputs.php }} extensions: ${{ env.extensions }} - ini-values: opcache.enable_cli=1, opcache.jit=tracing, opcache.jit_buffer_size=64M + ini-values: opcache.enable_cli=1, opcache.jit=off coverage: none - name: Install dependencies diff --git a/CHANGELOG.md b/CHANGELOG.md index d871b69..bd7829b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ All notable changes to **parallel-sdk** are documented in this file. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and the project adheres to [Semantic Versioning](https://semver.org/). +## Unreleased + +### Added +- `ParallelWorker::write()` and `ParallelWorker::writeln()` methods to emit console messages from workers without them being overwritten by the ProgressBar. +- `WriteOutputMessage` command to route `write()`/`writeln()` calls to the `Runner` coordinator. + +### Changed +- ProgressBar and console-message handling is now owned directly by the `Runner` thread on a `stderr` `OutputInterface`, so `clear()`/`write()`/`display()` work correctly together without extra coordinator threads. + ## `3.0.0` – 2025-07-04 ### Added diff --git a/README.md b/README.md index f9ad3a2..8ed634a 100644 --- a/README.md +++ b/README.md @@ -321,6 +321,8 @@ Available methods are: - `setProgress(int $step)` - `display()` - `clear()` +- `write(string $message, bool $newline = false)` +- `writeln(string $message)` ```php use HDSSolutions\Console\Parallel\ParallelWorker; @@ -332,6 +334,7 @@ final class ExampleWorker extends ParallelWorker { $microseconds = random_int(100, 500); $this->setMessage(sprintf("ExampleWorker >> Hello from task #%u, I'll wait %sms", $number, $microseconds)); usleep($microseconds * 1000); + $this->writeln(sprintf("ExampleWorker >> Finished task #%u", $number)); $this->advance(); // end example process @@ -349,6 +352,44 @@ final class ExampleWorker extends ParallelWorker { memory: 562 KiB, threads: 12x ~474 KiB, Σ 5,6 MiB ↑ 5,6 MiB ``` +#### Console messages + +Use `write()` or `writeln()` to emit ad-hoc console messages from a worker. When a ProgressBar is active, the bar is temporarily hidden, the message is printed, and the bar is redrawn below it so the message is not overwritten. + +These methods also work for workers that did not enable `withProgress()`; messages are then written directly to the console. + +```php +use HDSSolutions\Console\Parallel\ParallelWorker; + +final class ExampleWorker extends ParallelWorker { + + protected function process(int $number = 0): int { + $this->writeln(sprintf("Processing task #%u", $number)); + + // ... do work ... + + $this->writeln(sprintf("Finished task #%u", $number)); + + return $number; + } + +} +``` + +#### Example output +When used together with `withProgress()`, messages are printed between progress-bar redraws instead of being overwritten: + +```bash + 0 of 10: Starting... + [>------------------------------------------------------------------------] 0% + elapsed: < 1 sec, remaining: < 1 sec, ?? items/s,memory: ?? + Processing task #5 + Finished task #5 + 1 of 10: Task #5 + [=====>-------------------------------------------------------------------] 10% + elapsed: < 1 sec, remaining: < 1 sec, ~1.00 items/s,memory: ?? +``` + ### References 1. [parallel\bootstrap()](https://www.php.net/manual/en/parallel.bootstrap.php) 2. [parallel\Runtime](https://www.php.net/manual/en/class.parallel-runtime.php) diff --git a/docs/RFC-progressbar-console-messages.md b/docs/RFC-progressbar-console-messages.md new file mode 100644 index 0000000..e2a2824 --- /dev/null +++ b/docs/RFC-progressbar-console-messages.md @@ -0,0 +1,319 @@ +# RFC: Console message output from workers while a ProgressBar is active + +**Status:** Implemented in PR #28 + +> **Note:** The final implementation keeps the `ProgressBar` and `StreamOutput` inside the `Runner` thread instead of spawning a separate `ProgressBarWorker`/`ConsoleWorker`, because routing everything through the existing `Runner` channel proved simpler and avoided the CI deadlocks seen during development. The public API and behavior described below remain unchanged. + +## Problem + +When a worker calls `echo`/`fwrite` while the SDK is rendering a `Symfony\Component\Console\Helper\ProgressBar`, the next ProgressBar refresh overwrites the message. The ProgressBar keeps an internal cursor/line count and re-prints its output on top of whatever was last written to the terminal. + +## Goals + +- Allow workers to emit ad-hoc console messages that are **not** overwritten by the ProgressBar. +- Keep the feature optional: only workers that enabled `withProgress()` should rely on coordinated output. +- Preserve the existing architecture: workers run in isolated `parallel\Runtime`s and cannot share stream resources such as `ConsoleOutput`. + +## Non-goals + +- Provide a general `OutputInterface` injection point for arbitrary streams. +- Capture or redirect all `echo`/`print` statements automatically. + +## Constraints + +- `ext-parallel` cannot share objects that wrap PHP resources. `ConsoleOutput` holds `php://stdout`/`php://stderr` streams, so it cannot live in `Runner` and be passed into a worker thread. +- Messages must therefore be routed to a worker thread that owns the `ConsoleOutput`: the existing `ProgressBarWorker` when a progress bar is active, or a new `ConsoleWorker` spawned by `Runner` when it is not. + +## Proposed public API + +`ParallelWorker` will expose two new methods (implemented in `HDSSolutions\Console\Parallel\Internals\Worker\CommunicatesWithRunner`): + +```php +public function write(string $message, bool $newline = false): void; +public function writeln(string $message): void; +``` + +They are intentionally **not** added to the `Contracts\ParallelWorker` interface to avoid a backwards-compatibility break. The intended usage is to extend `ParallelWorker`, which uses `CommunicatesWithRunner` and therefore inherits the implementation. + +Usage inside a worker: + +```php +final class ExampleWorker extends ParallelWorker { + + protected function process(int $number = 0): int { + $this->setMessage("Processing #{$number}"); + $this->writeln("Starting heavy work for task #{$number}"); + + // ... do work ... + + $this->writeln("Finished task #{$number}"); + $this->advance(); + + return $number; + } + +} +``` + +`write()`/`writeln()` are different from `setMessage()`: + +- `setMessage()` changes a ProgressBar placeholder and is only visible inside the bar. +- `write()`/`writeln()` emit a real console line above the bar. + +## Implementation outline (final) + +### 1. New command message + +`src/Internals/Commands/Output/WriteOutputMessage.php` + +```php +sendOutputMessage(new Commands\Output\WriteOutputMessage($message, $newline)); +} + +final public function writeln(string $message): void { + $this->write($message, true); +} + +private function sendOutputMessage(Commands\Output\WriteOutputMessage $message): void { + if ($this->runner_channel !== null) { + if (PARALLEL_EXT_LOADED) { + $this->runner_channel->send($message); + } else { + ($this->runner_channel)($message); + } + + return; + } + + // fallback when no coordinator is available: write to a fresh stderr stream + $stream = fopen('php://stderr', 'w'); + if ($stream !== false) { + fwrite($stream, $message->args[0].($message->args[1] ? PHP_EOL : '')); + fclose($stream); + } +} +``` + +Notes: + +- Every worker is connected to the `Runner` main channel, so `runner_channel` is always set. +- If the channel cannot be established, the worker writes directly to a fresh `php://stderr` stream. + +### 4. Runner-owned output + +The `Runner` thread owns the Symfony `ProgressBar` and the output stream. `src/Internals/Runner/HasProgressBar.php` creates the `ProgressBar` on a `StreamOutput` tied to `php://stderr`: + +```php +use Symfony\Component\Console\Helper\ProgressBar; +use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\Console\Output\StreamOutput; + +trait HasProgressBar { + + private ProgressBar $progressBar; + private bool $progressBarStarted = false; + private OutputInterface $output; + + private function createProgressBar(): void { + // use a fresh stderr stream owned by this thread + $this->output = new StreamOutput(fopen('php://stderr', 'w')); + $this->progressBar = new ProgressBar($this->output); + + // existing configuration stays unchanged + $this->progressBar->setBarWidth(80); + $this->progressBar->setRedrawFrequency(100); + $this->progressBar->minSecondsBetweenRedraws(0.1); + $this->progressBar->maxSecondsBetweenRedraws(0.2); + $this->progressBar->setFormat( + "%current% of %max%: %message%\n". + "[%bar%] %percent:3s%%\n". + "elapsed: %elapsed:6s%, remaining: %remaining:-6s%, %items_per_second% items/s".(PARALLEL_EXT_LOADED ? "\n" : ","). + "memory: %threads_memory%\n"); + + $this->progressBar->setMessage('Starting...'); + $this->progressBar->setMessage('??', 'items_per_second'); + $this->progressBar->setMessage('??', 'threads_memory'); + } + +} +``` + +`src/Internals/Runner/HasSharedProgressBar.php` processes the `write_output`, `stats_report`, and `progress_bar_action` messages directly in the `Runner` thread: + +```php +private function writeOutput(string $message, bool $newline = true): void { + if ($this->progressBarStarted) { + $this->progressBar->clear(); + $this->output->write($message, $newline); + $this->progressBar->display(); + + return; + } + + $this->output->write($message, $newline); +} +``` + +This performs the exact sequence described in the issue: hide the bar, print the message, then redraw the bar so it recalculates its cursor position. + +### 5. Connect every worker to the Runner + +`src/Internals/Runner/ManagesTasks.php` ensures every worker gets a channel to the `Runner`: + +```php +// init progressbar (it also handles console messages from this worker) +$this->initProgressBar(); + +// connect worker to the Runner's output handler +$worker->connectRunner(fn(Commands\ParallelCommandMessage $message) => $this->processMessage($message)); + +// check if worker has ProgressBar enabled +if ($registered_worker->hasProgressEnabled() && !$this->progressBarStarted) { + // register worker + $this->registerProgressBar($worker_class, $registered_worker->getSteps()); +} +``` + +In threaded mode `connectRunner()` opens the `Runner` main channel; in sequential mode it stores the closure that dispatches messages back into `Runner::processMessage()`. + +### 6. No separate coordinator threads + +The original RFC considered a separate `ProgressBarWorker` thread and a `ConsoleWorker` thread for the fallback. The final implementation keeps the `ProgressBar` and `StreamOutput` inside the `Runner` thread and routes all worker messages through the existing `Runner` channel. This removes a persistent child-thread lifetime issue and avoids sharing stream resources across threads. + +## Behaviour + +### With a progress bar + +When `write()` is called in a worker that has `withProgress()` enabled: + +1. The worker thread sends a `WriteOutputMessage` through the `Runner` main channel. +2. The `Runner` thread receives it, calls `clear()` to erase the bar, writes the message line to `stderr` via the same `OutputInterface`, then calls `display()` to redraw the bar below the message. + +Because all ProgressBar actions are already processed sequentially through the channel, the `clear`/`write`/`display` sequence is atomic with respect to other bar updates. + +### Without a progress bar + +When `write()` is called in a worker that does **not** have `withProgress()` enabled: + +1. The worker thread still sends a `WriteOutputMessage` through the `Runner` main channel. +2. The `Runner` thread receives it and writes the message line directly to the `stderr` `OutputInterface`. + +There is no `clear()`/`display()` because no progress bar is active. + +## Example + +Scheduler code: + +```php +Scheduler::using(LogWorker::class) + ->withProgress(steps: 10); + +foreach (range(1, 10) as $i) { + Scheduler::runTask($i); +} + +Scheduler::awaitTasksCompletion(); +``` + +Worker: + +```php +final class LogWorker extends ParallelWorker { + + protected function process(int $number = 0): int { + $this->setMessage("Task #{$number}"); + $this->writeln("Starting task #{$number}"); + + usleep(100_000); + + $this->writeln("Finished task #{$number}"); + $this->advance(); + + return $number; + } + +} +``` + +Expected terminal flow: + +``` +Starting task #1 + 1 of 10: Task #1 + [=====>---------------------------------------------] 10% + ... +Finished task #1 + 2 of 10: Task #2 + [=========>-----------------------------------------] 20% +``` + +## Backwards compatibility + +- `write()` and `writeln()` are added to the `ParallelWorker` abstract class via the `CommunicatesWithRunner` trait, not to the `Contracts\ParallelWorker` interface. This avoids a BC break for any code that implements the interface directly. +- No existing methods are changed or removed. + +## Alternatives considered + +1. **Pass a `ConsoleOutput` object from `Runner` into workers** + - Not possible with `ext-parallel` because the output wraps stream resources. + - Sending message payloads to a `ConsoleOutput` owned by `Runner` (or a worker it spawns) is valid and is the chosen fallback. + +2. **Use `echo`/`fwrite` in the worker and pause the ProgressBar** + - The bar still overwrites the message on its next refresh because the cursor logic is unaware of the extra line. + +3. **Use `ConsoleOutput->section()`** + - More robust long-term, but requires a larger rewrite of `HasProgressBar` and coordination of multiple `ConsoleSectionOutput` instances. + - Symfony's `ProgressBar` supports section outputs, but the SDK currently uses a plain `ConsoleOutput`. This could be a future enhancement. + +4. **Single `writeMessage(string $message)` instead of `write`/`writeln`** + - Rejected in favor of Symfony's `write()` + `writeln()` naming. + +## Decisions made + +- **Naming:** Use Symfony `OutputInterface` naming: `write()` + `writeln()`. +- **Memory stats:** Do not update memory stats on `write()`. +- **Output stream:** Use a single `stderr` `StreamOutput` for the `ProgressBar` and messages. `ProgressBar` and `write()` output are emitted through the same `OutputInterface` so `clear()`/`write()`/`display()` work as Symfony intended. +- **Coordinator:** The `Runner` thread owns the `ProgressBar` and the `StreamOutput`. All worker messages (including `write_output`) are routed through the existing `Runner` channel; no separate `ProgressBarWorker` or `ConsoleWorker` threads are spawned. +- **Output injection:** Out of scope for this RFC. +- **Fallback:** If a worker cannot connect to the `Runner` channel, it opens a fresh `php://stderr` stream and writes the message directly. + +## Known caveats + +- None currently; messages and the progress bar share `stderr`, so `clear()`/`write()`/`display()` work as Symfony intended. + +## Recommended next steps + +- [x] Implement the approved design. +- [x] Add PHPUnit tests for both the progress-bar path and the non-progress-bar path. +- [x] Update `README.md` to document `write()`/`writeln()`. +- [x] Rename `CommunicatesWithProgressBarWorker` to a name that reflects both progress-bar and console-message responsibilities (e.g. `CommunicatesWithRunner`). diff --git a/src/Internals/Commands/Output/WriteOutputMessage.php b/src/Internals/Commands/Output/WriteOutputMessage.php new file mode 100644 index 0000000..cfbc945 --- /dev/null +++ b/src/Internals/Commands/Output/WriteOutputMessage.php @@ -0,0 +1,21 @@ +openChannels(); - $this->createProgressBar(); - - // threads memory usage and peak - $this->threads_memory = [ - 'current' => [ '__main__' => 0 ], - 'peak' => [ '__main__' => 0 ], - ]; - } - - public function afterListening(): void { - $this->closeChannels(); - } - - private function registerWorker(string $worker, int $steps = 0): void { - // check if ProgressBar isn't already started - if ( !$this->progressBarStarted) { - // start Worker ProgressBar - $this->progressBar->start($steps); - $this->progressBarStarted = true; - - } else { - // update steps - $this->progressBar->setMaxSteps($steps); - } - - $this->release(); - } - - private function progressBarAction(string $action, array $args): void { - // redirect action to ProgressBar instance - $this->progressBar->$action(...$args); - - if ($action === 'advance') { - // count processed item - $this->items[ time() ] = ($this->items[ time() ] ?? 0) + (int) array_shift($args); - // update ProgressBar items per second report - $this->progressBar->setMessage($this->getItemsPerSecond(), 'items_per_second'); - } - } - - private function statsReport(string $worker_id, int $memory_usage): void { - // save memory usage of thread - $this->threads_memory['current'][$worker_id] = $memory_usage; - // update peak memory usage - if ($this->threads_memory['current'][$worker_id] > ($this->threads_memory['peak'][$worker_id] ?? 0)) { - $this->threads_memory['peak'][$worker_id] = $this->threads_memory['current'][$worker_id]; - } - - // update ProgressBar memory report - $this->progressBar->setMessage($this->getMemoryUsage(), 'threads_memory'); - } - - private function getMemoryUsage(): string { - // main memory used - $main = Helper::formatMemory($this->threads_memory['current']['__main__']); - // total memory used (sum of all threads) - $total = Helper::formatMemory($total_raw = array_sum($this->threads_memory['current'])); - // average of each thread - $average = Helper::formatMemory((int) ($total_raw / (($count = count($this->threads_memory['current']) - 1) > 0 ? $count : 1))); - // peak memory usage - $peak = Helper::formatMemory(array_sum($this->threads_memory['peak'])); - - return "$main, threads: {$count}x ~$average, Σ $total ↑ $peak"; - } - - private function getItemsPerSecond(): string { - // check for empty list - if ($this->items === []) return '0'; - - // keep only last 15s for average - $this->items = array_slice($this->items, -15, preserve_keys: true); - - // return the average of items processed per second - return '~'.number_format(floor(array_sum($this->items) / count($this->items) * 100) / 100, 2); - } - -} diff --git a/src/Internals/ProgressBarWorker/HasChannels.php b/src/Internals/ProgressBarWorker/HasChannels.php deleted file mode 100644 index f42c330..0000000 --- a/src/Internals/ProgressBarWorker/HasChannels.php +++ /dev/null @@ -1,42 +0,0 @@ -progressbar_channel = TwoWayChannel::make(self::class.'@'.$this->uuid); - } - - protected function recv(): mixed { - return $this->progressbar_channel->receive(); - } - - protected function send(mixed $value): mixed { - return $this->progressbar_channel->send($value); - } - - protected function release(): void { - if (! PARALLEL_EXT_LOADED) return; - - $this->progressbar_channel->release(); - } - - private function closeChannels(): void { - // gracefully join - $this->progressbar_channel->send(false); - // close channel - $this->progressbar_channel->close(); - } - -} diff --git a/src/Internals/Runner.php b/src/Internals/Runner.php index 0d48d07..87cd905 100644 --- a/src/Internals/Runner.php +++ b/src/Internals/Runner.php @@ -208,12 +208,7 @@ private function enableProgressBar(string $worker_id, int $steps): bool { $worker->withProgress(steps: $steps); $this->initProgressBar(); - - $this->progressbar_channel->send(new Commands\ProgressBar\ProgressBarRegistrationMessage( - worker: $worker->getWorkerClass(), - steps: $steps, - )); - $this->progressbar_channel->receive(); + $this->registerProgressBar($worker->getWorkerClass(), $steps); return $this->send(true); } @@ -226,12 +221,8 @@ private function update(): void { $this->send($this->hasPendingTasks(), eater: true); - if ($this->progressbar_started) { - // - $this->progressbar_channel->send(new Commands\ProgressBar\StatsReportMessage( - worker_id: '__main__', - memory_usage: memory_get_usage(), - )); + if ($this->progressbar_initialized) { + $this->statsReport('__main__', memory_get_usage()); } } diff --git a/src/Internals/ProgressBarWorker/HasProgressBar.php b/src/Internals/Runner/HasProgressBar.php similarity index 59% rename from src/Internals/ProgressBarWorker/HasProgressBar.php rename to src/Internals/Runner/HasProgressBar.php index d35f822..bb6afd4 100644 --- a/src/Internals/ProgressBarWorker/HasProgressBar.php +++ b/src/Internals/Runner/HasProgressBar.php @@ -1,9 +1,10 @@ progressBar = new ProgressBar(new ConsoleOutput); + // Use a fresh stderr stream owned by this thread. StreamOutput is used instead of + // ConsoleOutput because ConsoleOutput would wrap two streams and ProgressBar would + // only write to the error output; a single StreamOutput on php://stderr is simpler + // and keeps the ProgressBar and worker messages on the same stream. + $this->output = new StreamOutput(fopen('php://stderr', 'w')); + $this->progressBar = new ProgressBar($this->output); // configure ProgressBar settings $this->progressBar->setBarWidth(80); diff --git a/src/Internals/Runner/HasSharedProgressBar.php b/src/Internals/Runner/HasSharedProgressBar.php index c698098..08999e9 100644 --- a/src/Internals/Runner/HasSharedProgressBar.php +++ b/src/Internals/Runner/HasSharedProgressBar.php @@ -2,67 +2,120 @@ namespace HDSSolutions\Console\Parallel\Internals\Runner; -use HDSSolutions\Console\Parallel\Internals; -use HDSSolutions\Console\Parallel\Internals\Communication\TwoWayChannel; -use parallel\Channel; -use parallel\Events\Event; -use parallel\Future; -use parallel; +use Symfony\Component\Console\Helper\Helper; trait HasSharedProgressBar { + use HasProgressBar; + /** - * @var Future|Internals\ProgressBarWorker Instance of the ProgressBar worker + * @var array Memory usage between threads */ - private Future|Internals\ProgressBarWorker $progressBar; + private array $threads_memory = [ + 'current' => [ '__main__' => 0 ], + 'peak' => [ '__main__' => 0 ], + ]; /** - * @var bool Flag to identify if ProgressBar is already started + * @var array Total of items processed per second */ - private bool $progressbar_started = false; + private array $items = []; /** - * @var TwoWayChannel|null Channel of communication with the ProgressBar worker + * @var bool Flag to identify if the ProgressBar instance is initialized */ - private ?TwoWayChannel $progressbar_channel = null; + private bool $progressbar_initialized = false; private function initProgressBar(): void { - // init ProgressBar worker, only if not already working - $this->progressBar ??= PARALLEL_EXT_LOADED - // create a ProgressBarWorker instance inside a thread - ? parallel\run(static function(string $uuid): void { - // create ProgressBarWorker instance - $progressBar = new Internals\ProgressBarWorker($uuid); - // listen for events - $progressBar->listen(); - }, [ $this->uuid ]) - - // create a ProgressBar instance for non-threaded environment - : new Internals\ProgressBarWorker($this->uuid); - - // check if progressbar is already started, or we are on a non-threaded environment - if ($this->progressbar_started || ! PARALLEL_EXT_LOADED) return; - - // open communication channel with the ProgressBar worker - while ($this->progressbar_channel === null) { - // open channel to communicate with the ProgressBar worker instance - try { $this->progressbar_channel = TwoWayChannel::open(Internals\ProgressBarWorker::class.'@'.$this->uuid); - // wait 1ms if channel does not exist yet and retry - } catch (Channel\Error\Existence) { usleep(1_000); } - } + if ($this->progressbar_initialized) return; - // wait until ProgressBar worker starts - $this->progressbar_channel->receive(); - $this->progressbar_started = true; + $this->createProgressBar(); + $this->progressbar_initialized = true; } private function stopProgressBar(): void { - if (! PARALLEL_EXT_LOADED || ! $this->progressbar_started) return; + // finish the ProgressBar if it was started so the terminal state is clean + if ($this->progressbar_initialized && $this->progressBarStarted) { + $this->progressBar->finish(); + } + } + + private function registerProgressBar(string $worker, int $steps = 0): bool { + if (!$this->progressBarStarted) { + $this->progressBar->start($steps); + $this->progressBarStarted = true; + + return true; + } + + $this->progressBar->setMaxSteps($steps); + + return true; + } + + private function progressBarAction(string $action, array $args): void { + // ignore progress actions until the bar is actually started + if (!$this->progressBarStarted) return; + + // redirect action to ProgressBar instance + $this->progressBar->$action(...$args); + + if ($action === 'advance') { + // count processed item + $this->items[ time() ] = ($this->items[ time() ] ?? 0) + (int) array_shift($args); + // update ProgressBar items per second report + $this->progressBar->setMessage($this->getItemsPerSecond(), 'items_per_second'); + } + } + + private function writeOutput(string $message, bool $newline = true): void { + if ($this->progressBarStarted) { + $this->progressBar->clear(); + $this->output->write($message, $newline); + $this->progressBar->display(); + + return; + } + + $this->output->write($message, $newline); + } + + private function statsReport(string $worker_id, int $memory_usage): void { + // save memory usage of thread + $this->threads_memory['current'][$worker_id] = $memory_usage; + // update peak memory usage + if ($this->threads_memory['current'][$worker_id] > ($this->threads_memory['peak'][$worker_id] ?? 0)) { + $this->threads_memory['peak'][$worker_id] = $this->threads_memory['current'][$worker_id]; + } + + if (!$this->progressBarStarted) return; + + // update ProgressBar memory report + $this->progressBar->setMessage($this->getMemoryUsage(), 'threads_memory'); + } + + private function getMemoryUsage(): string { + // main memory used + $main = Helper::formatMemory($this->threads_memory['current']['__main__']); + // total memory used (sum of all threads) + $total = Helper::formatMemory($total_raw = array_sum($this->threads_memory['current'])); + // average of each thread + $average = Helper::formatMemory((int) ($total_raw / (($count = count($this->threads_memory['current']) - 1) > 0 ? $count : 1))); + // peak memory usage + $peak = Helper::formatMemory(array_sum($this->threads_memory['peak'])); + + return "$main, threads: {$count}x ~$average, Σ $total ↑ $peak"; + } + + private function getItemsPerSecond(): string { + // check for empty list + if ($this->items === []) return '0'; + + // keep only last 15s for average + $this->items = array_slice($this->items, -15, preserve_keys: true); - // stop ProgressBar worker instance - $this->progressbar_channel->send(Event\Type::Close); - // wait until ProgressBar instance shutdowns - $this->progressbar_channel->receive(); + // return the average of items processed per second + return '~'.number_format(floor(array_sum($this->items) / count($this->items) * 100) / 100, 2); } } diff --git a/src/Internals/Runner/ManagesTasks.php b/src/Internals/Runner/ManagesTasks.php index 86c0cfe..a65b18d 100644 --- a/src/Internals/Runner/ManagesTasks.php +++ b/src/Internals/Runner/ManagesTasks.php @@ -79,6 +79,9 @@ private function startNextPendingTask(): void { // create starter channel to wait threads start event $this->starter ??= Channel::make(sprintf('starter@%s', $this->uuid)); + // ensure the Runner's output handler is available (handles both progress and console messages) + $this->initProgressBar(); + // parallel available, process task inside a thread $this->running_tasks[$task_id] = parallel\run(static function(string $uuid, int $task_id, RegisteredWorker $registered_worker, Task $task): array { // get Worker class to instantiate @@ -93,11 +96,12 @@ private function startNextPendingTask(): void { // process task using user Worker : [ ...$task->getInput() ]; - // check if worker has ProgressBar enabled - if ($registered_worker->hasProgressEnabled()) { - // connect worker to ProgressBar - $worker->connectProgressBar($uuid, $GLOBALS['worker_thread_id'] ??= sprintf('%s@%s', $uuid, substr(md5(uniqid($worker_class, true)), 0, 16))); - } + // connect worker to Runner (handles both progress and console messages) + $worker->connectRunner( + $uuid, + $GLOBALS['worker_thread_id'] ??= sprintf('%s@%s', $uuid, substr(md5(uniqid($worker_class, true)), 0, 16)), + $registered_worker->hasProgressEnabled(), + ); // notify that thread started Channel::open(sprintf('starter@%s', $uuid))->send(true); @@ -141,17 +145,18 @@ private function startNextPendingTask(): void { // process task using user Worker : [ ...$task->getInput() ]; + // init Runner's output handler (handles both progress and console messages) + $this->initProgressBar(); + // connect worker to the Runner's output handler + $worker->connectRunner( + fn(Commands\ParallelCommandMessage $message) => $this->processMessage($message), + progress_enabled: $registered_worker->hasProgressEnabled(), + ); + // check if worker has ProgressBar enabled - if ($registered_worker->hasProgressEnabled()) { - // init progressbar - $this->initProgressBar(); + if ($registered_worker->hasProgressEnabled() && !$this->progressBarStarted) { // register worker - $this->progressBar->processMessage(new Commands\ProgressBar\ProgressBarRegistrationMessage( - worker: $worker_class, - steps: $registered_worker->getSteps(), - )); - // connect worker to ProgressBar - $worker->connectProgressBar(fn(Commands\ProgressBar\ProgressBarActionMessage $message) => $this->progressBar->processMessage($message)); + $this->registerProgressBar($worker_class, $registered_worker->getSteps()); } $task->setState(Task::STATE_Processing); diff --git a/src/Internals/Worker/CommunicatesWithProgressBarWorker.php b/src/Internals/Worker/CommunicatesWithProgressBarWorker.php deleted file mode 100644 index e16919c..0000000 --- a/src/Internals/Worker/CommunicatesWithProgressBarWorker.php +++ /dev/null @@ -1,85 +0,0 @@ -progressbar_channel = $uuid; - - return true; - } - - // store worker identifier - $this->identifier = $identifier; - - // open channel if not already opened - while ($this->progressbar_channel === null) { - // open channel to communicate with the Runner instance - try { $this->progressbar_channel = TwoWayChannel::open(ProgressBarWorker::class.'@'.$uuid); - // wait 1ms if channel does not exist yet and retry - } catch (Channel\Error\Existence) { usleep(1_000); } - } - - return true; - } - - final public function setMessage(string $message, string $name = 'message'): void { - $this->newProgressBarAction(__FUNCTION__, $message, $name); - } - - final public function advance(int $steps = 1): void { - $this->newProgressBarAction(__FUNCTION__, $steps); - } - - final public function setProgress(int $step): void { - $this->newProgressBarAction(__FUNCTION__, $step); - } - - final public function display(): void { - $this->newProgressBarAction(__FUNCTION__); - } - - final public function clear(): void { - $this->newProgressBarAction(__FUNCTION__); - } - - private function newProgressBarAction(string $action, ...$args): void { - // check if progressbar is active - if ($this->progressbar_channel === null) return; - - $message = new Commands\ProgressBar\ProgressBarActionMessage( - action: $action, - args: $args, - ); - - // check if parallel is available - if (PARALLEL_EXT_LOADED) { - // report memory usage - $this->progressbar_channel->send(new Commands\ProgressBar\StatsReportMessage( - worker_id: $this->identifier, - memory_usage: memory_get_usage(), - )); - // request ProgressBar action - $this->progressbar_channel->send($message); - - return; - } - - // redirect action to ProgressBar executor - ($this->progressbar_channel)($message); - } - -} diff --git a/src/Internals/Worker/CommunicatesWithRunner.php b/src/Internals/Worker/CommunicatesWithRunner.php index 283dd68..a27ca1f 100644 --- a/src/Internals/Worker/CommunicatesWithRunner.php +++ b/src/Internals/Worker/CommunicatesWithRunner.php @@ -2,22 +2,68 @@ namespace HDSSolutions\Console\Parallel\Internals\Worker; -use HDSSolutions\Console\Parallel\Internals\Communication\TwoWayChannel; +use Closure; +use HDSSolutions\Console\Parallel\Internals\Commands; use HDSSolutions\Console\Parallel\Internals\Runner; +use HDSSolutions\Console\Parallel\Internals\Communication\TwoWayChannel; use parallel\Channel; trait CommunicatesWithRunner { /** - * @var TwoWayChannel|null Communication channel with the Runner + * @var TwoWayChannel|Closure|null Channel of communication between the worker and the Runner + */ + private TwoWayChannel | Closure | null $runner_channel = null; + + /** + * @var bool Whether the worker has an active ProgressBar + */ + private bool $progress_enabled = false; + + /** + * @var string|null UUID used to open the Runner channel */ - private ?TwoWayChannel $runner_channel = null; + private ?string $runner_uuid = null; + + final public function connectRunner(string | Closure $uuid, string $identifier = null, bool $progress_enabled = false): bool { + $this->progress_enabled = $progress_enabled; + if (is_string($uuid)) { + $this->runner_uuid = $uuid; + } + + if (! PARALLEL_EXT_LOADED) { + $this->runner_channel = $uuid; + + return true; + } + + // store worker identifier + $this->identifier = $identifier; - protected function getRunnerChannel(): TwoWayChannel { // open channel if not already opened while ($this->runner_channel === null) { // open channel to communicate with the Runner instance - try { $this->runner_channel = TwoWayChannel::open(Runner::class.'@'.$this->uuid); + try { $this->runner_channel = TwoWayChannel::open(Runner::class.'@'.$uuid); + // wait 1ms if channel does not exist yet and retry + } catch (Channel\Error\Existence) { usleep(1_000); } + } + + return true; + } + + final protected function getRunnerChannel(): TwoWayChannel { + if ($this->runner_channel instanceof TwoWayChannel) { + return $this->runner_channel; + } + + $uuid = property_exists($this, 'uuid') ? $this->uuid : ($this->runner_uuid ?? null); + if ($uuid === null) { + throw new \RuntimeException('Cannot determine Runner UUID'); + } + + while ($this->runner_channel === null) { + // open channel to communicate with the Runner instance + try { $this->runner_channel = TwoWayChannel::open(Runner::class.'@'.$uuid); // wait 1ms if channel does not exist yet and retry } catch (Channel\Error\Existence) { usleep(1_000); } } @@ -25,4 +71,75 @@ protected function getRunnerChannel(): TwoWayChannel { return $this->runner_channel; } + final public function setMessage(string $message, string $name = 'message'): void { + $this->newProgressBarAction(__FUNCTION__, $message, $name); + } + + final public function advance(int $steps = 1): void { + $this->newProgressBarAction(__FUNCTION__, $steps); + } + + final public function setProgress(int $step): void { + $this->newProgressBarAction(__FUNCTION__, $step); + } + + final public function display(): void { + $this->newProgressBarAction(__FUNCTION__); + } + + final public function clear(): void { + $this->newProgressBarAction(__FUNCTION__); + } + + final public function write(string $message, bool $newline = false): void { + $this->sendOutputMessage(new Commands\Output\WriteOutputMessage($message, $newline)); + } + + final public function writeln(string $message): void { + $this->write($message, true); + } + + private function sendOutputMessage(Commands\Output\WriteOutputMessage $message): void { + if ($this->runner_channel !== null) { + if (PARALLEL_EXT_LOADED) { + $this->runner_channel->send($message); + } else { + ($this->runner_channel)($message); + } + + return; + } + + // fallback when no coordinator is available: write to a fresh stderr stream + $stream = fopen('php://stderr', 'w'); + if ($stream !== false) { + fwrite($stream, $message->args[0].($message->args[1] ? PHP_EOL : '')); + fclose($stream); + } + } + + private function newProgressBarAction(string $action, ...$args): void { + // check if progressbar is active + if (!$this->progress_enabled || $this->runner_channel === null) return; + + $message = new Commands\ProgressBar\ProgressBarActionMessage( + action: $action, + args: $args, + ); + + // check if parallel is available + if (PARALLEL_EXT_LOADED) { + // report memory usage + $this->runner_channel->send(new Commands\ProgressBar\StatsReportMessage( + worker_id: $this->identifier, + memory_usage: memory_get_usage(), + )); + $this->runner_channel->send($message); + + return; + } + + ($this->runner_channel)($message); + } + } diff --git a/src/ParallelWorker.php b/src/ParallelWorker.php index d763d7f..1e6fbec 100644 --- a/src/ParallelWorker.php +++ b/src/ParallelWorker.php @@ -6,7 +6,7 @@ use Throwable; abstract class ParallelWorker implements Contracts\ParallelWorker { - use Internals\Worker\CommunicatesWithProgressBarWorker; + use Internals\Worker\CommunicatesWithRunner; /** * @var int Current Worker state diff --git a/tests/ParallelTest.php b/tests/ParallelTest.php index 0da9674..03a5b57 100644 --- a/tests/ParallelTest.php +++ b/tests/ParallelTest.php @@ -5,6 +5,7 @@ use HDSSolutions\Console\Parallel\Internals\Worker; use HDSSolutions\Console\Parallel\RegisteredWorker; use HDSSolutions\Console\Parallel\Scheduler; +use HDSSolutions\Console\Tests\Workers\Writer; use PHPUnit\Framework\TestCase; use RuntimeException; use Throwable; @@ -226,4 +227,79 @@ public function testThatCpuUsageCanBeControlled(): void { $this->assertGreaterThanOrEqual(1, time() - $start); } + public function testThatWorkerCanWriteMessagesWithoutProgressBar(): void { + $output = $this->runWorkerScript(<<<'PHP' +Scheduler::using(Writer::class); +foreach (range(1, 3) as $i) { + Scheduler::runTask($i); +} +Scheduler::awaitTasksCompletion(); +PHP); + + $this->assertStringContainsString('Starting #1', $output); + $this->assertStringContainsString('Starting #2', $output); + $this->assertStringContainsString('Starting #3', $output); + } + + public function testThatWorkerCanWriteMessagesWithProgressBar(): void { + $output = $this->runWorkerScript(<<<'PHP' +Scheduler::using(Writer::class)->withProgress(steps: 3); +foreach (range(1, 3) as $i) { + Scheduler::runTask($i); +} +Scheduler::awaitTasksCompletion(); +PHP); + + $this->assertStringContainsString('Starting #1', $output); + $this->assertStringContainsString('Done #3', $output); + $this->assertStringContainsString('3 of 3:', $output); + $this->assertStringContainsString('Task #', $output); + + $start1 = strpos($output, 'Starting #1'); + $done1 = strpos($output, 'Done #1'); + $done3 = strpos($output, 'Done #3'); + $final = strpos($output, '3 of 3:'); + + $this->assertNotFalse($start1); + $this->assertNotFalse($done1); + $this->assertNotFalse($done3); + $this->assertNotFalse($final); + + $this->assertGreaterThan($start1, $done1, 'Done #1 should come after Starting #1'); + $this->assertGreaterThan($done3, $final, 'Final progress bar should come after Done #3'); + $this->assertGreaterThan($start1, $final, 'Final progress bar should come after Starting #1'); + } + + private function runWorkerScript(string $body): string { + $autoload = __DIR__.'/../vendor/autoload.php'; + + $script = <<<'PHP' +&1', escapeshellarg(PHP_BINARY), escapeshellarg($file)), $output, $exit); + + unlink($file); + + $combined = implode("\n", $output); + $this->assertSame(0, $exit, $combined ?: 'Worker script exited with an error'); + + return $combined; + } + } diff --git a/tests/Workers/Writer.php b/tests/Workers/Writer.php new file mode 100644 index 0000000..8bf903e --- /dev/null +++ b/tests/Workers/Writer.php @@ -0,0 +1,18 @@ +setMessage(sprintf('Task #%d', $n)); + $this->writeln(sprintf('Starting #%d', $n)); + $this->writeln(sprintf('Done #%d', $n)); + $this->advance(); + + return $n; + } + +}