From fb5d3c6a9b160d2acdd7d9d851c5264dbfa3de63 Mon Sep 17 00:00:00 2001 From: Dmitriy Derepko Date: Mon, 22 Jan 2024 14:36:45 +0700 Subject: [PATCH 01/19] Add envelope stack + MessageSerializer (#188) --- config/di.php | 3 + src/Message/EnvelopeInterface.php | 4 + src/Message/EnvelopeTrait.php | 21 ++- src/Message/IdEnvelope.php | 6 +- src/Message/JsonMessageSerializer.php | 59 +++++++ src/Message/Message.php | 8 + src/Message/MessageSerializerInterface.php | 12 ++ .../FailureHandling/FailureEnvelope.php | 2 +- src/QueueInterface.php | 4 - src/Worker/Worker.php | 2 +- tests/Unit/EnvelopeTest.php | 47 +++++ .../Message/JsonMessageSerializerTest.php | 160 ++++++++++++++++++ tests/Unit/WorkerTest.php | 6 +- 13 files changed, 320 insertions(+), 14 deletions(-) create mode 100644 src/Message/JsonMessageSerializer.php create mode 100644 src/Message/MessageSerializerInterface.php create mode 100644 tests/Unit/EnvelopeTest.php create mode 100644 tests/Unit/Message/JsonMessageSerializerTest.php diff --git a/config/di.php b/config/di.php index ea0f54b3..dcb37cbf 100644 --- a/config/di.php +++ b/config/di.php @@ -8,6 +8,8 @@ use Yiisoft\Queue\Cli\SimpleLoop; use Yiisoft\Queue\Command\ListenAllCommand; use Yiisoft\Queue\Command\RunCommand; +use Yiisoft\Queue\Message\JsonMessageSerializer; +use Yiisoft\Queue\Message\MessageSerializerInterface; use Yiisoft\Queue\Middleware\Consume\ConsumeMiddlewareDispatcher; use Yiisoft\Queue\Middleware\Consume\MiddlewareFactoryConsume; use Yiisoft\Queue\Middleware\Consume\MiddlewareFactoryConsumeInterface; @@ -54,6 +56,7 @@ FailureMiddlewareDispatcher::class => [ '__construct()' => ['middlewareDefinitions' => $params['yiisoft/queue']['middlewares-fail']], ], + MessageSerializerInterface::class => JsonMessageSerializer::class, RunCommand::class => [ '__construct()' => [ 'channels' => array_keys($params['yiisoft/yii-queue']['channel-definitions']), diff --git a/src/Message/EnvelopeInterface.php b/src/Message/EnvelopeInterface.php index bf336112..b0f8d89f 100644 --- a/src/Message/EnvelopeInterface.php +++ b/src/Message/EnvelopeInterface.php @@ -9,6 +9,10 @@ */ interface EnvelopeInterface extends MessageInterface { + public const ENVELOPE_STACK_KEY = 'envelopes'; + + public static function fromMessage(MessageInterface $message): self; + public function getMessage(): MessageInterface; public function withMessage(MessageInterface $message): self; diff --git a/src/Message/EnvelopeTrait.php b/src/Message/EnvelopeTrait.php index 13e593ab..7e58a97b 100644 --- a/src/Message/EnvelopeTrait.php +++ b/src/Message/EnvelopeTrait.php @@ -31,8 +31,27 @@ public function getData(): mixed return $this->message->getData(); } + public static function fromMessage(MessageInterface $message): self + { + return new static($message); + } + public function getMetadata(): array { - return $this->message->getMetadata(); + return array_merge( + $this->message->getMetadata(), + [ + self::ENVELOPE_STACK_KEY => array_merge( + $this->message->getMetadata()[self::ENVELOPE_STACK_KEY] ?? [], + [self::class], + ), + ], + $this->getEnvelopeMetadata(), + ); + } + + public function getEnvelopeMetadata(): array + { + return []; } } diff --git a/src/Message/IdEnvelope.php b/src/Message/IdEnvelope.php index a1ffccad..dec2d679 100644 --- a/src/Message/IdEnvelope.php +++ b/src/Message/IdEnvelope.php @@ -29,10 +29,8 @@ public function getId(): string|int|null return $this->id ?? $this->message->getMetadata()[self::MESSAGE_ID_KEY] ?? null; } - public function getMetadata(): array + private function getEnvelopeMetadata(): array { - return array_merge($this->message->getMetadata(), [ - self::MESSAGE_ID_KEY => $this->getId(), - ]); + return [self::MESSAGE_ID_KEY => $this->getId()]; } } diff --git a/src/Message/JsonMessageSerializer.php b/src/Message/JsonMessageSerializer.php new file mode 100644 index 00000000..81a6220c --- /dev/null +++ b/src/Message/JsonMessageSerializer.php @@ -0,0 +1,59 @@ + $message->getHandlerName(), + 'data' => $message->getData(), + 'meta' => $message->getMetadata(), + ]; + + return json_encode($payload, JSON_THROW_ON_ERROR); + } + + /** + * @throws JsonException + * @throws InvalidArgumentException + */ + public function unserialize(string $value): MessageInterface + { + $payload = json_decode($value, true, 512, JSON_THROW_ON_ERROR); + if (!is_array($payload)) { + throw new InvalidArgumentException('Payload must be array. Got ' . get_debug_type($payload) . '.'); + } + + $meta = $payload['meta'] ?? []; + if (!is_array($meta)) { + throw new InvalidArgumentException('Metadata must be array. Got ' . get_debug_type($meta) . '.'); + } + + // TODO: will be removed later + $message = new Message($payload['name'] ?? '$name', $payload['data'] ?? null, $meta); + + if (isset($meta[EnvelopeInterface::ENVELOPE_STACK_KEY]) && is_array($meta[EnvelopeInterface::ENVELOPE_STACK_KEY])) { + $message = $message->withMetadata( + array_merge($message->getMetadata(), [EnvelopeInterface::ENVELOPE_STACK_KEY => []]), + ); + foreach ($meta[EnvelopeInterface::ENVELOPE_STACK_KEY] as $envelope) { + if (is_string($envelope) && class_exists($envelope) && is_subclass_of($envelope, EnvelopeInterface::class)) { + $message = $envelope::fromMessage($message); + } + } + } + + + return $message; + } +} diff --git a/src/Message/Message.php b/src/Message/Message.php index 07278d1f..a414ffb0 100644 --- a/src/Message/Message.php +++ b/src/Message/Message.php @@ -32,4 +32,12 @@ public function getMetadata(): array { return $this->metadata; } + + public function withMetadata(array $metadata): self + { + $instance = clone $this; + $instance->metadata = $metadata; + + return $instance; + } } diff --git a/src/Message/MessageSerializerInterface.php b/src/Message/MessageSerializerInterface.php new file mode 100644 index 00000000..b034590c --- /dev/null +++ b/src/Message/MessageSerializerInterface.php @@ -0,0 +1,12 @@ +getHandlerName(); $handler = $this->getHandler($name); if ($handler === null) { - throw new RuntimeException("Queue handler with name $name doesn't exist"); + throw new RuntimeException(sprintf('Queue handler with name "%s" does not exist', $name)); } $request = new ConsumeRequest($message, $queue); diff --git a/tests/Unit/EnvelopeTest.php b/tests/Unit/EnvelopeTest.php new file mode 100644 index 00000000..af412125 --- /dev/null +++ b/tests/Unit/EnvelopeTest.php @@ -0,0 +1,47 @@ +assertEquals('test', $message->getMessage()->getData()); + + $stack = $message->getMetadata()[EnvelopeInterface::ENVELOPE_STACK_KEY]; + $this->assertIsArray($stack); + + $this->assertEquals([ + IdEnvelope::class, + ], $stack); + } + + public function testEnvelopeDuplicates(): void + { + $message = new Message('handler', 'test'); + $message = new IdEnvelope($message, 'test-id'); + $message = new IdEnvelope($message, 'test-id'); + $message = new IdEnvelope($message, 'test-id'); + + $this->assertEquals('test', $message->getMessage()->getData()); + + $stack = $message->getMetadata()[EnvelopeInterface::ENVELOPE_STACK_KEY]; + $this->assertIsArray($stack); + + $this->assertEquals([ + IdEnvelope::class, + IdEnvelope::class, + IdEnvelope::class, + ], $stack); + } +} diff --git a/tests/Unit/Message/JsonMessageSerializerTest.php b/tests/Unit/Message/JsonMessageSerializerTest.php new file mode 100644 index 00000000..776a9835 --- /dev/null +++ b/tests/Unit/Message/JsonMessageSerializerTest.php @@ -0,0 +1,160 @@ +createSerializer(); + + $this->expectExceptionMessage(sprintf('Payload must be array. Got %s.', get_debug_type($payload))); + $this->expectException(InvalidArgumentException::class); + $serializer->unserialize(json_encode($payload)); + } + + public static function dataUnsupportedPayloadFormat(): iterable + { + yield 'string' => ['']; + yield 'number' => [1]; + yield 'boolean' => [true]; + yield 'null' => [null]; + } + + /** + * @dataProvider dataUnsupportedMetadataFormat + */ + public function testMetadataFormat(mixed $meta): void + { + $payload = ['data' => 'test', 'meta' => $meta]; + $serializer = $this->createSerializer(); + + $this->expectExceptionMessage(sprintf('Metadata must be array. Got %s.', get_debug_type($meta))); + $this->expectException(InvalidArgumentException::class); + $serializer->unserialize(json_encode($payload)); + } + + public static function dataUnsupportedMetadataFormat(): iterable + { + yield 'string' => ['']; + yield 'number' => [1]; + yield 'boolean' => [true]; + } + + public function testUnserializeFromData(): void + { + $payload = ['data' => 'test']; + $serializer = $this->createSerializer(); + + $message = $serializer->unserialize(json_encode($payload)); + + $this->assertInstanceOf(MessageInterface::class, $message); + $this->assertEquals($payload['data'], $message->getData()); + $this->assertEquals([], $message->getMetadata()); + } + + public function testUnserializeWithMetadata(): void + { + $payload = ['data' => 'test', 'meta' => ['int' => 1, 'str' => 'string', 'bool' => true]]; + $serializer = $this->createSerializer(); + + $message = $serializer->unserialize(json_encode($payload)); + + $this->assertInstanceOf(MessageInterface::class, $message); + $this->assertEquals($payload['data'], $message->getData()); + $this->assertEquals(['int' => 1, 'str' => 'string', 'bool' => true], $message->getMetadata()); + } + + public function testUnserializeEnvelopeStack(): void + { + $payload = [ + 'data' => 'test', + 'meta' => [ + EnvelopeInterface::ENVELOPE_STACK_KEY => [ + IdEnvelope::class, + ], + ], + ]; + $serializer = $this->createSerializer(); + + $message = $serializer->unserialize(json_encode($payload)); + + $this->assertInstanceOf(MessageInterface::class, $message); + $this->assertEquals($payload['data'], $message->getData()); + $this->assertEquals([IdEnvelope::class], $message->getMetadata()[EnvelopeInterface::ENVELOPE_STACK_KEY]); + + $this->assertInstanceOf(IdEnvelope::class, $message); + $this->assertNull($message->getId()); + $this->assertInstanceOf(Message::class, $message->getMessage()); + } + + public function testSerialize(): void + { + $message = new Message('handler', 'test'); + + $serializer = $this->createSerializer(); + + $json = $serializer->serialize($message); + + $this->assertEquals( + '{"name":"handler","data":"test","meta":[]}', + $json, + ); + } + + public function testSerializeEnvelopeStack(): void + { + $message = new Message('handler', 'test'); + $message = new IdEnvelope($message, 'test-id'); + + $serializer = $this->createSerializer(); + + $json = $serializer->serialize($message); + + $this->assertEquals( + sprintf( + '{"name":"handler","data":"test","meta":{"envelopes":["%s"],"%s":"test-id"}}', + str_replace('\\', '\\\\', IdEnvelope::class), + IdEnvelope::MESSAGE_ID_KEY, + ), + $json, + ); + + $message = $serializer->unserialize($json); + + $this->assertInstanceOf(IdEnvelope::class, $message); + $this->assertEquals('test-id', $message->getId()); + $this->assertEquals([ + EnvelopeInterface::ENVELOPE_STACK_KEY => [ + IdEnvelope::class, + ], + IdEnvelope::MESSAGE_ID_KEY => 'test-id', + ], $message->getMetadata()); + + $this->assertEquals([ + EnvelopeInterface::ENVELOPE_STACK_KEY => [], + IdEnvelope::MESSAGE_ID_KEY => 'test-id', + ], $message->getMessage()->getMetadata()); + } + + private function createSerializer(): JsonMessageSerializer + { + return new JsonMessageSerializer(); + } +} diff --git a/tests/Unit/WorkerTest.php b/tests/Unit/WorkerTest.php index c9ae1d6b..d0fc9733 100644 --- a/tests/Unit/WorkerTest.php +++ b/tests/Unit/WorkerTest.php @@ -108,7 +108,7 @@ public function testJobExecutedWithStaticDefinitionHandler(): void public function testJobFailWithDefinitionUndefinedMethodHandler(): void { - $this->expectExceptionMessage("Queue handler with name simple doesn't exist"); + $this->expectExceptionMessage('Queue handler with name "simple" does not exist'); $message = new Message('simple', ['test-data']); $logger = new SimpleLogger(); @@ -124,7 +124,7 @@ public function testJobFailWithDefinitionUndefinedMethodHandler(): void public function testJobFailWithDefinitionUndefinedClassHandler(): void { - $this->expectExceptionMessage("Queue handler with name simple doesn't exist"); + $this->expectExceptionMessage('Queue handler with name "simple" does not exist'); $message = new Message('simple', ['test-data']); $logger = new SimpleLogger(); @@ -146,7 +146,7 @@ public function testJobFailWithDefinitionUndefinedClassHandler(): void public function testJobFailWithDefinitionClassNotFoundInContainerHandler(): void { - $this->expectExceptionMessage("Queue handler with name simple doesn't exist"); + $this->expectExceptionMessage('Queue handler with name "simple" does not exist'); $message = new Message('simple', ['test-data']); $logger = new SimpleLogger(); $container = new SimpleContainer(); From 4ace666ec6fb1f48843640514abf7043556ef687 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Feb 2024 09:35:33 +0300 Subject: [PATCH 02/19] Update rector/rector requirement from ^0.19.0 to ^1.0.0 (#195) Updates the requirements on [rector/rector](https://github.com/rectorphp/rector) to permit the latest version. - [Release notes](https://github.com/rectorphp/rector/releases) - [Commits](https://github.com/rectorphp/rector/compare/0.19.0...1.0.0) --- updated-dependencies: - dependency-name: rector/rector dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index 3f7fbd6c..69d46812 100644 --- a/composer.json +++ b/composer.json @@ -37,7 +37,7 @@ "require-dev": { "maglnet/composer-require-checker": "^4.7", "phpunit/phpunit": "^10.5", - "rector/rector": "^0.19.0", + "rector/rector": "^1.0.0", "roave/infection-static-analysis-plugin": "^1.34", "spatie/phpunit-watcher": "^1.23", "vimeo/psalm": "^5.20", From b47a41ffcd993812b63b6b87c2b7cd8988f24596 Mon Sep 17 00:00:00 2001 From: nkondrashov Date: Wed, 21 Feb 2024 12:57:33 +0300 Subject: [PATCH 03/19] Fixes from reanimation demo app (#196) Co-authored-by: nkondrashov --- config/di.php | 4 ++-- src/Command/ListenAllCommand.php | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/di.php b/config/di.php index dcb37cbf..3af1ad8b 100644 --- a/config/di.php +++ b/config/di.php @@ -59,12 +59,12 @@ MessageSerializerInterface::class => JsonMessageSerializer::class, RunCommand::class => [ '__construct()' => [ - 'channels' => array_keys($params['yiisoft/yii-queue']['channel-definitions']), + 'channels' => array_keys($params['yiisoft/queue']['channel-definitions']), ], ], ListenAllCommand::class => [ '__construct()' => [ - 'channels' => array_keys($params['yiisoft/yii-queue']['channel-definitions']), + 'channels' => array_keys($params['yiisoft/queue']['channel-definitions']), ], ], ]; diff --git a/src/Command/ListenAllCommand.php b/src/Command/ListenAllCommand.php index f3d522d8..e94a77fe 100644 --- a/src/Command/ListenAllCommand.php +++ b/src/Command/ListenAllCommand.php @@ -2,7 +2,7 @@ declare(strict_types=1); -namespace Yiisoft\Yii\Queue\Command; +namespace Yiisoft\Queue\Command; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Input\InputArgument; From e6993323627bfe569c55633d2ff0872ce9fd667a Mon Sep 17 00:00:00 2001 From: Luiz Marin <67489841+luizcmarin@users.noreply.github.com> Date: Tue, 21 May 2024 10:42:56 -0300 Subject: [PATCH 04/19] Document standardization (#199) Co-authored-by: Sergei Predvoditelev --- README.md | 87 +++++++------------ UPGRADE.md | 4 +- composer.json | 14 ++- docs/guide/README.md | 11 --- docs/guide/en/README.md | 11 +++ docs/guide/{ => en}/adapter-list.md | 0 docs/guide/{ => en}/adapter-sync.md | 0 docs/guide/{ => en}/error-handling.md | 0 docs/guide/{ => en}/loops.md | 0 .../{ => en}/migrating-from-yii2-queue.md | 0 docs/guide/{ => en}/usage.md | 0 docs/guide/{ => en}/worker.md | 0 docs/internals.md | 44 ++++++++++ 13 files changed, 102 insertions(+), 69 deletions(-) delete mode 100644 docs/guide/README.md create mode 100644 docs/guide/en/README.md rename docs/guide/{ => en}/adapter-list.md (100%) rename docs/guide/{ => en}/adapter-sync.md (100%) rename docs/guide/{ => en}/error-handling.md (100%) rename docs/guide/{ => en}/loops.md (100%) rename docs/guide/{ => en}/migrating-from-yii2-queue.md (100%) rename docs/guide/{ => en}/usage.md (100%) rename docs/guide/{ => en}/worker.md (100%) create mode 100644 docs/internals.md diff --git a/README.md b/README.md index 5755c3ff..6089da18 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,11 @@

- + Yii -

Yii Queue Extension

+

Yii Queue


-An extension for running tasks asynchronously via queues. - -Documentation is at [docs/guide/README.md](docs/guide/README.md). - [![Latest Stable Version](https://poser.pugx.org/yiisoft/queue/v/stable.svg)](https://packagist.org/packages/yiisoft/queue) [![Total Downloads](https://poser.pugx.org/yiisoft/queue/downloads.svg)](https://packagist.org/packages/yiisoft/queue) [![Build status](https://github.com/yiisoft/queue/workflows/build/badge.svg)](https://github.com/yiisoft/queue/actions) @@ -19,29 +15,26 @@ Documentation is at [docs/guide/README.md](docs/guide/README.md). [![static analysis](https://github.com/yiisoft/queue/workflows/static%20analysis/badge.svg)](https://github.com/yiisoft/queue/actions?query=workflow%3A%22static+analysis%22) [![type-coverage](https://shepherd.dev/github/yiisoft/queue/coverage.svg)](https://shepherd.dev/github/yiisoft/queue) -## Installation +An extension for running tasks asynchronously via queues. -The preferred way to install this extension is through [composer](http://getcomposer.org/download/). +## Requirements -Either run +- PHP 8.1 or higher. -```shell -composer require yiisoft/queue -``` +## Installation -or add +The package could be installed with [Composer](https://getcomposer.org): +```shell +composer require yiisoft/queue ``` -"yiisoft/queue": "~3.0" -``` - -to the `require` section of your `composer.json` file. ## Ready for yiisoft/config If you are using [yiisoft/config](https://github.com/yiisoft/config), you'll find out this package has some defaults in the [`common`](config/di.php) and [`params`](config/params.php) configurations saving your time. Things you should change to start working with the queue: + - Optionally: define default `\Yiisoft\Queue\Adapter\AdapterInterface` implementation. - And/or define channel-specific `AdapterInterface` implementations in the `channel-definitions` params key to be used with the [queue factory](#different-queue-channels). @@ -53,7 +46,7 @@ change to start working with the queue: If you have experience with `yiisoft/yii2-queue`, you will find out that this package is similar. Though, there are some key differences which are described in the "[migrating from yii2-queue](docs/guide/migrating-from-yii2-queue.md)" article. -## Basic Usage +## General usage Each queue task consists of two parts: @@ -215,6 +208,7 @@ twice for a queue message. That means you can add extra functionality on message of the two classes: `PushMiddlewareDispatcher` and `ConsumeMiddlewareDispatcher` respectively. You can use any of these formats to define a middleware: + - A ready-to-use middleware object: `new FooMiddleware()`. It must implement `MiddlewarePushInterface`, `MiddlewareConsumeInterface` or `MiddlewareFailureInterface` depending on the place you use it. - An array in the format of [yiisoft/definitions](https://github.com/yiisoft/definitions). @@ -225,6 +219,7 @@ You can use any of these formats to define a middleware: Middleware will be executed forwards in the same order they are defined. If you define it like the following: `[$middleware1, $midleware2]`, the execution will look like this: + ```mermaid graph LR StartPush((Start)) --> PushMiddleware1[$middleware1] --> PushMiddleware2[$middleware2] --> Push(Push to a queue) @@ -238,7 +233,8 @@ graph LR ``` ### Push pipeline -When you push a message, you can use middlewares to modify both message and queue adapter. + +When you push a message, you can use middlewares to modify both message and queue adapter. With message modification you can add extra data, obfuscate data, collect metrics, etc. With queue adapter modification you can redirect message to another queue, delay message consuming, and so on. @@ -256,17 +252,17 @@ in the `PushRequest` object. You will get a `AdapterNotConfiguredException`, if You have three places to define push middlewares: 1. `PushMiddlewareDispatcher`. You can pass it either to the constructor, or to the `withMiddlewares()` method, which -creates a completely new dispatcher object with only those middlewares, which are passed as arguments. -If you use [yiisoft/config](yiisoft/config), you can add middleware to the `middlewares-push` key of the +creates a completely new dispatcher object with only those middlewares, which are passed as arguments. +If you use [yiisoft/config](yiisoft/config), you can add middleware to the `middlewares-push` key of the `yiisoft/queue` array in the `params`. -2. Pass middlewares to either `Queue::withMiddlewares()` or `Queue::withMiddlewaresAdded()` methods. The difference is -that the former will completely replace an existing middleware stack, while the latter will add passed middlewares to -the end of the existing stack. These middlewares will be executed after the common ones, passed directly to the -`PushMiddlewareDispatcher`. It's useful when defining a queue channel. Both methods return a new instance of the `Queue` +2. Pass middlewares to either `Queue::withMiddlewares()` or `Queue::withMiddlewaresAdded()` methods. The difference is +that the former will completely replace an existing middleware stack, while the latter will add passed middlewares to +the end of the existing stack. These middlewares will be executed after the common ones, passed directly to the +`PushMiddlewareDispatcher`. It's useful when defining a queue channel. Both methods return a new instance of the `Queue` class. 3. Put middlewares into the `Queue::push()` method like this: `$queue->push($message, ...$middlewares)`. These -middlewares have the lowest priority and will be executed after those which are in the `PushMiddlewareDispatcher` and -the ones passed to the `Queue::withMiddlewares()` and `Queue::withMiddlewaresAdded()` and only for the message passed +middlewares have the lowest priority and will be executed after those which are in the `PushMiddlewareDispatcher` and +the ones passed to the `Queue::withMiddlewares()` and `Queue::withMiddlewaresAdded()` and only for the message passed along with them. ### Consume pipeline @@ -276,6 +272,7 @@ You can set a middleware pipeline for a message when it will be consumed from a ### Error handling pipeline Often when some job is failing, we want to retry its execution a couple more times or redirect it to another queue channel. This can be done in `yiisoft/queue` with Failure middleware pipeline. They are triggered each time message processing via the Consume middleware pipeline is interrupted with any `Throwable`. The key differences from the previous two pipelines: + - You should set up the middleware pipeline separately for each queue channel. That means, the format should be `['channel-name' => [FooMiddleware::class]]` instead of `[FooMiddleware::class]`, like for the other two pipelines. There is also a default key, which will be used for those channels without their own one: `FailureMiddlewareDispatcher::DEFAULT_PIPELINE`. - The last middleware will throw the exception, which will come with the `FailureHandlingRequest` object. If you don't want the exception to be thrown, your middlewares should `return` a request without calling `$handler->handleFailure()`. @@ -283,31 +280,20 @@ You can declare error handling middleware pipeline in the `FailureMiddlewareDisp See [error handling docs](docs/guide/error-handling.md) for details. -## Extra +## Documentation -### Unit testing +- [Guide](docs/guide/en/README.md) +- [Internals](docs/internals.md) -The package is tested with [PHPUnit](https://phpunit.de/). To run tests: +If you need help or have a question, the [Yii Forum](https://forum.yiiframework.com/c/yii-3-0/63) is a good place for that. +You may also check out other [Yii Community Resources](https://www.yiiframework.com/community). -```shell -./vendor/bin/phpunit -``` - -### Mutation testing - -The package tests are checked with [Infection](https://infection.github.io/) mutation framework. To run it: - -```shell -./vendor/bin/infection -``` - -### Static analysis +## License -The code is statically analyzed with [Psalm](https://psalm.dev/). To run static analysis: +The Yii Queue is free software. It is released under the terms of the BSD License. +Please see [`LICENSE`](./LICENSE.md) for more information. -```shell -./vendor/bin/psalm -``` +Maintained by [Yii Software](https://www.yiiframework.com/). ### Support the project @@ -320,10 +306,3 @@ The code is statically analyzed with [Psalm](https://psalm.dev/). To run static [![Telegram](https://img.shields.io/badge/telegram-join-1DA1F2?style=flat&logo=telegram)](https://t.me/yii3en) [![Facebook](https://img.shields.io/badge/facebook-join-1DA1F2?style=flat&logo=facebook&logoColor=ffffff)](https://www.facebook.com/groups/yiitalk) [![Slack](https://img.shields.io/badge/slack-join-1DA1F2?style=flat&logo=slack)](https://yiiframework.com/go/slack) - -## License - -The Yii Queue Extension is free software. It is released under the terms of the BSD License. -Please see [`LICENSE`](./LICENSE.md) for more information. - -Maintained by [Yii Software](https://www.yiiframework.com/). diff --git a/UPGRADE.md b/UPGRADE.md index 426b6ef3..c1aa6527 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -1,6 +1,6 @@ -Upgrading Instructions -====================== +# Yii Queue Upgrading Instructions This file contains the upgrade notes. These notes highlight changes that could break your application when you upgrade the package from one version to another. +## Changes summary diff --git a/composer.json b/composer.json index 69d46812..c59f8ca9 100644 --- a/composer.json +++ b/composer.json @@ -17,12 +17,22 @@ "license": "BSD-3-Clause", "support": { "issues": "https://github.com/yiisoft/queue/issues?state=open", + "source": "https://github.com/yiisoft/queue", "forum": "https://www.yiiframework.com/forum/", "wiki": "https://www.yiiframework.com/wiki/", "irc": "ircs://irc.libera.chat:6697/yii", - "chat": "https://t.me/yii3en", - "source": "https://github.com/yiisoft/queue" + "chat": "https://t.me/yii3en" }, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/yiisoft" + }, + { + "type": "github", + "url": "https://github.com/sponsors/yiisoft" + } + ], "minimum-stability": "dev", "prefer-stable": true, "require": { diff --git a/docs/guide/README.md b/docs/guide/README.md deleted file mode 100644 index 6d6f175a..00000000 --- a/docs/guide/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Yii Queue extension - -An extension for running tasks asynchronously via queues. - -## Guides and concept explanations - -* [Usage basics](usage.md) -* [Migrating from `yii2-queue`](migrating-from-yii2-queue.md) -* [Errors and retryable jobs](error-handling.md) -* [Workers](worker.md) -* [Adapter list](adapter-list.md) diff --git a/docs/guide/en/README.md b/docs/guide/en/README.md new file mode 100644 index 00000000..564b8ca9 --- /dev/null +++ b/docs/guide/en/README.md @@ -0,0 +1,11 @@ +# Yii Queue + +An extension for running tasks asynchronously via queues. + +## Guides and concept explanations + +- [Usage basics](usage.md) +- [Migrating from `yii2-queue`](migrating-from-yii2-queue.md) +- [Errors and retryable jobs](error-handling.md) +- [Workers](worker.md) +- [Adapter list](adapter-list.md) diff --git a/docs/guide/adapter-list.md b/docs/guide/en/adapter-list.md similarity index 100% rename from docs/guide/adapter-list.md rename to docs/guide/en/adapter-list.md diff --git a/docs/guide/adapter-sync.md b/docs/guide/en/adapter-sync.md similarity index 100% rename from docs/guide/adapter-sync.md rename to docs/guide/en/adapter-sync.md diff --git a/docs/guide/error-handling.md b/docs/guide/en/error-handling.md similarity index 100% rename from docs/guide/error-handling.md rename to docs/guide/en/error-handling.md diff --git a/docs/guide/loops.md b/docs/guide/en/loops.md similarity index 100% rename from docs/guide/loops.md rename to docs/guide/en/loops.md diff --git a/docs/guide/migrating-from-yii2-queue.md b/docs/guide/en/migrating-from-yii2-queue.md similarity index 100% rename from docs/guide/migrating-from-yii2-queue.md rename to docs/guide/en/migrating-from-yii2-queue.md diff --git a/docs/guide/usage.md b/docs/guide/en/usage.md similarity index 100% rename from docs/guide/usage.md rename to docs/guide/en/usage.md diff --git a/docs/guide/worker.md b/docs/guide/en/worker.md similarity index 100% rename from docs/guide/worker.md rename to docs/guide/en/worker.md diff --git a/docs/internals.md b/docs/internals.md new file mode 100644 index 00000000..087a514a --- /dev/null +++ b/docs/internals.md @@ -0,0 +1,44 @@ +# Internals + +## Unit testing + +The package is tested with [PHPUnit](https://phpunit.de/). To run tests: + +```shell +./vendor/bin/phpunit +``` + +## Mutation testing + +The package tests are checked with [Infection](https://infection.github.io/) mutation framework with +[Infection Static Analysis Plugin](https://github.com/Roave/infection-static-analysis-plugin). To run it: + +```shell +./vendor/bin/roave-infection-static-analysis-plugin +``` + +## Static analysis + +The code is statically analyzed with [Psalm](https://psalm.dev/). To run static analysis: + +```shell +./vendor/bin/psalm +``` + +## Code style + +Use [Rector](https://github.com/rectorphp/rector) to make codebase follow some specific rules or +use either newest or any specific version of PHP: + +```shell +./vendor/bin/rector +``` + +## Dependencies + +This package uses [composer-require-checker](https://github.com/maglnet/ComposerRequireChecker) to check if +all dependencies are correctly defined in `composer.json`. To run the checker, execute the following command: + +```shell +./vendor/bin/composer-require-checker +``` From 75325a3618d3a4ee9a5be347e8e8456db5c7c94c Mon Sep 17 00:00:00 2001 From: Alexander Makarov Date: Fri, 24 May 2024 12:01:39 +0300 Subject: [PATCH 05/19] Add NATS --- docs/guide/en/adapter-list.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/guide/en/adapter-list.md b/docs/guide/en/adapter-list.md index 3b9bb1be..2ce0e4dc 100644 --- a/docs/guide/en/adapter-list.md +++ b/docs/guide/en/adapter-list.md @@ -3,3 +3,4 @@ Queue Adapters * [Synchronous](adapter-sync.md) - adapter for development and test environments * [AMQP](https://github.com/yiisoft/queue-amqp) - adapter over AMQP protocol via [amqplib](https://github.com/php-amqplib/php-amqplib) +* [NATS](https://github.com/g41797/queue-nats) - [NATS](https://nats.io/) JetStream adapter From 4167ef3e623ad1f6afc94e10e4db9fb08bdb5f42 Mon Sep 17 00:00:00 2001 From: Luiz Marin <67489841+luizcmarin@users.noreply.github.com> Date: Sat, 25 May 2024 10:20:21 -0300 Subject: [PATCH 06/19] Delete UPGRADE.md (#203) Co-authored-by: Sergei Predvoditelev --- UPGRADE.md | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 UPGRADE.md diff --git a/UPGRADE.md b/UPGRADE.md deleted file mode 100644 index c1aa6527..00000000 --- a/UPGRADE.md +++ /dev/null @@ -1,6 +0,0 @@ -# Yii Queue Upgrading Instructions - -This file contains the upgrade notes. These notes highlight changes that could break your -application when you upgrade the package from one version to another. - -## Changes summary From 10b88880222677300ec2517c8712957c44556c9e Mon Sep 17 00:00:00 2001 From: Alexander Tebiev <326840+beeyev@users.noreply.github.com> Date: Tue, 28 May 2024 11:33:17 +0200 Subject: [PATCH 07/19] Fix broken link (#206) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6089da18..ed467fab 100644 --- a/README.md +++ b/README.md @@ -198,7 +198,7 @@ See the documentation for more details about adapter specific console commands a The component also has the ability to track the status of a job which was pushed into queue. -For more details see [the guide](docs/guide/README.md). +For more details see [the guide](docs/guide/en/README.md). ## Middleware pipelines From d21ca215090f6eafd3961527f554a3c7ebea4b03 Mon Sep 17 00:00:00 2001 From: g41797 Date: Wed, 19 Jun 2024 21:09:50 +0300 Subject: [PATCH 08/19] Add community maintained adapters to the list of queue adapters (#209) --- docs/guide/en/adapter-list.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/guide/en/adapter-list.md b/docs/guide/en/adapter-list.md index 2ce0e4dc..17bf91f0 100644 --- a/docs/guide/en/adapter-list.md +++ b/docs/guide/en/adapter-list.md @@ -3,4 +3,8 @@ Queue Adapters * [Synchronous](adapter-sync.md) - adapter for development and test environments * [AMQP](https://github.com/yiisoft/queue-amqp) - adapter over AMQP protocol via [amqplib](https://github.com/php-amqplib/php-amqplib) + + +There are other queue adapters contributed and maintained by the community and available on GitHub, such as: * [NATS](https://github.com/g41797/queue-nats) - [NATS](https://nats.io/) JetStream adapter +* [Pulsar](https://github.com/g41797/queue-pulsar) - [Apache Pulsar](https://pulsar.apache.org/) adapter From b2c408705695a87229c704fd03e0dc6d82aa4723 Mon Sep 17 00:00:00 2001 From: g41797 Date: Sat, 22 Jun 2024 16:00:20 +0300 Subject: [PATCH 09/19] Add SQS to the list of queue adapters (#210) --- docs/guide/en/adapter-list.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/guide/en/adapter-list.md b/docs/guide/en/adapter-list.md index 17bf91f0..f4a86090 100644 --- a/docs/guide/en/adapter-list.md +++ b/docs/guide/en/adapter-list.md @@ -8,3 +8,4 @@ Queue Adapters There are other queue adapters contributed and maintained by the community and available on GitHub, such as: * [NATS](https://github.com/g41797/queue-nats) - [NATS](https://nats.io/) JetStream adapter * [Pulsar](https://github.com/g41797/queue-pulsar) - [Apache Pulsar](https://pulsar.apache.org/) adapter +* [SQS](https://github.com/g41797/queue-sqs) - [Amazon SQS](https://aws.amazon.com/sqs/) adapter From 4c4a41c0df2194aa1d6a157a4a95356ff657d616 Mon Sep 17 00:00:00 2001 From: g41797 Date: Wed, 10 Jul 2024 09:57:32 +0300 Subject: [PATCH 10/19] Add Apache Kafka to the list of queue adapters (#211) --- docs/guide/en/adapter-list.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/guide/en/adapter-list.md b/docs/guide/en/adapter-list.md index f4a86090..ef740218 100644 --- a/docs/guide/en/adapter-list.md +++ b/docs/guide/en/adapter-list.md @@ -9,3 +9,4 @@ There are other queue adapters contributed and maintained by the community and a * [NATS](https://github.com/g41797/queue-nats) - [NATS](https://nats.io/) JetStream adapter * [Pulsar](https://github.com/g41797/queue-pulsar) - [Apache Pulsar](https://pulsar.apache.org/) adapter * [SQS](https://github.com/g41797/queue-sqs) - [Amazon SQS](https://aws.amazon.com/sqs/) adapter +* [Kafka](https://github.com/g41797/queue-kafka) - [Apache Kafka](https://kafka.apache.org/) adapter From c07610096ac200c1b357c0d0a399d4f1a1accfb8 Mon Sep 17 00:00:00 2001 From: g41797 Date: Sat, 24 Aug 2024 17:29:47 +0300 Subject: [PATCH 11/19] Add Valkey and Beanstalkd to the list of queue adapters (#212) * Add Valkey to the list of queue adapters * Add Beanstalkd to the list of queue adapters --- docs/guide/en/adapter-list.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/guide/en/adapter-list.md b/docs/guide/en/adapter-list.md index ef740218..a24b199f 100644 --- a/docs/guide/en/adapter-list.md +++ b/docs/guide/en/adapter-list.md @@ -10,3 +10,5 @@ There are other queue adapters contributed and maintained by the community and a * [Pulsar](https://github.com/g41797/queue-pulsar) - [Apache Pulsar](https://pulsar.apache.org/) adapter * [SQS](https://github.com/g41797/queue-sqs) - [Amazon SQS](https://aws.amazon.com/sqs/) adapter * [Kafka](https://github.com/g41797/queue-kafka) - [Apache Kafka](https://kafka.apache.org/) adapter +* [Valkey](https://github.com/g41797/queue-valkey) - [Valkey NoSQL data store ](https://valkey.io/) adapter +* [Beanstalkd](https://github.com/g41797/queue-beanstalkd) - [Beanstalkd - simple, fast work queue](https://beanstalkd.github.io/) adapter From 8ee8fb652ee946ccc1f0468dc4551a732409564b Mon Sep 17 00:00:00 2001 From: Viktor Babanov Date: Sun, 25 Aug 2024 17:02:32 +0500 Subject: [PATCH 12/19] Set channel name to a newly created adapter in QueueFactory (#213) --- src/QueueFactory.php | 2 +- tests/Unit/QueueFactoryTest.php | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/QueueFactory.php b/src/QueueFactory.php index ef35c9b8..e6b963d0 100644 --- a/src/QueueFactory.php +++ b/src/QueueFactory.php @@ -76,7 +76,7 @@ private function create(string $channel): QueueInterface if (isset($this->channelConfiguration[$channel])) { $definition = $this->channelConfiguration[$channel]; $this->checkDefinitionType($channel, $definition); - $adapter = $this->createFromDefinition($channel, $definition); + $adapter = $this->createFromDefinition($channel, $definition)->withChannel($channel); return $this->queue->withChannelName($channel)->withAdapter($adapter); } diff --git a/tests/Unit/QueueFactoryTest.php b/tests/Unit/QueueFactoryTest.php index 3597a5ae..a72f9751 100644 --- a/tests/Unit/QueueFactoryTest.php +++ b/tests/Unit/QueueFactoryTest.php @@ -112,7 +112,10 @@ public function testThrowExceptionOnIncorrectDefinition(): void public function testSuccessfulDefinitionWithDefaultAdapter(): void { $adapterDefault = $this->createMock(AdapterInterface::class); + $adapterDefault->method('withChannel')->willReturn($adapterDefault); + $adapterNew = $this->createMock(AdapterInterface::class); + $adapterNew->method('withChannel')->willReturn($adapterNew); $queue = $this->createMock(QueueInterface::class); $queue @@ -142,6 +145,7 @@ public function testSuccessfulDefinitionWithDefaultAdapter(): void public function testSuccessfulDefinitionWithoutDefaultAdapter(): void { $adapterNew = $this->createMock(AdapterInterface::class); + $adapterNew->method('withChannel')->willReturn($adapterNew); $queue = $this->createMock(QueueInterface::class); $queue From 6baa9ceb2a38f7a0f4fd11970fb4437c3c72266a Mon Sep 17 00:00:00 2001 From: viktorprogger Date: Sun, 25 Aug 2024 17:34:20 +0500 Subject: [PATCH 13/19] Suppress Psalm warnings on PHP 8.3 --- src/Cli/SignalLoop.php | 15 ++++++++++++--- src/Enum/JobStatus.php | 3 +++ src/Message/EnvelopeInterface.php | 1 + src/QueueFactoryInterface.php | 1 + 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/Cli/SignalLoop.php b/src/Cli/SignalLoop.php index 5d765abe..6e0524ba 100644 --- a/src/Cli/SignalLoop.php +++ b/src/Cli/SignalLoop.php @@ -8,11 +8,20 @@ class SignalLoop implements LoopInterface { use SoftLimitTrait; - /** @psalm-suppress UndefinedConstant */ + /** + * @psalm-suppress UndefinedConstant + * @psalm-suppress MissingClassConstType + */ protected const SIGNALS_EXIT = [SIGHUP, SIGINT, SIGTERM]; - /** @psalm-suppress UndefinedConstant */ + /** + * @psalm-suppress UndefinedConstant + * @psalm-suppress MissingClassConstType + */ protected const SIGNALS_SUSPEND = [SIGTSTP]; - /** @psalm-suppress UndefinedConstant */ + /** + * @psalm-suppress UndefinedConstant + * @psalm-suppress MissingClassConstType + */ protected const SIGNALS_RESUME = [SIGCONT]; protected bool $pause = false; protected bool $exit = false; diff --git a/src/Enum/JobStatus.php b/src/Enum/JobStatus.php index beca6c3c..764f98d1 100644 --- a/src/Enum/JobStatus.php +++ b/src/Enum/JobStatus.php @@ -8,8 +8,11 @@ class JobStatus { + /** @psalm-suppress MissingClassConstType */ final public const WAITING = 1; + /** @psalm-suppress MissingClassConstType */ final public const RESERVED = 2; + /** @psalm-suppress MissingClassConstType */ final public const DONE = 3; protected int $status; diff --git a/src/Message/EnvelopeInterface.php b/src/Message/EnvelopeInterface.php index b0f8d89f..7c58daa1 100644 --- a/src/Message/EnvelopeInterface.php +++ b/src/Message/EnvelopeInterface.php @@ -9,6 +9,7 @@ */ interface EnvelopeInterface extends MessageInterface { + /** @psalm-suppress MissingClassConstType */ public const ENVELOPE_STACK_KEY = 'envelopes'; public static function fromMessage(MessageInterface $message): self; diff --git a/src/QueueFactoryInterface.php b/src/QueueFactoryInterface.php index 82c53f7f..9761e583 100644 --- a/src/QueueFactoryInterface.php +++ b/src/QueueFactoryInterface.php @@ -8,6 +8,7 @@ interface QueueFactoryInterface { + /** @psalm-suppress MissingClassConstType */ public const DEFAULT_CHANNEL_NAME = 'yii-queue'; /** From 22a0456d43b4d2d1a3c55ffe3afd04b666aa8b82 Mon Sep 17 00:00:00 2001 From: viktorprogger Date: Sun, 25 Aug 2024 17:42:11 +0500 Subject: [PATCH 14/19] A couple more assertions in tests --- tests/Unit/EnvelopeTest.php | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/Unit/EnvelopeTest.php b/tests/Unit/EnvelopeTest.php index af412125..d7e4650e 100644 --- a/tests/Unit/EnvelopeTest.php +++ b/tests/Unit/EnvelopeTest.php @@ -24,24 +24,27 @@ public function testEnvelopeStack(): void $this->assertEquals([ IdEnvelope::class, ], $stack); + + $this->assertEquals('test-id', $message->getMetadata()[IdEnvelope::MESSAGE_ID_KEY]); } public function testEnvelopeDuplicates(): void { $message = new Message('handler', 'test'); - $message = new IdEnvelope($message, 'test-id'); - $message = new IdEnvelope($message, 'test-id'); - $message = new IdEnvelope($message, 'test-id'); + $message = new IdEnvelope($message, 'test-id-1'); + $message = new IdEnvelope($message, 'test-id-2'); + $message = new IdEnvelope($message, 'test-id-3'); $this->assertEquals('test', $message->getMessage()->getData()); $stack = $message->getMetadata()[EnvelopeInterface::ENVELOPE_STACK_KEY]; $this->assertIsArray($stack); - $this->assertEquals([ IdEnvelope::class, IdEnvelope::class, IdEnvelope::class, ], $stack); + + $this->assertEquals('test-id-3', $message->getMetadata()[IdEnvelope::MESSAGE_ID_KEY]); } } From d231e82313609a1d1470934b99c2b6a395cd3779 Mon Sep 17 00:00:00 2001 From: viktorprogger Date: Sun, 15 Sep 2024 22:01:15 +0500 Subject: [PATCH 15/19] Move fromMessage() method out of EnvelopeTrait and create its own key for the FailureEnvelope. --- src/Message/EnvelopeTrait.php | 5 ----- src/Message/IdEnvelope.php | 5 +++++ .../FailureHandling/FailureEnvelope.php | 12 +++++++++++- .../ExponentialDelayMiddleware.php | 16 +++++++--------- .../Implementation/SendAgainMiddleware.php | 18 ++++++++---------- .../ExponentialDelayMiddlewareTest.php | 13 ++++++++++--- .../Implementation/SendAgainMiddlewareTest.php | 4 ++++ 7 files changed, 45 insertions(+), 28 deletions(-) diff --git a/src/Message/EnvelopeTrait.php b/src/Message/EnvelopeTrait.php index 7e58a97b..cc52cf15 100644 --- a/src/Message/EnvelopeTrait.php +++ b/src/Message/EnvelopeTrait.php @@ -31,11 +31,6 @@ public function getData(): mixed return $this->message->getData(); } - public static function fromMessage(MessageInterface $message): self - { - return new static($message); - } - public function getMetadata(): array { return array_merge( diff --git a/src/Message/IdEnvelope.php b/src/Message/IdEnvelope.php index dec2d679..bbf0d67f 100644 --- a/src/Message/IdEnvelope.php +++ b/src/Message/IdEnvelope.php @@ -19,6 +19,11 @@ public function __construct( ) { } + public static function fromMessage(MessageInterface $message): self + { + return new self($message, $message->getMetadata()[self::MESSAGE_ID_KEY] ?? null); + } + public function setId(string|int|null $id): void { $this->id = $id; diff --git a/src/Middleware/FailureHandling/FailureEnvelope.php b/src/Middleware/FailureHandling/FailureEnvelope.php index a8726285..0f6a62e9 100644 --- a/src/Middleware/FailureHandling/FailureEnvelope.php +++ b/src/Middleware/FailureHandling/FailureEnvelope.php @@ -12,14 +12,24 @@ final class FailureEnvelope implements EnvelopeInterface { use EnvelopeTrait; + public const FAILURE_META_KEY = 'failure-meta'; + public function __construct( private MessageInterface $message, private array $meta = [], ) { } + public static function fromMessage(MessageInterface $message): self + { + return new self($message, $message->getMetadata()[self::FAILURE_META_KEY] ?? []); + } + public function getMetadata(): array { - return array_merge($this->message->getMetadata(), $this->meta); + $meta = $this->message->getMetadata(); + $meta[self::FAILURE_META_KEY] = array_merge($meta[self::FAILURE_META_KEY] ?? [], $this->meta); + + return $meta; } } diff --git a/src/Middleware/FailureHandling/Implementation/ExponentialDelayMiddleware.php b/src/Middleware/FailureHandling/Implementation/ExponentialDelayMiddleware.php index 07d0f640..643ab31c 100644 --- a/src/Middleware/FailureHandling/Implementation/ExponentialDelayMiddleware.php +++ b/src/Middleware/FailureHandling/Implementation/ExponentialDelayMiddleware.php @@ -5,7 +5,6 @@ namespace Yiisoft\Queue\Middleware\FailureHandling\Implementation; use InvalidArgumentException; -use Yiisoft\Queue\Message\Message; use Yiisoft\Queue\Message\MessageInterface; use Yiisoft\Queue\Middleware\FailureHandling\FailureHandlingRequest; use Yiisoft\Queue\Middleware\FailureHandling\MessageFailureHandlerInterface; @@ -24,7 +23,7 @@ final class ExponentialDelayMiddleware implements MiddlewareFailureInterface public const META_KEY_DELAY = 'failure-strategy-exponential-delay-delay'; /** - * @param string $id A unique id to differentiate two and more objects of this class + * @param string $id A unique id to differentiate two and more instances of this class * @param int $maxAttempts Maximum attempts count for this strategy with the given $id before it will give up * @param float $delayInitial The first delay period * @param float $delayMaximum The maximum delay period @@ -84,21 +83,20 @@ private function suites(MessageInterface $message): bool private function createNewMeta(MessageInterface $message): array { - $meta = $message->getMetadata(); - $meta[self::META_KEY_DELAY . "-$this->id"] = $this->getDelay($message); - $meta[self::META_KEY_ATTEMPTS . "-$this->id"] = $this->getAttempts($message) + 1; - - return $meta; + return [ + self::META_KEY_DELAY . "-$this->id" => $this->getDelay($message), + self::META_KEY_ATTEMPTS . "-$this->id" => $this->getAttempts($message) + 1, + ]; } private function getAttempts(MessageInterface $message): int { - return $message->getMetadata()[self::META_KEY_ATTEMPTS . "-$this->id"] ?? 0; + return $message->getMetadata()[FailureEnvelope::FAILURE_META_KEY][self::META_KEY_ATTEMPTS . "-$this->id"] ?? 0; } private function getDelay(MessageInterface $message): float { - $meta = $message->getMetadata(); + $meta = $message->getMetadata()[FailureEnvelope::FAILURE_META_KEY] ?? []; $key = self::META_KEY_DELAY . "-$this->id"; $delayOriginal = (float) ($meta[$key] ?? 0); diff --git a/src/Middleware/FailureHandling/Implementation/SendAgainMiddleware.php b/src/Middleware/FailureHandling/Implementation/SendAgainMiddleware.php index 43ac16f4..a7ae792a 100644 --- a/src/Middleware/FailureHandling/Implementation/SendAgainMiddleware.php +++ b/src/Middleware/FailureHandling/Implementation/SendAgainMiddleware.php @@ -20,14 +20,15 @@ final class SendAgainMiddleware implements MiddlewareFailureInterface public const META_KEY_RESEND = 'failure-strategy-resend-attempts'; /** - * @param string $id A unique id to differentiate two and more objects of this class + * @param string $id A unique id to differentiate two and more instances of this class * @param int $maxAttempts Maximum attempts count for this strategy with the given $id before it will give up - * @param QueueInterface|null $queue + * @param QueueInterface|null $targetQueue Messages will be sent to this queue if set. + * They will be resent to an original queue otherwise. */ public function __construct( private string $id, private int $maxAttempts, - private ?QueueInterface $queue = null, + private ?QueueInterface $targetQueue = null, ) { if ($maxAttempts < 1) { throw new InvalidArgumentException("maxAttempts parameter must be a positive integer, $this->maxAttempts given."); @@ -41,10 +42,10 @@ public function processFailure( $message = $request->getMessage(); if ($this->suites($message)) { $envelope = new FailureEnvelope($message, $this->createMeta($message)); - $envelope = ($this->queue ?? $request->getQueue())->push($envelope); + $envelope = ($this->targetQueue ?? $request->getQueue())->push($envelope); return $request->withMessage($envelope) - ->withQueue($this->queue ?? $request->getQueue()); + ->withQueue($this->targetQueue ?? $request->getQueue()); } return $handler->handleFailure($request); @@ -57,15 +58,12 @@ private function suites(MessageInterface $message): bool private function createMeta(MessageInterface $message): array { - $metadata = $message->getMetadata(); - $metadata[$this->getMetaKey()] = $this->getAttempts($message) + 1; - - return $metadata; + return [$this->getMetaKey() => $this->getAttempts($message) + 1]; } private function getAttempts(MessageInterface $message): int { - $result = $message->getMetadata()[$this->getMetaKey()] ?? 0; + $result = $message->getMetadata()[FailureEnvelope::FAILURE_META_KEY][$this->getMetaKey()] ?? 0; if ($result < 0) { $result = 0; } diff --git a/tests/Unit/Middleware/FailureHandling/Implementation/ExponentialDelayMiddlewareTest.php b/tests/Unit/Middleware/FailureHandling/Implementation/ExponentialDelayMiddlewareTest.php index 5d24f46a..707a1ad7 100644 --- a/tests/Unit/Middleware/FailureHandling/Implementation/ExponentialDelayMiddlewareTest.php +++ b/tests/Unit/Middleware/FailureHandling/Implementation/ExponentialDelayMiddlewareTest.php @@ -8,6 +8,7 @@ use InvalidArgumentException; use PHPUnit\Framework\Attributes\DataProvider; use Yiisoft\Queue\Message\Message; +use Yiisoft\Queue\Middleware\FailureHandling\FailureEnvelope; use Yiisoft\Queue\Middleware\FailureHandling\FailureHandlingRequest; use Yiisoft\Queue\Middleware\FailureHandling\Implementation\ExponentialDelayMiddleware; use Yiisoft\Queue\Middleware\FailureHandling\MessageFailureHandlerInterface; @@ -148,8 +149,11 @@ public function testPipelineSuccess(): void self::assertNotEquals($request, $result); $message = $result->getMessage(); - self::assertArrayHasKey(ExponentialDelayMiddleware::META_KEY_ATTEMPTS . '-test', $message->getMetadata()); - self::assertArrayHasKey(ExponentialDelayMiddleware::META_KEY_DELAY . '-test', $message->getMetadata()); + self::assertArrayHasKey(FailureEnvelope::FAILURE_META_KEY, $message->getMetadata()); + + $meta = $message->getMetadata()[FailureEnvelope::FAILURE_META_KEY]; + self::assertArrayHasKey(ExponentialDelayMiddleware::META_KEY_ATTEMPTS . '-test', $meta); + self::assertArrayHasKey(ExponentialDelayMiddleware::META_KEY_DELAY . '-test', $meta); } public function testPipelineFailure(): void @@ -157,7 +161,10 @@ public function testPipelineFailure(): void $this->expectException(Exception::class); $this->expectExceptionMessage('test'); - $message = new Message('test', null, [ExponentialDelayMiddleware::META_KEY_ATTEMPTS . '-test' => 2]); + $message = new Message( + 'test', + null, + [FailureEnvelope::FAILURE_META_KEY => [ExponentialDelayMiddleware::META_KEY_ATTEMPTS . '-test' => 2]]); $queue = $this->createMock(QueueInterface::class); $middleware = new ExponentialDelayMiddleware( 'test', diff --git a/tests/Unit/Middleware/FailureHandling/Implementation/SendAgainMiddlewareTest.php b/tests/Unit/Middleware/FailureHandling/Implementation/SendAgainMiddlewareTest.php index 57dc4eb9..665c6bea 100644 --- a/tests/Unit/Middleware/FailureHandling/Implementation/SendAgainMiddlewareTest.php +++ b/tests/Unit/Middleware/FailureHandling/Implementation/SendAgainMiddlewareTest.php @@ -10,6 +10,7 @@ use RuntimeException; use Yiisoft\Queue\Message\Message; use Yiisoft\Queue\Message\MessageInterface; +use Yiisoft\Queue\Middleware\FailureHandling\FailureEnvelope; use Yiisoft\Queue\Middleware\FailureHandling\FailureHandlingRequest; use Yiisoft\Queue\Middleware\FailureHandling\Implementation\ExponentialDelayMiddleware; use Yiisoft\Queue\Middleware\FailureHandling\Implementation\SendAgainMiddleware; @@ -150,6 +151,9 @@ public function testQueueSendingStrategies( $this->expectExceptionMessage('testException'); } + $metaInitial = [FailureEnvelope::FAILURE_META_KEY => $metaInitial]; + $metaResult = [FailureEnvelope::FAILURE_META_KEY => $metaResult]; + $handler = $this->getHandler($metaResult, $suites); $queue = $this->getPreparedQueue($metaResult, $suites); From 9392df6ff1d8b79a7cba1b3a60ceddcfd345dd31 Mon Sep 17 00:00:00 2001 From: StyleCI Bot Date: Sun, 15 Sep 2024 17:01:29 +0000 Subject: [PATCH 16/19] Apply fixes from StyleCI --- .../Implementation/ExponentialDelayMiddlewareTest.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/Unit/Middleware/FailureHandling/Implementation/ExponentialDelayMiddlewareTest.php b/tests/Unit/Middleware/FailureHandling/Implementation/ExponentialDelayMiddlewareTest.php index 707a1ad7..29ac6827 100644 --- a/tests/Unit/Middleware/FailureHandling/Implementation/ExponentialDelayMiddlewareTest.php +++ b/tests/Unit/Middleware/FailureHandling/Implementation/ExponentialDelayMiddlewareTest.php @@ -164,7 +164,8 @@ public function testPipelineFailure(): void $message = new Message( 'test', null, - [FailureEnvelope::FAILURE_META_KEY => [ExponentialDelayMiddleware::META_KEY_ATTEMPTS . '-test' => 2]]); + [FailureEnvelope::FAILURE_META_KEY => [ExponentialDelayMiddleware::META_KEY_ATTEMPTS . '-test' => 2]] + ); $queue = $this->createMock(QueueInterface::class); $middleware = new ExponentialDelayMiddleware( 'test', From 9c0ee155637c8997db8f42beaf734ee3a34aca9d Mon Sep 17 00:00:00 2001 From: viktorprogger Date: Sun, 15 Sep 2024 22:33:40 +0500 Subject: [PATCH 17/19] Fix typo in docs --- docs/guide/en/error-handling.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/en/error-handling.md b/docs/guide/en/error-handling.md index 20837c2a..14db9054 100644 --- a/docs/guide/en/error-handling.md +++ b/docs/guide/en/error-handling.md @@ -76,7 +76,7 @@ It's configured via constructor parameters, too. Here they are: - `maxAttempts` - Maximum attempts count for this strategy with the given $id before it will give up. - `delayInitial` - The initial delay that will be applied to a message for the first time. It must be a positive float. - `delayMaximum` - The maximum delay which can be applied to a single message. Must be above the `delayInitial`. -- `exponent` - Message handling delay will be muliplied by exponent each time it fails. +- `exponent` - Message handling delay will be multiplied by exponent each time it fails. - `queue` - The strategy will send the message to the given queue when it's not `null`. That means you can use this strategy to push a message not to the same queue channel it came from. When the `queue` parameter is set to `null`, a message will be sent to the same channel it came from. ## How to create a custom Failure Middleware? From 9fc8dee90766329e6db2448b28299d7ba259e646 Mon Sep 17 00:00:00 2001 From: viktorprogger Date: Sun, 15 Sep 2024 22:34:15 +0500 Subject: [PATCH 18/19] Move PhpUnit cache dir to the vendor dir --- phpunit.xml.dist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 6da3c15a..1968c822 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -2,7 +2,7 @@ Date: Sun, 13 Oct 2024 19:42:34 +0500 Subject: [PATCH 19/19] First benchmarks (#219) * First benchmarks * Apply fixes from StyleCI * remove debug * Add benchmark CI step --------- Co-authored-by: StyleCI Bot --- .github/workflows/bechmark.yml | 33 +++++++ composer.json | 1 + phpbench.json | 8 ++ tests/Benchmark/MetadataBench.php | 109 +++++++++++++++++++++++ tests/Benchmark/QueueBench.php | 112 ++++++++++++++++++++++++ tests/Benchmark/Support/VoidAdapter.php | 52 +++++++++++ tests/docker/php/Dockerfile | 12 +-- tests/docker/php/php.ini | 2 + 8 files changed, 324 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/bechmark.yml create mode 100644 phpbench.json create mode 100644 tests/Benchmark/MetadataBench.php create mode 100644 tests/Benchmark/QueueBench.php create mode 100644 tests/Benchmark/Support/VoidAdapter.php create mode 100644 tests/docker/php/php.ini diff --git a/.github/workflows/bechmark.yml b/.github/workflows/bechmark.yml new file mode 100644 index 00000000..36190113 --- /dev/null +++ b/.github/workflows/bechmark.yml @@ -0,0 +1,33 @@ +on: + pull_request: + paths-ignore: + - 'docs/**' + - 'README.md' + - 'CHANGELOG.md' + - '.gitignore' + - '.gitattributes' + - 'infection.json.dist' + - 'psalm.xml' + - 'tests/**' + + push: + paths-ignore: + - 'docs/**' + - 'README.md' + - 'CHANGELOG.md' + - '.gitignore' + - '.gitattributes' + - 'infection.json.dist' + - 'psalm.xml' + - 'tests/**' + +name: bechmark + +jobs: + phpbench: + uses: yiisoft/actions/.github/workflows/phpbench.yml@master + with: + os: >- + ['ubuntu-latest', 'windows-latest'] + php: >- + ['8.1'] diff --git a/composer.json b/composer.json index c59f8ca9..664f5853 100644 --- a/composer.json +++ b/composer.json @@ -46,6 +46,7 @@ }, "require-dev": { "maglnet/composer-require-checker": "^4.7", + "phpbench/phpbench": "^1.3", "phpunit/phpunit": "^10.5", "rector/rector": "^1.0.0", "roave/infection-static-analysis-plugin": "^1.34", diff --git a/phpbench.json b/phpbench.json new file mode 100644 index 00000000..04436028 --- /dev/null +++ b/phpbench.json @@ -0,0 +1,8 @@ +{ + "$schema":"./vendor/phpbench/phpbench/phpbench.schema.json", + "runner.bootstrap": "vendor/autoload.php", + "runner.path": "tests/Benchmark", + "runner.revs": 100000, + "runner.iterations": 5, + "runner.warmup": 5 +} diff --git a/tests/Benchmark/MetadataBench.php b/tests/Benchmark/MetadataBench.php new file mode 100644 index 00000000..459cd53b --- /dev/null +++ b/tests/Benchmark/MetadataBench.php @@ -0,0 +1,109 @@ + 1]); + $id = $message->getMetadata()['id']; + } + + /** + * Create metadata as object and read its value immediately + */ + #[Tag('metadata_read')] + public function benchEnvelopeRead(): void + { + $message = new IdEnvelope(new Message('foo', 'bar'), 1); + $id = $message->getId(); + } + + /** + * Create metadata as array and read its value from an envelope object + */ + #[Tag('metadata_read')] + public function benchEnvelopeReadRestored(): void + { + $message = IdEnvelope::fromMessage(new Message('foo', 'bar', ['id' => 1])); + $id = $message->getId(); + } + + public function provideEnvelopeStack(): Generator + { + $config = [1 => 'one', 5 => 'three', 15 => 'fifteen']; + $message = new IdEnvelope(new Message('foo', 'bar'), 1); + + for ($i = 1; $i <= max(...array_keys($config)); $i++) { + $message = new FailureEnvelope($message, ["fail$i" => "fail$i"]); + if (isset($config[$i])) { + yield $config[$i] => ['message' => $message]; + } + } + } + + /** + * Read metadata value from an envelope object restored from an envelope stacks of different depth + * + * @psalm-param array{message: MessageInterface} $params + */ + #[ParamProviders('provideEnvelopeStack')] + #[Tag('metadata_read')] + public function benchEnvelopeReadFromStack(array $params): void + { + $id = IdEnvelope::fromMessage($params['message'])->getId(); + } + + public function provideEnvelopeStackCounts(): Generator + { + yield 'one' => [1]; + yield 'three' => [3]; + yield 'fifteen' => [15]; + } + + /** + * Create envelope stack with the given depth + * + * @psalm-param array{0: int} $params + */ + #[ParamProviders('provideEnvelopeStackCounts')] + #[Tag('metadata_create')] + public function benchEnvelopeStackCreation(array $params): void + { + $message = new Message('foo', 'bar'); + for ($i = 0; $i < $params[0]; $i++) { + $message = new FailureEnvelope($message, ["fail$i" => "fail$i"]); + } + } + + /** + * Create metadata array with the given elements count + * + * @psalm-param array{0: int} $params + */ + #[ParamProviders('provideEnvelopeStackCounts')] + #[Tag('metadata_create')] + public function benchMetadataArrayCreation(array $params): void + { + $metadata = ['failure-meta' => []]; + for ($i = 0; $i < $params[0]; $i++) { + $metadata['failure-meta']["fail$i"] = "fail$i"; + } + $message = new Message('foo', 'bar', $metadata); + } +} diff --git a/tests/Benchmark/QueueBench.php b/tests/Benchmark/QueueBench.php new file mode 100644 index 00000000..76480523 --- /dev/null +++ b/tests/Benchmark/QueueBench.php @@ -0,0 +1,112 @@ + static function (): void { + }, + ], + $logger, + new Injector($container), + $container, + new ConsumeMiddlewareDispatcher(new MiddlewareFactoryConsume($container, $callableFactory)), + new FailureMiddlewareDispatcher( + new MiddlewareFactoryFailure($container, $callableFactory), + [], + ), + ); + $this->serializer = new JsonMessageSerializer(); + $this->adapter = new VoidAdapter($this->serializer); + + $this->queue = new Queue( + $worker, + new SimpleLoop(0), + $logger, + new PushMiddlewareDispatcher(new MiddlewareFactoryPush($container, $callableFactory)), + $this->adapter, + ); + } + + public function providePush(): Generator + { + yield 'simple' => ['message' => new Message('foo', 'bar')]; + yield 'with envelopes' => [ + 'message' => new FailureEnvelope( + new IdEnvelope( + new Message('foo', 'bar'), + 'test id', + ), + ['failure-1' => ['a', 'b', 'c']], + ), + ]; + } + + #[ParamProviders('providePush')] + #[Tag('queue_push')] + public function benchPush(array $params): void + { + $this->queue->push($params['message']); + } + + public function provideConsume(): Generator + { + yield 'simple mapping' => ['message' => $this->serializer->serialize(new Message('foo', 'bar'))]; + yield 'with envelopes mapping' => [ + 'message' => $this->serializer->serialize( + new FailureEnvelope( + new IdEnvelope( + new Message('foo', 'bar'), + 'test id', + ), + ['failure-1' => ['a', 'b', 'c']], + ), + ), + ]; + } + + #[ParamProviders('provideConsume')] + #[Tag('queue_consume')] + public function benchConsume(array $params): void + { + $this->adapter->message = $params['message']; + $this->queue->run(); + } +} diff --git a/tests/Benchmark/Support/VoidAdapter.php b/tests/Benchmark/Support/VoidAdapter.php new file mode 100644 index 00000000..edd927aa --- /dev/null +++ b/tests/Benchmark/Support/VoidAdapter.php @@ -0,0 +1,52 @@ +serializer->unserialize($this->message)); + } + + public function status(int|string $id): JobStatus + { + throw new InvalidArgumentException(); + } + + public function push(MessageInterface $message): MessageInterface + { + $this->serializer->serialize($message); + + return new IdEnvelope($message, 1); + } + + public function subscribe(callable $handlerCallback): void + { + throw new RuntimeException('Method is not implemented'); + } + + public function withChannel(string $channel): AdapterInterface + { + throw new RuntimeException('Method is not implemented'); + } +} diff --git a/tests/docker/php/Dockerfile b/tests/docker/php/Dockerfile index a2051c02..cd3e6270 100644 --- a/tests/docker/php/Dockerfile +++ b/tests/docker/php/Dockerfile @@ -1,17 +1,19 @@ # Important! Do not use this image in production! ARG PHP_VERSION -FROM --platform=linux/amd64 php:${PHP_VERSION}-cli-alpine +FROM php:${PHP_VERSION}-cli-alpine -RUN apk add git autoconf g++ make linux-headers +RUN apk add git autoconf g++ make linux-headers && \ + docker-php-ext-install pcntl && \ + pecl install xdebug pcov && \ + docker-php-ext-enable xdebug pcov -RUN docker-php-ext-install pcntl -RUN pecl install xdebug pcov -RUN docker-php-ext-enable xdebug pcov +ADD ./tests/docker/php/php.ini /usr/local/etc/php/conf.d/40-custom.ini COPY --from=composer:latest /usr/bin/composer /usr/local/bin/composer ENV COMPOSER_ALLOW_SUPERUSER 1 WORKDIR /app +RUN git config --global --add safe.directory /app ENTRYPOINT ["sh", "tests/docker/php/entrypoint.sh"] CMD ["sleep", "infinity"] diff --git a/tests/docker/php/php.ini b/tests/docker/php/php.ini new file mode 100644 index 00000000..18fdd3e5 --- /dev/null +++ b/tests/docker/php/php.ini @@ -0,0 +1,2 @@ +opcache.enable=1 +opcache.enable_cli=1