Skip to content

Commit

Permalink
Merge pull request #94 from eliashaeussler/feature/configurable-clien…
Browse files Browse the repository at this point in the history
…t-config

[FEATURE] Make crawler client configurable in default crawlers
  • Loading branch information
eliashaeussler authored Oct 2, 2022
2 parents f99cc69 + a172c0e commit 4ae6e80
Show file tree
Hide file tree
Showing 3 changed files with 64 additions and 4 deletions.
9 changes: 7 additions & 2 deletions src/Crawler/ConcurrentCrawler.php
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@
* @extends AbstractConfigurableCrawler<array{
* concurrency: int,
* request_method: string,
* request_headers: array<string, string>
* request_headers: array<string, string>,
* client_config: array<string, mixed>
* }>
*/
class ConcurrentCrawler extends AbstractConfigurableCrawler
Expand All @@ -50,13 +51,17 @@ class ConcurrentCrawler extends AbstractConfigurableCrawler
'concurrency' => 5,
'request_method' => 'HEAD',
'request_headers' => [],
'client_config' => [],
];

protected readonly ClientInterface $client;

public function __construct(
array $options = [],
protected readonly ClientInterface $client = new Client(),
ClientInterface $client = null,
) {
parent::__construct($options);
$this->client = null !== $client ? $client : new Client($this->options['client_config']);
}

public function crawl(array $urls): Result\CacheWarmupResult
Expand Down
3 changes: 1 addition & 2 deletions src/Crawler/OutputtingCrawler.php
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@

use EliasHaeussler\CacheWarmup\Exception;
use EliasHaeussler\CacheWarmup\Result;
use GuzzleHttp\Client;
use GuzzleHttp\ClientInterface;
use Psr\Http\Message;
use Symfony\Component\Console;
Expand All @@ -49,7 +48,7 @@ class OutputtingCrawler extends ConcurrentCrawler implements VerboseCrawlerInter

public function __construct(
array $options = [],
ClientInterface $client = new Client(),
ClientInterface $client = null,
) {
parent::__construct($options, $client);
Console\Helper\ProgressBar::setFormatDefinition('cache-warmup', self::PROGRESS_BAR_FORMAT);
Expand Down
56 changes: 56 additions & 0 deletions tests/Unit/Crawler/ConcurrentCrawlerTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
use EliasHaeussler\CacheWarmup\Crawler;
use EliasHaeussler\CacheWarmup\Result;
use EliasHaeussler\CacheWarmup\Tests;
use GuzzleHttp\Client;
use GuzzleHttp\Handler;
use GuzzleHttp\Psr7;
use PHPUnit\Framework;

Expand All @@ -39,6 +41,52 @@ final class ConcurrentCrawlerTest extends Framework\TestCase
{
use Tests\Unit\CacheWarmupResultProcessorTrait;

private bool $handlerPassed;
private Crawler\ConcurrentCrawler $subject;

protected function setUp(): void
{
$this->handlerPassed = false;
$this->subject = new Crawler\ConcurrentCrawler([
'client_config' => [
'handler' => $this->createMockHandler(),
],
]);
}

/**
* @test
*/
public function constructorInstantiatesClientWithGivenClientConfig(): void
{
self::assertFalse($this->handlerPassed);

$this->subject->crawl([new Psr7\Uri('https://www.example.org')]);

self::assertTrue($this->handlerPassed);
}

/**
* @test
*/
public function constructorIgnoresGivenClientConfigIfInstantiatedClientIsPassed(): void
{
$this->subject = new Crawler\ConcurrentCrawler(
[
'client_config' => [
'handler' => $this->createMockHandler(),
],
],
new Client(),
);

self::assertFalse($this->handlerPassed);

$this->subject->crawl([new Psr7\Uri('https://www.example.org')]);

self::assertFalse($this->handlerPassed);
}

/**
* @test
*/
Expand Down Expand Up @@ -90,4 +138,12 @@ public function crawlHandlesFailedRequestsOfAllGivenUrls(): void
self::assertSame($urls, $this->getProcessedUrlsFromCacheWarmupResult($result, Result\CrawlingState::Failed));
self::assertSame([], $result->getSuccessful());
}

private function createMockHandler(): Handler\MockHandler
{
return new Handler\MockHandler(
[new Psr7\Response()],
fn () => $this->handlerPassed = true,
);
}
}

0 comments on commit 4ae6e80

Please sign in to comment.