Merge pull request #2 from chrisrhymes/feature/rate-limiter

Add rate limiter

README.md

@@ -2,6 +2,13 @@
 
 A package that will check for broken links in the HTML of a specified model's fields.
 
+## Contents
+
+- [Getting Started](#getting-started)
+- [Usage](#usage)
+- [Rate Limiting](#rate-limiting)
+- [Tests](#tests)
+
 ## Getting Started
 
 ```bash
@@ -32,9 +39,11 @@ class Post extends Model
 }
 ```
 
-## Publish the config (optional)
+### Publish the config (optional)
 
-By default, the timeout for link checks is set to 10 seconds. If you wish to change this then publish the configuration file and update the values.
+By default, the timeout for link checks is set to 10 seconds. There are also settings for the rate limiting.
+
+If you wish to change this then publish the configuration file and update the values.
 
 ```bash
 php artisan vendor:publish --provider="ChrisRhymes\LinkChecker\ServiceProvider"
@@ -80,6 +89,14 @@ $post->brokenLinks[0]->broken_link; // The link that is broken
 $post->brokenLinks[0]->exception_message; // The optional exception message
 ```
 
+## Rate Limiting
+
+In order to reduce the amount of requests sent to a domain at a time, this package has rate limiting enabled.
+
+The configuration file allows you to set the `rate_limit` to set how many requests can be sent to a single domain within a minute. The default is set to 5, so adjust as required for your circumstances.
+
+The configuration file also allows you to set the `retry_until` so the job will be retried until the time limit (in munites) is reached.
+
 ## Tests
 
 The tests are built with [Pest](https://pestphp.com/).

config/link-checker.php

@@ -8,4 +8,15 @@
      * The time to wait for a response
      */
     'timeout' => 10,
+
+    /**
+     * The rate limit to check broken links per minute
+     * This is applied on a per domain basis
+     */
+    'rate_limit' => 5,
+
+    /**
+     * Retry the CheckLinkFailed job until a specified time (in minutes)
+     */
+    'retry_until' => 10,
 ];

src/Jobs/CheckLinkFailed.php

@@ -8,17 +8,25 @@
 use Illuminate\Database\Eloquent\Model;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\Middleware\RateLimited;
 use Illuminate\Queue\SerializesModels;
 use Illuminate\Support\Facades\Http;
 
 class CheckLinkFailed implements ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
-    private string $link;
+    public string $link;
 
     private Model $model;
 
+    /**
+     * The number of seconds the job can run before timing out.
+     *
+     * @var int
+     */
+    public $timeout = 30;
+
     /**
      * Create a new job instance.
      *
@@ -30,13 +38,44 @@ public function __construct(Model $model, string $link)
         $this->model = $model;
     }
 
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array
+     */
+    public function middleware()
+    {
+        return [new RateLimited('link-checker')];
+    }
+
+    /**
+     * Determine the time at which the job should timeout.
+     *
+     * @return \DateTime
+     */
+    public function retryUntil()
+    {
+        return now()->addMinutes(config('link-checker.retry_until', 10));
+    }
+
     /**
      * Execute the job.
      *
      * @return void
      */
     public function handle()
     {
+        // Prevent unnecessary request for empty links
+        if (empty($this->link)) {
+            $this->model->brokenLinks()
+                ->create([
+                    'broken_link' => $this->link,
+                    'exception_message' => 'Empty link',
+                ]);
+
+            return;
+        }
+
         try {
             $failed = Http::timeout(config('link-checker.timeout', 10))
                 ->get($this->link)->failed();

src/ServiceProvider.php

@@ -2,6 +2,8 @@
 
 namespace ChrisRhymes\LinkChecker;
 
+use Illuminate\Cache\RateLimiting\Limit;
+use Illuminate\Support\Facades\RateLimiter;
 use Illuminate\Support\ServiceProvider as SupportServiceProvider;
 
 class ServiceProvider extends SupportServiceProvider
@@ -13,6 +15,13 @@ public function boot()
         $this->publishes([
             __DIR__.'/../config/link-checker.php' => config_path('link-checker.php'),
         ]);
+
+        $this->mergeConfigFrom(__DIR__.'/../config/link-checker.php', 'link-checker');
+
+        RateLimiter::for('link-checker', function ($job) {
+            return Limit::perMinute(config('link-checker.rate_limit', 5))
+                ->by(parse_url($job->link, PHP_URL_HOST));
+        });
     }
 
     public function register()

tests/Feature/CheckForBrokenLinksTest.php

@@ -2,6 +2,7 @@
 
 use ChrisRhymes\LinkChecker\Facades\LinkChecker;
 use ChrisRhymes\LinkChecker\Jobs\CheckModelForBrokenLinks;
+use ChrisRhymes\LinkChecker\Models\BrokenLink;
 use ChrisRhymes\LinkChecker\Test\Models\Post;
 use Illuminate\Foundation\Testing\RefreshDatabase;
 use Illuminate\Support\Facades\Http;
@@ -68,3 +69,18 @@
 
     Queue::assertPushed(CheckModelForBrokenLinks::class);
 });
+
+it('handles empty links and prevents unnecessary requests', function () {
+    $post = Post::factory()
+        ->create([
+            'content' => '<a href="">Empty link</a><a href="">Empty link</a>',
+        ]);
+
+    CheckModelForBrokenLinks::dispatch($post, ['content']);
+
+    Http::assertNothingSent();
+
+    expect(BrokenLink::get())
+        ->toHaveCount(2)
+        ->first()->exception_message->toBe('Empty link');
+});

tests/Feature/RateLimitTest.php

@@ -0,0 +1,35 @@
+<?php
+
+use ChrisRhymes\LinkChecker\Jobs\CheckModelForBrokenLinks;
+use ChrisRhymes\LinkChecker\Models\BrokenLink;
+use ChrisRhymes\LinkChecker\Test\Models\Post;
+use Illuminate\Foundation\Testing\RefreshDatabase;
+use Illuminate\Support\Facades\Config;
+use Illuminate\Support\Facades\Http;
+
+uses(RefreshDatabase::class);
+
+beforeEach(function () {
+    Http::fake([
+        '*' => Http::response(null, 404),
+    ]);
+});
+
+it('triggers the rate limit and only finds one broken link', function () {
+    Config::set('link-checker.rate_limit', 1);
+
+    $post = Post::factory()
+        ->create([
+            'content' => '
+                <a href="https://this-is-broken.com/test1">Broken link</a>
+                <p>Some other content here</p>
+                <a href="https://this-is-broken.com/test2">Broken link</a>
+                <a href="https://this-is-broken.com/test3">Broken link</a>',
+        ]);
+
+    CheckModelForBrokenLinks::dispatch($post, ['content']);
+
+    expect(BrokenLink::get())
+        ->toHaveCount(1)
+        ->first()->broken_link->toBe('https://this-is-broken.com/test1');
+});