Edit

Editing: RetryMiddlewareV2.php

<?php
namespace Aws;

use Aws\Exception\AwsException;
use Aws\Retry\ConfigurationInterface;
use Aws\Retry\QuotaManager;
use Aws\Retry\RateLimiter;
use Aws\Retry\RetryHelperTrait;
use GuzzleHttp\Exception\RequestException;
use GuzzleHttp\Promise;
use Psr\Http\Message\RequestInterface;

/**
 * Middleware that retries failures. V2 implementation that supports 'standard'
 * and 'adaptive' modes.
 *
 * @internal
 */
class RetryMiddlewareV2
{
    use RetryHelperTrait;

private static $standardThrottlingErrors = [
        'Throttling'                                => true,
        'ThrottlingException'                       => true,
        'ThrottledException'                        => true,
        'RequestThrottledException'                 => true,
        'TooManyRequestsException'                  => true,
        'ProvisionedThroughputExceededException'    => true,
        'TransactionInProgressException'            => true,
        'RequestLimitExceeded'                      => true,
        'BandwidthLimitExceeded'                    => true,
        'LimitExceededException'                    => true,
        'RequestThrottled'                          => true,
        'SlowDown'                                  => true,
        'PriorRequestNotComplete'                   => true,
        'EC2ThrottledException'                     => true,
    ];

private static $standardTransientErrors = [
        'RequestTimeout'            => true,
        'RequestTimeoutException'   => true,
    ];

private static $standardTransientStatusCodes = [
        500 => true,
        502 => true,
        503 => true,
        504 => true,
    ];

private $collectStats;
    private $decider;
    private $delayer;
    private $maxAttempts;
    private $maxBackoff;
    private $mode;
    private $nextHandler;
    private $options;
    private $quotaManager;
    private $rateLimiter;

public static function wrap($config, $options)
    {
        return function (callable $handler) use (
            $config,
            $options
        ) {
            return new static(
                $config,
                $handler,
                $options
            );
        };
    }

public static function createDefaultDecider(
        QuotaManager $quotaManager,
        $maxAttempts = 3,
        $options = []
    ) {
        $retryCurlErrors = [];
        if (extension_loaded('curl')) {
            $retryCurlErrors[CURLE_RECV_ERROR] = true;
        }

return function(
            $attempts,
            CommandInterface $command,
            $result
        ) use ($options, $quotaManager, $retryCurlErrors, $maxAttempts) {

// Release retry tokens back to quota on a successful result
            $quotaManager->releaseToQuota($result);

// Allow command-level option to override this value
            // # of attempts = # of retries + 1
            $maxAttempts = (null !== $command['@retries'])
                ? $command['@retries'] + 1
                : $maxAttempts;

$isRetryable = self::isRetryable(
                $result,
                $retryCurlErrors,
                $options
            );

if ($isRetryable) {

// Retrieve retry tokens and check if quota has been exceeded
                if (!$quotaManager->hasRetryQuota($result)) {
                    return false;
                }

if ($attempts >= $maxAttempts) {
                    if (!empty($result) && $result instanceof AwsException) {
                        $result->setMaxRetriesExceeded();
                    }
                    return false;
                }
            }

return $isRetryable;
        };
    }

public function __construct(
        ConfigurationInterface $config,
        callable $handler,
        $options = []
    ) {
        $this->options = $options;
        $this->maxAttempts = $config->getMaxAttempts();
        $this->mode = $config->getMode();
        $this->nextHandler = $handler;
        $this->quotaManager = new QuotaManager();

$this->maxBackoff = isset($options['max_backoff'])
            ? $options['max_backoff']
            : 20000;

$this->collectStats = isset($options['collect_stats'])
            ? (bool) $options['collect_stats']
            : false;

$this->decider = isset($options['decider'])
            ? $options['decider']
            : self::createDefaultDecider(
                $this->quotaManager,
                $this->maxAttempts,
                $options
            );

$this->delayer = isset($options['delayer'])
            ? $options['delayer']
            : function ($attempts) {
                return $this->exponentialDelayWithJitter($attempts);
            };

if ($this->mode === 'adaptive') {
            $this->rateLimiter = isset($options['rate_limiter'])
                ? $options['rate_limiter']
                : new RateLimiter();
        }
    }

public function __invoke(CommandInterface $cmd, RequestInterface $req)
    {
        $decider = $this->decider;
        $delayer = $this->delayer;
        $handler = $this->nextHandler;

$attempts = 1;
        $monitoringEvents = [];
        $requestStats = [];

$req = $this->addRetryHeader($req, 0, 0);

$callback = function ($value) use (
            $handler,
            $cmd,
            $req,
            $decider,
            $delayer,
            &$attempts,
            &$requestStats,
            &$monitoringEvents,
            &$callback
        ) {
            if ($this->mode === 'adaptive') {
                $this->rateLimiter->updateSendingRate($this->isThrottlingError($value));
            }

$this->updateHttpStats($value, $requestStats);

if ($value instanceof MonitoringEventsInterface) {
                $reversedEvents = array_reverse($monitoringEvents);
                $monitoringEvents = array_merge($monitoringEvents, $value->getMonitoringEvents());
                foreach ($reversedEvents as $event) {
                    $value->prependMonitoringEvent($event);
                }
            }
            if ($value instanceof \Exception || $value instanceof \Throwable) {
                if (!$decider($attempts, $cmd, $value)) {
                    return Promise\Create::rejectionFor(
                        $this->bindStatsToReturn($value, $requestStats)
                    );
                }
            } elseif ($value instanceof ResultInterface
                && !$decider($attempts, $cmd, $value)
            ) {
                return $this->bindStatsToReturn($value, $requestStats);
            }

$delayBy = $delayer($attempts++);
            $cmd['@http']['delay'] = $delayBy;
            if ($this->collectStats) {
                $this->updateStats($attempts - 1, $delayBy, $requestStats);
            }

// Update retry header with retry count and delayBy
            $req = $this->addRetryHeader($req, $attempts - 1, $delayBy);

// Get token from rate limiter, which will sleep if necessary
            if ($this->mode === 'adaptive') {
                $this->rateLimiter->getSendToken();
            }

return $handler($cmd, $req)->then($callback, $callback);
        };

// Get token from rate limiter, which will sleep if necessary
        if ($this->mode === 'adaptive') {
            $this->rateLimiter->getSendToken();
        }

return $handler($cmd, $req)->then($callback, $callback);
    }

/**
     * Amount of milliseconds to delay as a function of attempt number
     *
     * @param $attempts
     * @return mixed
     */
    public function exponentialDelayWithJitter($attempts)
    {
        $rand = mt_rand() / mt_getrandmax();
        return min(1000 * $rand * pow(2, $attempts) , $this->maxBackoff);
    }

private static function isRetryable(
        $result,
        $retryCurlErrors,
        $options = []
    ) {
        $errorCodes = self::$standardThrottlingErrors + self::$standardTransientErrors;
        if (!empty($options['transient_error_codes'])
            && is_array($options['transient_error_codes'])
        ) {
            foreach($options['transient_error_codes'] as $code) {
                $errorCodes[$code] = true;
            }
        }
        if (!empty($options['throttling_error_codes'])
            && is_array($options['throttling_error_codes'])
        ) {
            foreach($options['throttling_error_codes'] as $code) {
                $errorCodes[$code] = true;
            }
        }

$statusCodes = self::$standardTransientStatusCodes;
        if (!empty($options['status_codes'])
            && is_array($options['status_codes'])
        ) {
            foreach($options['status_codes'] as $code) {
                $statusCodes[$code] = true;
            }
        }

if (!empty($options['curl_errors'])
            && is_array($options['curl_errors'])
        ) {
            foreach($options['curl_errors'] as $code) {
                $retryCurlErrors[$code] = true;
            }
        }

if ($result instanceof \Exception || $result instanceof \Throwable) {
            $isError = true;
        } else {
            $isError = false;
        }

if (!$isError) {
            if (!isset($result['@metadata']['statusCode'])) {
                return false;
            }
            return isset($statusCodes[$result['@metadata']['statusCode']]);
        }

if (!($result instanceof AwsException)) {
            return false;
        }

if ($result->isConnectionError()) {
            return true;
        }

if (!empty($errorCodes[$result->getAwsErrorCode()])) {
            return true;
        }

if (!empty($statusCodes[$result->getStatusCode()])) {
            return true;
        }

if (count($retryCurlErrors)
            && ($previous = $result->getPrevious())
            && $previous instanceof RequestException
        ) {
            if (method_exists($previous, 'getHandlerContext')) {
                $context = $previous->getHandlerContext();
                return !empty($context['errno'])
                    && isset($retryCurlErrors[$context['errno']]);
            }

$message = $previous->getMessage();
            foreach (array_keys($retryCurlErrors) as $curlError) {
                if (strpos($message, 'cURL error ' . $curlError . ':') === 0) {
                    return true;
                }
            }
        }

// Check error shape for the retryable trait
        if (!empty($errorShape = $result->getAwsErrorShape())) {
            $definition = $errorShape->toArray();
            if (!empty($definition['retryable'])) {
                return true;
            }
        }

return false;
    }

private function isThrottlingError($result)
    {
        if ($result instanceof AwsException) {
            // Check pre-defined throttling errors
            $throttlingErrors = self::$standardThrottlingErrors;
            if (!empty($this->options['throttling_error_codes'])
                && is_array($this->options['throttling_error_codes'])
            ) {
                foreach($this->options['throttling_error_codes'] as $code) {
                    $throttlingErrors[$code] = true;
                }
            }
            if (!empty($result->getAwsErrorCode())
                && !empty($throttlingErrors[$result->getAwsErrorCode()])
            ) {
                return true;
            }

// Check error shape for the throttling trait
            if (!empty($errorShape = $result->getAwsErrorShape())) {
                $definition = $errorShape->toArray();
                if (!empty($definition['retryable']['throttling'])) {
                    return true;
                }
            }
        }

return false;
    }
}

Back