r2 - range over http request

r2 is a proof of concept for the Go 1.22 range functions and provides a simple and easy-to-use interface for sending HTTP requests with retries.

Quick Start

Install

go get github.com/miyamo2/r2

Setup GOEXPERIMENT

Important

If your Go project is Go 1.23 or higher, this section is not necessary.

go env -w GOEXPERIMENT=rangefunc

Simple Usage

url := "http://example.com"
ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
opts := []r2.Option{
	r2.WithMaxRequestAttempts(3),
	r2.WithPeriod(time.Second),
}
for res, err := range r2.Get(ctx, url, opts...) {
	if err != nil {
		slog.WarnContext(ctx, "something happened.", slog.Any("error", err))
		// Note: Even if continue is used, the iterator could be terminated.
		// Likewise, if break is used, the request could be re-executed in the background once more.
		continue
	}
	if res == nil {
		slog.WarnContext(ctx, "response is nil")
		continue
	}
	if res.StatusCode != http.StatusOK {
		slog.WarnContext(ctx, "unexpected status code.", slog.Int("expect", http.StatusOK), slog.Int("got", res.StatusCode))
		continue
	}

	buf, err := io.ReadAll(res.Body)
	if err != nil {
		slog.ErrorContext(ctx, "failed to read response body.", slog.Any("error", err))
		continue
	}
	slog.InfoContext(ctx, "response", slog.String("response", string(buf)))
	// There is no need to close the response body yourself as auto closing is enabled by default.
}

vs 'github.com/avast/retry-go'

url := "http://example.com"
var buf []byte

ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()

type ErrTooManyRequests struct{
	error
	RetryAfter time.Duration
}

opts := []retry.Option{
	retry.Attempts(3),
	retry.Context(ctx),
	// In r2, the delay is calculated with the backoff and jitter by default. 
	// And, if 429 Too Many Requests are returned, the delay is set based on the Retry-After.
	retry.DelayType(
		func(n uint, err error, config *Config) time.Duration {
			if err != nil {
				var errTooManyRequests ErrTooManyRequests
				if errors.As(err, &ErrTooManyRequests) {
					if ErrTooManyRequests.RetryAfter != 0 {
						return ErrTooManyRequests.RetryAfter
					}
				}
			}
			return retry.BackOffDelay(n, err, config)
		}),
}

// In r2, the timeout period per request can be specified with the `WithPeriod` option.
client := http.Client{
	Timeout: time.Second,
}

err := retry.Do(
	func() error {
		res, err := client.Get(url)
		if err != nil {
			return err
		}
		if res == nil {
			return fmt.Errorf("response is nil")
		}
		if res.StatusCode == http.StatusTooManyRequests {
			retryAfter := res.Header.Get("Retry-After")
			if retryAfter != "" {
				retryAfterDuration, err := time.ParseDuration(retryAfter)
				if err != nil {
					return &ErrTooManyRequests{error: fmt.Errorf("429: too many requests")}
				}
				return &ErrTooManyRequests{error: fmt.Errorf("429: too many requests"), RetryAfter: retryAfterDuration}
			}
			return &ErrTooManyRequests{error: fmt.Errorf("429: too many requests")}
		}
		if res.StatusCode >= http.StatusBadRequest && res.StatusCode < http.StatusInternalServerError {
			// In r2, client errors other than TooManyRequests are excluded from retries by default.
			return nil
		}
		if res.StatusCode >= http.StatusInternalServerError {
			// In r2, automatically retry if the server error response is returned by default.
			return fmt.Errorf("5xx: server error response")
		}

		if res.StatusCode != http.StatusOK {
			return fmt.Errorf("unexpected status code: expected %d, got %d", http.StatusOK, res.StatusCode)
		}

		// In r2, the response body is automatically closed by default.
		defer res.Body.Close()
		buf, err = io.ReadAll(res.Body)
		if err != nil {
			slog.ErrorContext(ctx, "failed to read response body.", slog.Any("error", err))
			return err
		}
		return nil
	},
	opts...,
)

if err != nil {
	// handle error
}

slog.InfoContext(ctx, "response", slog.String("response", string(buf)))

Features

Feature	Description
Get	Send HTTP Get requests until the termination condition is satisfied.
Head	Send HTTP Head requests until the termination condition is satisfied.
Post	Send HTTP Post requests until the termination condition is satisfied.
Put	Send HTTP Put requests until the termination condition is satisfied.
Patch	Send HTTP Patch requests until the termination condition is satisfied.
Delete	Send HTTP Delete requests until the termination condition is satisfied.
PostForm	Send HTTP Post requests with form until the termination condition is satisfied.
Do	Send HTTP requests with the given method until the termination condition is satisfied.

Get

ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
opts := []r2.Option{
	r2.WithMaxRequestAttempts(3),
	r2.WithPeriod(time.Second),
}
for res, err := range r2.Get(ctx, "https://example.com", opts...) {
	// do something
}

Head

ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
opts := []r2.Option{
	r2.WithMaxRequestAttempts(3),
	r2.WithPeriod(time.Second),
}
for res, err := range r2.Head(ctx, "https://example.com", opts...) {
	// do something
}

Post

ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
opts := []r2.Option{
	r2.WithMaxRequestAttempts(3),
	r2.WithPeriod(time.Second),
	r2.WithContentType(r2.ContentTypeApplicationJson),
}
body := bytes.NewBuffer([]byte(`{"foo": "bar"}`))
for res, err := range r2.Post(ctx, "https://example.com", body, opts...) {
	// do something
}

Put

ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
opts := []r2.Option{
	r2.WithMaxRequestAttempts(3),
	r2.WithPeriod(time.Second),
	r2.WithContentType(r2.ContentTypeApplicationJson),
}
body := bytes.NewBuffer([]byte(`{"foo": "bar"}`))
for res, err := range r2.Put(ctx, "https://example.com", body, opts...) {
	// do something
}

Patch

ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
opts := []r2.Option{
	r2.WithMaxRequestAttempts(3),
	r2.WithPeriod(time.Second),
	r2.WithContentType(r2.ContentTypeApplicationJson),
}
body := bytes.NewBuffer([]byte(`{"foo": "bar"}`))
for res, err := range r2.Patch(ctx, "https://example.com", body, opts...) {
	// do something
}

Delete

ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
opts := []r2.Option{
	r2.WithMaxRequestAttempts(3),
	r2.WithPeriod(time.Second),
	r2.WithContentType(r2.ContentTypeApplicationJson),
}
body := bytes.NewBuffer([]byte(`{"foo": "bar"}`))
for res, err := range r2.Delete(ctx, "https://example.com", body, opts...) {
	// do something
}

PostForm

ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
opts := []r2.Option{
	r2.WithMaxRequestAttempts(3),
	r2.WithPeriod(time.Second),
	r2.WithContentType(r2.ContentTypeApplicationJson),
}
form := url.Values{"foo": []string{"bar"}}
for res, err := range r2.Post(ctx, "https://example.com", form, opts...) {
	// do something
}

Do

ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
opts := []r2.Option{
	r2.WithMaxRequestAttempts(3),
	r2.WithPeriod(time.Second),
	r2.WithContentType(r2.ContentTypeApplicationJson),
}
body := bytes.NewBuffer([]byte(`{"foo": "bar"}`))
for res, err := range r2.Do(ctx, http,MethodPost, "https://example.com", body, opts...) {
	// do something
}

Termination Conditions

• Request succeeded and no termination condition is specified by WithTerminateIf.
• Condition that specified in WithTerminateIf is satisfied.
• Response status code is a 4xx Client Error other than 429: Too Many Request.
• Maximum number of requests specified in WithMaxRequestAttempts is reached.
• Exceeds the deadline for the context.Context passed in the argument.
• When the for range loop is interrupted by break.

Options

r2 provides the following request options

Option	Description	Default
WithMaxRequestAttempts	The maximum number of requests to be performed. If less than or equal to 0 is specified, maximum number of requests does not apply.	0
WithPeriod	The timeout period of the per request. If less than or equal to 0 is specified, the timeout period does not apply. If http.Client.Timeout is set, the shorter one is applied.	0
WithInterval	The interval between next request. By default, the interval is calculated by the exponential backoff and jitter. If response status code is 429(Too Many Request), the interval conforms to 'Retry-After' header.	0
WithTerminateIf	The termination condition of the iterator that references the response.	nil
WithHttpClient	The client to use for requests.	http.DefaultClient
WithHeader	The custom http headers for the request.	http.Header(blank)
WithContentType	The 'Content-Type' for the request.	''
WithAspect	The behavior to the pre-request/post-request.	-
WithAutoCloseResponseBody	Whether the response body is automatically closed. By default, this setting is enabled.	true

WithMaxRequestAttempts

ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
opts := []r2.Option{
	r2.WithMaxRequestAttempts(3),
}
for res, err := range r2.Get(ctx, "https://example.com", opts...) {
	// do something
}

WithPeriod

ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
opts := []r2.Option{
	r2.WithPeriod(time.Second),
}
for res, err := range r2.Get(ctx, "https://example.com", opts...) {
	// do something
}

WithInterval

ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
opts := []r2.Option{
	r2.WithInterval(time.Second),
}
for res, err := range r2.Get(ctx, "https://example.com", opts...) {
	// do something
}

WithTerminateIf

ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
opts := []r2.Option{
	r2.WithTerminateIf(func(res *http.Response, _ error) bool {
		myHeader := res.Header.Get("X-My-Header")
		return len(myHeader) > 0
	}),
}
for res, err := range r2.Get(ctx, "https://example.com", opts...) {
	// do something
}

WithHttpClient

ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
var myHttpClient *http.Client = getMyHttpClient()
opts := []r2.Option{
	r2.WithHttpClient(myHttpClient),
}
for res, err := range r2.Get(ctx, "https://example.com", opts...) {
	// do something
}

WithHeader

ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
opts := []r2.Option{
	r2.WithHeader(http.Header{"X-My-Header": []string{"my-value"}}),
}
for res, err := range r2.Get(ctx, "https://example.com", opts...) {
	// do something
}

WithContentType

ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
opts := []r2.Option{
	r2.WithContentType("application/json"),
}
for res, err := range r2.Get(ctx, "https://example.com", opts...) {
	// do something
}

WithAspect

ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
opts := []r2.Option{
    r2.WithAspect(func(req *http.Request, do func(req *http.Request) (*http.Response, error)) (*http.Response, error) {
        res, err := do(req)
        res.StatusCode += 1
        return res, err
    }),
}
for res, err := range r2.Get(ctx, "https://example.com", opts...) {
    // do something
}

WithAutoCloseResponseBody

ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
opts := []r2.Option{
    r2.WithAutoCloseResponseBody(true),
}
for res, err := range r2.Get(ctx, "https://example.com", opts...) {
    // do something
}

Advanced Usage

Read more advanced usages

For Contributors

Feel free to open a PR or an Issue.
However, you must promise to follow our Code of Conduct.

Tree

.
├ .doc/            # Documentation
├ .github/
│    └ workflows/  # GitHub Actions Workflow
├ internal/        # Internal Package; Shared with sub-packages.
└ tests/            
    ├ integration/ # Integration Test
    └ unit/        # Unit Test

Tasks

We recommend that this section be run with xc.

setup:deps

Install mockgen and golangci-lint.

go install go.uber.org/mock/mockgen@latest
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest

setup:goenv

Set GOEXPERIMENT to rangefunc if Go version is 1.22.

GOVER=$(go mod graph)
if [[ $GOVER == *"go@1.22"* ]]; then
  go env -w GOEXPERIMENT=rangefunc
fi

setup:mocks

Generate mock files.

go mod tidy
go generate ./...

lint

golangci-lint run --fix

test:unit

Run Unit Test

cd ./tests/unit
go test -v -coverpkg=github.com/miyamo2/r2 ./... -coverprofile=coverage.out 

test:integration

Run Integration Test

cd ./tests/integration
go test -v -coverpkg=github.com/miyamo2/r2 ./... -coverprofile=coverage.out 

License

r2 released under the MIT License