pax_global_header00006660000000000000000000000064146473272100014520gustar00rootroot0000000000000052 comment=8e6cec3089ffc70fa060d9aac509e2263a9bab94 golang-github-lestrrat-go-httprc-1.0.6/000077500000000000000000000000001464732721000200165ustar00rootroot00000000000000golang-github-lestrrat-go-httprc-1.0.6/.github/000077500000000000000000000000001464732721000213565ustar00rootroot00000000000000golang-github-lestrrat-go-httprc-1.0.6/.github/workflows/000077500000000000000000000000001464732721000234135ustar00rootroot00000000000000golang-github-lestrrat-go-httprc-1.0.6/.github/workflows/autodoc.yml000066400000000000000000000006531464732721000256000ustar00rootroot00000000000000name: Auto-Doc on: pull_request: branches: - main types: - closed jobs: autodoc: runs-on: ubuntu-latest name: "Run commands to generate documentation" if: github.event.pull_request.merged == true steps: - name: Checkout repository uses: actions/checkout@v4 - name: Process markdown files run: | find . -name '*.md' | xargs perl tools/autodoc.pl golang-github-lestrrat-go-httprc-1.0.6/.github/workflows/ci.yml000066400000000000000000000014131464732721000245300ustar00rootroot00000000000000name: CI on: [push, pull_request] jobs: build: runs-on: ubuntu-latest strategy: matrix: go: [ '1.19', '1.18' ] name: Go ${{ matrix.go }} test steps: - name: Checkout repository uses: actions/checkout@v4 - name: Check documentation generator run: | find . -name '*.md' | xargs env AUTODOC_DRYRUN=1 perl tools/autodoc.pl - name: Install Go stable version uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - name: Test run: go test -v -race -coverprofile=coverage.out -coverpkg=./... ./... - name: Upload code coverage to codecov if: matrix.go == '1.19' uses: codecov/codecov-action@v4 with: file: ./coverage.out golang-github-lestrrat-go-httprc-1.0.6/.github/workflows/lint.yml000066400000000000000000000006341464732721000251070ustar00rootroot00000000000000name: lint on: [push] jobs: golangci: name: lint runs-on: ubuntu-latest strategy: matrix: go: [1.19] steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - uses: golangci/golangci-lint-action@v4 with: version: v1.50.0 - name: Run go vet run: | go vet ./... golang-github-lestrrat-go-httprc-1.0.6/.gitignore000066400000000000000000000004151464732721000220060ustar00rootroot00000000000000# Binaries for programs and plugins *.exe *.exe~ *.dll *.so *.dylib # Test binary, built with `go test -c` *.test # Output of the go coverage tool, specifically when used with LiteIDE *.out # Dependency directories (remove the comment below to include it) # vendor/ golang-github-lestrrat-go-httprc-1.0.6/.golangci.yml000066400000000000000000000031461464732721000224060ustar00rootroot00000000000000run: linters-settings: govet: enable-all: true disable: - shadow - fieldalignment linters: enable-all: true disable: - cyclop - dupl - exhaustive - exhaustivestruct - exhaustruct - errorlint - funlen - gci - gochecknoglobals - gochecknoinits - gocognit - gocritic - gocyclo - godot - godox - goerr113 - gofumpt - golint #deprecated - gomnd - gosec - govet - interfacer # deprecated - ifshort - ireturn # No, I _LIKE_ returning interfaces - lll - maligned # deprecated - makezero - nakedret - nestif - nlreturn - paralleltest - scopelint # deprecated - tagliatelle - testpackage - thelper - varnamelen # short names are ok - wrapcheck - wsl issues: exclude-rules: # not needed - path: /*.go text: "ST1003: should not use underscores in package names" linters: - stylecheck - path: /*.go text: "don't use an underscore in package name" linters: - revive - path: /main.go linters: - errcheck - path: internal/codegen/codegen.go linters: - errcheck - path: /*_test.go linters: - errcheck - forcetypeassert - path: /*_example_test.go linters: - forbidigo - path: cmd/jwx/jwx.go linters: - forbidigo # Maximum issues count per one linter. Set to 0 to disable. Default is 50. max-issues-per-linter: 0 # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. max-same-issues: 0 golang-github-lestrrat-go-httprc-1.0.6/Changes000066400000000000000000000013001464732721000213030ustar00rootroot00000000000000Changes ======= v1.0.6 13 Jul 2024 * Apply #31 which reverts changes for v1.0.5. In some cases the changes for v1.0.5 were causing panics due to writes against closed channels (#25). v1.0.5 07 Mar 2024 * Avoid extra locks that forces cache.Get() to hang while cache.Refresh() is being called against a slow/flaky server. v1.0.4 19 Jul 2022 * Fix sloppy API breakage v1.0.3 19 Jul 2022 * Fix queue insertion in the middle of the queue (#7) v1.0.2 13 Jun 2022 * Properly release a lock when the fetch fails (#5) v1.0.1 29 Mar 2022 * Bump dependency for github.com/lestrrat-go/httpcc to v1.0.1 v1.0.0 29 Mar 2022 * Initial release, refactored out of github.com/lestrrat-go/jwx golang-github-lestrrat-go-httprc-1.0.6/LICENSE000066400000000000000000000020511464732721000210210ustar00rootroot00000000000000MIT License Copyright (c) 2022 lestrrat Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. golang-github-lestrrat-go-httprc-1.0.6/README.md000066400000000000000000000071361464732721000213040ustar00rootroot00000000000000# github.com/lestrrat-go/httprc ![](https://github.com/lestrrat-go/httprc/workflows/CI/badge.svg) [![Go Reference](https://pkg.go.dev/badge/github.com/lestrrat-go/httprc.svg)](https://pkg.go.dev/github.com/lestrrat-go/httprc) [![codecov.io](https://codecov.io/github/lestrrat-go/httprc/coverage.svg)](https://codecov.io/github/lestrrat-go/httprc) `httprc` is a HTTP "Refresh" Cache. Its aim is to cache a remote resource that can be fetched via HTTP, but keep the cached content up-to-date based on periodic refreshing. # SYNOPSIS ```go package httprc_test import ( "context" "fmt" "net/http" "net/http/httptest" "sync" "time" "github.com/lestrrat-go/httprc" ) const ( helloWorld = `Hello World!` goodbyeWorld = `Goodbye World!` ) func ExampleCache() { var mu sync.RWMutex msg := helloWorld srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set(`Cache-Control`, fmt.Sprintf(`max-age=%d`, 2)) w.WriteHeader(http.StatusOK) mu.RLock() fmt.Fprint(w, msg) mu.RUnlock() })) defer srv.Close() ctx, cancel := context.WithCancel(context.Background()) defer cancel() errSink := httprc.ErrSinkFunc(func(err error) { fmt.Printf("%s\n", err) }) c := httprc.NewCache(ctx, httprc.WithErrSink(errSink), httprc.WithRefreshWindow(time.Second), // force checks every second ) c.Register(srv.URL, httprc.WithHTTPClient(srv.Client()), // we need client with TLS settings httprc.WithMinRefreshInterval(time.Second), // allow max-age=1 (smallest) ) payload, err := c.Get(ctx, srv.URL) if err != nil { fmt.Printf("%s\n", err) return } if string(payload.([]byte)) != helloWorld { fmt.Printf("payload mismatch: %s\n", payload) return } mu.Lock() msg = goodbyeWorld mu.Unlock() time.Sleep(4 * time.Second) payload, err = c.Get(ctx, srv.URL) if err != nil { fmt.Printf("%s\n", err) return } if string(payload.([]byte)) != goodbyeWorld { fmt.Printf("payload mismatch: %s\n", payload) return } cancel() // OUTPUT: } ``` source: [httprc_example_test.go](https://github.com/lestrrat-go/jwx/blob/refs/heads/main/httprc_example_test.go) # Sequence Diagram ```mermaid sequenceDiagram autonumber actor User participant httprc.Cache participant httprc.Storage User->>httprc.Cache: Fetch URL `u` activate httprc.Storage httprc.Cache->>httprc.Storage: Fetch local cache for `u` alt Cache exists httprc.Storage-->httprc.Cache: Return local cache httprc.Cache-->>User: Return data Note over httprc.Storage: If the cache exists, there's nothing more to do.
The cached content will be updated periodically in httprc.Refresher deactivate httprc.Storage else Cache does not exist activate httprc.Fetcher httprc.Cache->>httprc.Fetcher: Fetch remote resource `u` httprc.Fetcher-->>httprc.Cache: Return fetched data deactivate httprc.Fetcher httprc.Cache-->>User: Return data httprc.Cache-)httprc.Refresher: Enqueue into auto-refresh queue activate httprc.Refresher loop Refresh Loop Note over httprc.Storage,httprc.Fetcher: Cached contents are updated synchronously httprc.Refresher->>httprc.Refresher: Wait until next refresh httprc.Refresher-->>httprc.Fetcher: Request fetch httprc.Fetcher->>httprc.Refresher: Return fetched data httprc.Refresher-->>httprc.Storage: Store new version in cache httprc.Refresher->>httprc.Refresher: Enqueue into auto-refresh queue (again) end deactivate httprc.Refresher end ``` golang-github-lestrrat-go-httprc-1.0.6/cache.go000066400000000000000000000117241464732721000214150ustar00rootroot00000000000000package httprc import ( "context" "fmt" "net/http" "sync" "time" ) // HTTPClient defines the interface required for the HTTP client // used within the fetcher. type HTTPClient interface { Get(string) (*http.Response, error) } // Cache represents a cache that stores resources locally, while // periodically refreshing the contents based on HTTP header values // and/or user-supplied hints. // // Refresh is performed _periodically_, and therefore the contents // are not kept up-to-date in real time. The interval between checks // for refreshes is called the refresh window. // // The default refresh window is 15 minutes. This means that if a // resource is fetched is at time T, and it is supposed to be // refreshed in 20 minutes, the next refresh for this resource will // happen at T+30 minutes (15+15 minutes). type Cache struct { mu sync.RWMutex queue *queue wl Whitelist } const defaultRefreshWindow = 15 * time.Minute // New creates a new Cache object. // // The context object in the argument controls the life-cycle of the // auto-refresh worker. If you cancel the `ctx`, then the automatic // refresh will stop working. // // Refresh will only be performed periodically where the interval between // refreshes are controlled by the `refresh window` variable. For example, // if the refresh window is every 5 minutes and the resource was queued // to be refreshed at 7 minutes, the resource will be refreshed after 10 // minutes (in 2 refresh window time). // // The refresh window can be configured by using `httprc.WithRefreshWindow` // option. If you want refreshes to be performed more often, provide a smaller // refresh window. If you specify a refresh window that is smaller than 1 // second, it will automatically be set to the default value, which is 15 // minutes. // // Internally the HTTP fetching is done using a pool of HTTP fetch // workers. The default number of workers is 3. You may change this // number by specifying the `httprc.WithFetcherWorkerCount` func NewCache(ctx context.Context, options ...CacheOption) *Cache { var refreshWindow time.Duration var errSink ErrSink var wl Whitelist var fetcherOptions []FetcherOption for _, option := range options { //nolint:forcetypeassert switch option.Ident() { case identRefreshWindow{}: refreshWindow = option.Value().(time.Duration) case identFetcherWorkerCount{}, identWhitelist{}: fetcherOptions = append(fetcherOptions, option) case identErrSink{}: errSink = option.Value().(ErrSink) } } if refreshWindow < time.Second { refreshWindow = defaultRefreshWindow } fetch := NewFetcher(ctx, fetcherOptions...) queue := newQueue(ctx, refreshWindow, fetch, errSink) return &Cache{ queue: queue, wl: wl, } } // Register configures a URL to be stored in the cache. // // For any given URL, the URL must be registered _BEFORE_ it is // accessed using `Get()` method. func (c *Cache) Register(u string, options ...RegisterOption) error { c.mu.Lock() defer c.mu.Unlock() if wl := c.wl; wl != nil { if !wl.IsAllowed(u) { return fmt.Errorf(`httprc.Cache: url %q has been rejected by whitelist`, u) } } return c.queue.Register(u, options...) } // Unregister removes the given URL `u` from the cache. // // Subsequent calls to `Get()` will fail until `u` is registered again. func (c *Cache) Unregister(u string) error { c.mu.Lock() defer c.mu.Unlock() return c.queue.Unregister(u) } // IsRegistered returns true if the given URL `u` has already been // registered in the cache. func (c *Cache) IsRegistered(u string) bool { c.mu.RLock() defer c.mu.RUnlock() return c.queue.IsRegistered(u) } // Refresh is identical to Get(), except it always fetches the // specified resource anew, and updates the cached content func (c *Cache) Refresh(ctx context.Context, u string) (interface{}, error) { return c.getOrFetch(ctx, u, true) } // Get returns the cached object. // // The context.Context argument is used to control the timeout for // synchronous fetches, when they need to happen. Synchronous fetches // will be performed when the cache does not contain the specified // resource. func (c *Cache) Get(ctx context.Context, u string) (interface{}, error) { return c.getOrFetch(ctx, u, false) } func (c *Cache) getOrFetch(ctx context.Context, u string, forceRefresh bool) (interface{}, error) { c.mu.RLock() e, ok := c.queue.getRegistered(u) if !ok { c.mu.RUnlock() return nil, fmt.Errorf(`url %q is not registered (did you make sure to call Register() first?)`, u) } c.mu.RUnlock() // Only one goroutine may enter this section. e.acquireSem() // has this entry been fetched? (but ignore and do a fetch // if forceRefresh is true) if forceRefresh || !e.hasBeenFetched() { if err := c.queue.fetchAndStore(ctx, e); err != nil { e.releaseSem() return nil, fmt.Errorf(`failed to fetch %q: %w`, u, err) } } e.releaseSem() e.mu.RLock() data := e.data e.mu.RUnlock() return data, nil } func (c *Cache) Snapshot() *Snapshot { c.mu.RLock() defer c.mu.RUnlock() return c.queue.snapshot() } golang-github-lestrrat-go-httprc-1.0.6/fetcher.go000066400000000000000000000071301464732721000217660ustar00rootroot00000000000000package httprc import ( "context" "fmt" "net/http" "sync" ) type fetchRequest struct { mu sync.RWMutex // client contains the HTTP Client that can be used to make a // request. By setting a custom *http.Client, you can for example // provide a custom http.Transport // // If not specified, http.DefaultClient will be used. client HTTPClient wl Whitelist // u contains the URL to be fetched url string // reply is a field that is only used by the internals of the fetcher // it is used to return the result of fetching reply chan *fetchResult } type fetchResult struct { mu sync.RWMutex res *http.Response err error } func (fr *fetchResult) reply(ctx context.Context, reply chan *fetchResult) error { select { case <-ctx.Done(): return ctx.Err() case reply <- fr: } close(reply) return nil } type fetcher struct { requests chan *fetchRequest } type Fetcher interface { Fetch(context.Context, string, ...FetchOption) (*http.Response, error) fetch(context.Context, *fetchRequest) (*http.Response, error) } func NewFetcher(ctx context.Context, options ...FetcherOption) Fetcher { var nworkers int var wl Whitelist for _, option := range options { //nolint:forcetypeassert switch option.Ident() { case identFetcherWorkerCount{}: nworkers = option.Value().(int) case identWhitelist{}: wl = option.Value().(Whitelist) } } if nworkers < 1 { nworkers = 3 } incoming := make(chan *fetchRequest) for i := 0; i < nworkers; i++ { go runFetchWorker(ctx, incoming, wl) } return &fetcher{ requests: incoming, } } func (f *fetcher) Fetch(ctx context.Context, u string, options ...FetchOption) (*http.Response, error) { var client HTTPClient var wl Whitelist for _, option := range options { //nolint:forcetypeassert switch option.Ident() { case identHTTPClient{}: client = option.Value().(HTTPClient) case identWhitelist{}: wl = option.Value().(Whitelist) } } req := fetchRequest{ client: client, url: u, wl: wl, } return f.fetch(ctx, &req) } // fetch (unexported) is the main fetching implemntation. // it allows the caller to reuse the same *fetchRequest object func (f *fetcher) fetch(ctx context.Context, req *fetchRequest) (*http.Response, error) { reply := make(chan *fetchResult, 1) req.mu.Lock() req.reply = reply req.mu.Unlock() // Send a request to the backend select { case <-ctx.Done(): return nil, ctx.Err() case f.requests <- req: } // wait until we get a reply select { case <-ctx.Done(): return nil, ctx.Err() case fr := <-reply: fr.mu.RLock() res := fr.res err := fr.err fr.mu.RUnlock() return res, err } } func runFetchWorker(ctx context.Context, incoming chan *fetchRequest, wl Whitelist) { LOOP: for { select { case <-ctx.Done(): break LOOP case req := <-incoming: req.mu.RLock() reply := req.reply client := req.client if client == nil { client = http.DefaultClient } url := req.url reqwl := req.wl req.mu.RUnlock() var wls []Whitelist for _, v := range []Whitelist{wl, reqwl} { if v != nil { wls = append(wls, v) } } if len(wls) > 0 { for _, wl := range wls { if !wl.IsAllowed(url) { r := &fetchResult{ err: fmt.Errorf(`fetching url %q rejected by whitelist`, url), } if err := r.reply(ctx, reply); err != nil { break LOOP } continue LOOP } } } // The body is handled by the consumer of the fetcher //nolint:bodyclose res, err := client.Get(url) r := &fetchResult{ res: res, err: err, } if err := r.reply(ctx, reply); err != nil { break LOOP } } } } golang-github-lestrrat-go-httprc-1.0.6/go.mod000066400000000000000000000006221464732721000211240ustar00rootroot00000000000000module github.com/lestrrat-go/httprc go 1.17 require ( github.com/lestrrat-go/httpcc v1.0.1 github.com/lestrrat-go/option v1.0.0 github.com/stretchr/testify v1.7.1 ) require ( github.com/davecgh/go-spew v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect ) retract v1.0.3 // Accidentally changed a published API golang-github-lestrrat-go-httprc-1.0.6/go.sum000066400000000000000000000026741464732721000211620ustar00rootroot00000000000000github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= github.com/lestrrat-go/option v1.0.0 h1:WqAWL8kh8VcSoD6xjSH34/1m8yxluXQbDeKNfvFeEO4= github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= golang-github-lestrrat-go-httprc-1.0.6/httprc.go000066400000000000000000000013021464732721000216450ustar00rootroot00000000000000//go:generate tools/genoptions.sh // Package httprc implements a cache for resources available // over http(s). Its aim is not only to cache these resources so // that it saves on HTTP roundtrips, but it also periodically // attempts to auto-refresh these resources once they are cached // based on the user-specified intervals and HTTP `Expires` and // `Cache-Control` headers, thus keeping the entries _relatively_ fresh. package httprc import "fmt" // RefreshError is the underlying error type that is sent to // the `httprc.ErrSink` objects type RefreshError struct { URL string Err error } func (re *RefreshError) Error() string { return fmt.Sprintf(`refresh error (%q): %s`, re.URL, re.Err) } golang-github-lestrrat-go-httprc-1.0.6/httprc_example_test.go000066400000000000000000000027401464732721000244260ustar00rootroot00000000000000package httprc_test import ( "context" "fmt" "net/http" "net/http/httptest" "sync" "time" "github.com/lestrrat-go/httprc" ) const ( helloWorld = `Hello World!` goodbyeWorld = `Goodbye World!` ) func ExampleCache() { var mu sync.RWMutex msg := helloWorld srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set(`Cache-Control`, fmt.Sprintf(`max-age=%d`, 2)) w.WriteHeader(http.StatusOK) mu.RLock() fmt.Fprint(w, msg) mu.RUnlock() })) defer srv.Close() ctx, cancel := context.WithCancel(context.Background()) defer cancel() errSink := httprc.ErrSinkFunc(func(err error) { fmt.Printf("%s\n", err) }) c := httprc.NewCache(ctx, httprc.WithErrSink(errSink), httprc.WithRefreshWindow(time.Second), // force checks every second ) c.Register(srv.URL, httprc.WithHTTPClient(srv.Client()), // we need client with TLS settings httprc.WithMinRefreshInterval(time.Second), // allow max-age=1 (smallest) ) payload, err := c.Get(ctx, srv.URL) if err != nil { fmt.Printf("%s\n", err) return } if string(payload.([]byte)) != helloWorld { fmt.Printf("payload mismatch: %s\n", payload) return } mu.Lock() msg = goodbyeWorld mu.Unlock() time.Sleep(4 * time.Second) payload, err = c.Get(ctx, srv.URL) if err != nil { fmt.Printf("%s\n", err) return } if string(payload.([]byte)) != goodbyeWorld { fmt.Printf("payload mismatch: %s\n", payload) return } cancel() // OUTPUT: } golang-github-lestrrat-go-httprc-1.0.6/httprc_test.go000066400000000000000000000043551464732721000227170ustar00rootroot00000000000000package httprc_test import ( "context" "fmt" "net/http" "net/http/httptest" "sync" "testing" "time" "github.com/lestrrat-go/httprc" "github.com/stretchr/testify/assert" ) type dummyErrSink struct { mu sync.RWMutex errors []error } func (d *dummyErrSink) Error(err error) { d.mu.Lock() defer d.mu.Unlock() d.errors = append(d.errors, err) } func (d *dummyErrSink) getErrors() []error { d.mu.RLock() defer d.mu.RUnlock() return d.errors } func TestCache(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var muCalled sync.Mutex var called int srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { select { case <-ctx.Done(): return default: } muCalled.Lock() called++ muCalled.Unlock() w.Header().Set(`Cache-Control`, fmt.Sprintf(`max-age=%d`, 3)) w.WriteHeader(http.StatusOK) })) errSink := &dummyErrSink{} c := httprc.NewCache(ctx, httprc.WithRefreshWindow(time.Second), httprc.WithErrSink(errSink), ) c.Register(srv.URL, httprc.WithHTTPClient(srv.Client()), httprc.WithMinRefreshInterval(time.Second)) if !assert.True(t, c.IsRegistered(srv.URL)) { return } for i := 0; i < 3; i++ { v, err := c.Get(ctx, srv.URL) if !assert.NoError(t, err, `c.Get should succeed`) { return } if !assert.IsType(t, []byte(nil), v, `c.Get should return []byte`) { return } } muCalled.Lock() if !assert.Equal(t, 1, called, `there should only be one fetch request`) { return } muCalled.Unlock() time.Sleep(4 * time.Second) for i := 0; i < 3; i++ { _, err := c.Get(ctx, srv.URL) if !assert.NoError(t, err, `c.Get should succeed`) { return } } muCalled.Lock() if !assert.Equal(t, 2, called, `there should only be one fetch request`) { return } muCalled.Unlock() if !assert.True(t, len(errSink.errors) == 0) { return } c.Register(srv.URL, httprc.WithHTTPClient(srv.Client()), httprc.WithMinRefreshInterval(time.Second), httprc.WithTransformer(httprc.TransformFunc(func(_ string, _ *http.Response) (interface{}, error) { return nil, fmt.Errorf(`dummy error`) })), ) _, _ = c.Get(ctx, srv.URL) time.Sleep(3 * time.Second) cancel() if !assert.True(t, len(errSink.getErrors()) > 0) { return } } golang-github-lestrrat-go-httprc-1.0.6/options.yaml000066400000000000000000000114041464732721000223750ustar00rootroot00000000000000package_name: httprc output: options_gen.go interfaces: - name: RegisterOption comment: | RegisterOption desribes options that can be passed to `(httprc.Cache).Register()` - name: CacheOption comment: | CacheOption desribes options that can be passed to `New()` - name: FetcherOption methods: - cacheOption comment: | FetcherOption describes options that can be passed to `(httprc.Fetcher).NewFetcher()` - name: FetchOption comment: | FetchOption describes options that can be passed to `(httprc.Fetcher).Fetch()` - name: FetchRegisterOption methods: - fetchOption - registerOption - name: FetchFetcherRegisterOption methods: - fetchOption - fetcherOption - registerOption options: - ident: FetcherWorkerCount interface: FetcherOption argument_type: int comment: | WithFetchWorkerCount specifies the number of HTTP fetch workers that are spawned in the backend. By default 3 workers are spawned. - ident: Whitelist interface: FetchFetcherRegisterOption argument_type: Whitelist comment: | WithWhitelist specifies the Whitelist object that can control which URLs are allowed to be processed. It can be passed to `httprc.NewCache` as a whitelist applied to all URLs that are fetched by the cache, or it can be passed on a per-URL basis using `(httprc.Cache).Register()`. If both are specified, the url must fulfill _both_ the cache-wide whitelist and the per-URL whitelist. - ident: Transformer interface: RegisterOption argument_type: Transformer comment: | WithTransformer specifies the `httprc.Transformer` object that should be applied to the fetched resource. The `Transform()` method is only called if the HTTP request returns a `200 OK` status. - ident: HTTPClient interface: FetchRegisterOption argument_type: HTTPClient comment: | WithHTTPClient specififes the HTTP Client object that should be used to fetch the resource. For example, if you need an `*http.Client` instance that requires special TLS or Authorization setup, you might want to pass it using this option. - ident: MinRefreshInterval interface: RegisterOption argument_type: time.Duration comment: | WithMinRefreshInterval specifies the minimum refresh interval to be used. When we fetch the key from a remote URL, we first look at the `max-age` directive from `Cache-Control` response header. If this value is present, we compare the `max-age` value and the value specified by this option and take the larger one (e.g. if `max-age` = 5 minutes and `min refresh` = 10 minutes, then next fetch will happen in 10 minutes) Next we check for the `Expires` header, and similarly if the header is present, we compare it against the value specified by this option, and take the larger one. Finally, if neither of the above headers are present, we use the value specified by this option as the interval until the next refresh. If unspecified, the minimum refresh interval is 1 hour. This value and the header values are ignored if `WithRefreshInterval` is specified. - ident: RefreshInterval interface: RegisterOption argument_type: time.Duration comment: | WithRefreshInterval specifies the static interval between refreshes of resources controlled by `httprc.Cache`. Providing this option overrides the adaptive token refreshing based on Cache-Control/Expires header (and `httprc.WithMinRefreshInterval`), and refreshes will *always* happen in this interval. You generally do not want to make this value too small, as it can easily be considered a DoS attack, and there is no backoff mechanism for failed attempts. - ident: RefreshWindow interface: CacheOption argument_type: time.Duration comment: | WithRefreshWindow specifies the interval between checks for refreshes. `httprc.Cache` does not check for refreshes in exact intervals. Instead, it wakes up at every tick that occurs in the interval specified by `WithRefreshWindow` option, and refreshes all entries that need to be refreshed within this window. The default value is 15 minutes. You generally do not want to make this value too small, as it can easily be considered a DoS attack, and there is no backoff mechanism for failed attempts. - ident: ErrSink interface: CacheOption argument_type: ErrSink comment: | WithErrSink specifies the `httprc.ErrSink` object that handles errors that occurred during the cache's execution. For example, you will be able to intercept errors that occurred during the execution of Transformers. golang-github-lestrrat-go-httprc-1.0.6/options_gen.go000066400000000000000000000146421464732721000227000ustar00rootroot00000000000000// This file is auto-generated by github.com/lestrrat-go/option/cmd/genoptions. DO NOT EDIT package httprc import ( "time" "github.com/lestrrat-go/option" ) type Option = option.Interface // CacheOption desribes options that can be passed to `New()` type CacheOption interface { Option cacheOption() } type cacheOption struct { Option } func (*cacheOption) cacheOption() {} type FetchFetcherRegisterOption interface { Option fetchOption() fetcherOption() registerOption() } type fetchFetcherRegisterOption struct { Option } func (*fetchFetcherRegisterOption) fetchOption() {} func (*fetchFetcherRegisterOption) fetcherOption() {} func (*fetchFetcherRegisterOption) registerOption() {} // FetchOption describes options that can be passed to `(httprc.Fetcher).Fetch()` type FetchOption interface { Option fetchOption() } type fetchOption struct { Option } func (*fetchOption) fetchOption() {} type FetchRegisterOption interface { Option fetchOption() registerOption() } type fetchRegisterOption struct { Option } func (*fetchRegisterOption) fetchOption() {} func (*fetchRegisterOption) registerOption() {} // FetcherOption describes options that can be passed to `(httprc.Fetcher).NewFetcher()` type FetcherOption interface { Option cacheOption() } type fetcherOption struct { Option } func (*fetcherOption) cacheOption() {} // RegisterOption desribes options that can be passed to `(httprc.Cache).Register()` type RegisterOption interface { Option registerOption() } type registerOption struct { Option } func (*registerOption) registerOption() {} type identErrSink struct{} type identFetcherWorkerCount struct{} type identHTTPClient struct{} type identMinRefreshInterval struct{} type identRefreshInterval struct{} type identRefreshWindow struct{} type identTransformer struct{} type identWhitelist struct{} func (identErrSink) String() string { return "WithErrSink" } func (identFetcherWorkerCount) String() string { return "WithFetcherWorkerCount" } func (identHTTPClient) String() string { return "WithHTTPClient" } func (identMinRefreshInterval) String() string { return "WithMinRefreshInterval" } func (identRefreshInterval) String() string { return "WithRefreshInterval" } func (identRefreshWindow) String() string { return "WithRefreshWindow" } func (identTransformer) String() string { return "WithTransformer" } func (identWhitelist) String() string { return "WithWhitelist" } // WithErrSink specifies the `httprc.ErrSink` object that handles errors // that occurred during the cache's execution. For example, you will be // able to intercept errors that occurred during the execution of Transformers. func WithErrSink(v ErrSink) CacheOption { return &cacheOption{option.New(identErrSink{}, v)} } // WithFetchWorkerCount specifies the number of HTTP fetch workers that are spawned // in the backend. By default 3 workers are spawned. func WithFetcherWorkerCount(v int) FetcherOption { return &fetcherOption{option.New(identFetcherWorkerCount{}, v)} } // WithHTTPClient specififes the HTTP Client object that should be used to fetch // the resource. For example, if you need an `*http.Client` instance that requires // special TLS or Authorization setup, you might want to pass it using this option. func WithHTTPClient(v HTTPClient) FetchRegisterOption { return &fetchRegisterOption{option.New(identHTTPClient{}, v)} } // WithMinRefreshInterval specifies the minimum refresh interval to be used. // // When we fetch the key from a remote URL, we first look at the `max-age` // directive from `Cache-Control` response header. If this value is present, // we compare the `max-age` value and the value specified by this option // and take the larger one (e.g. if `max-age` = 5 minutes and `min refresh` = 10 // minutes, then next fetch will happen in 10 minutes) // // Next we check for the `Expires` header, and similarly if the header is // present, we compare it against the value specified by this option, // and take the larger one. // // Finally, if neither of the above headers are present, we use the // value specified by this option as the interval until the next refresh. // // If unspecified, the minimum refresh interval is 1 hour. // // This value and the header values are ignored if `WithRefreshInterval` is specified. func WithMinRefreshInterval(v time.Duration) RegisterOption { return ®isterOption{option.New(identMinRefreshInterval{}, v)} } // WithRefreshInterval specifies the static interval between refreshes // of resources controlled by `httprc.Cache`. // // Providing this option overrides the adaptive token refreshing based // on Cache-Control/Expires header (and `httprc.WithMinRefreshInterval`), // and refreshes will *always* happen in this interval. // // You generally do not want to make this value too small, as it can easily // be considered a DoS attack, and there is no backoff mechanism for failed // attempts. func WithRefreshInterval(v time.Duration) RegisterOption { return ®isterOption{option.New(identRefreshInterval{}, v)} } // WithRefreshWindow specifies the interval between checks for refreshes. // `httprc.Cache` does not check for refreshes in exact intervals. Instead, // it wakes up at every tick that occurs in the interval specified by // `WithRefreshWindow` option, and refreshes all entries that need to be // refreshed within this window. // // The default value is 15 minutes. // // You generally do not want to make this value too small, as it can easily // be considered a DoS attack, and there is no backoff mechanism for failed // attempts. func WithRefreshWindow(v time.Duration) CacheOption { return &cacheOption{option.New(identRefreshWindow{}, v)} } // WithTransformer specifies the `httprc.Transformer` object that should be applied // to the fetched resource. The `Transform()` method is only called if the HTTP request // returns a `200 OK` status. func WithTransformer(v Transformer) RegisterOption { return ®isterOption{option.New(identTransformer{}, v)} } // WithWhitelist specifies the Whitelist object that can control which URLs are // allowed to be processed. // // It can be passed to `httprc.NewCache` as a whitelist applied to all // URLs that are fetched by the cache, or it can be passed on a per-URL // basis using `(httprc.Cache).Register()`. If both are specified, // the url must fulfill _both_ the cache-wide whitelist and the per-URL // whitelist. func WithWhitelist(v Whitelist) FetchFetcherRegisterOption { return &fetchFetcherRegisterOption{option.New(identWhitelist{}, v)} } golang-github-lestrrat-go-httprc-1.0.6/options_gen_test.go000066400000000000000000000013601464732721000237300ustar00rootroot00000000000000// This file is auto-generated by internal/cmd/genoptions/main.go. DO NOT EDIT package httprc import ( "testing" "github.com/stretchr/testify/require" ) func TestOptionIdent(t *testing.T) { require.Equal(t, "WithErrSink", identErrSink{}.String()) require.Equal(t, "WithFetcherWorkerCount", identFetcherWorkerCount{}.String()) require.Equal(t, "WithHTTPClient", identHTTPClient{}.String()) require.Equal(t, "WithMinRefreshInterval", identMinRefreshInterval{}.String()) require.Equal(t, "WithRefreshInterval", identRefreshInterval{}.String()) require.Equal(t, "WithRefreshWindow", identRefreshWindow{}.String()) require.Equal(t, "WithTransformer", identTransformer{}.String()) require.Equal(t, "WithWhitelist", identWhitelist{}.String()) } golang-github-lestrrat-go-httprc-1.0.6/queue.go000066400000000000000000000255301464732721000214760ustar00rootroot00000000000000package httprc import ( "bytes" "context" "fmt" "io" "net/http" "sync" "time" "github.com/lestrrat-go/httpcc" ) // ErrSink is an abstraction that allows users to consume errors // produced while the cache queue is running. type ErrSink interface { // Error accepts errors produced during the cache queue's execution. // The method should never block, otherwise the fetch loop may be // paused for a prolonged amount of time. Error(error) } type ErrSinkFunc func(err error) func (f ErrSinkFunc) Error(err error) { f(err) } // Transformer is responsible for converting an HTTP response // into an appropriate form of your choosing. type Transformer interface { // Transform receives an HTTP response object, and should // return an appropriate object that suits your needs. // // If you happen to use the response body, you are responsible // for closing the body Transform(string, *http.Response) (interface{}, error) } type TransformFunc func(string, *http.Response) (interface{}, error) func (f TransformFunc) Transform(u string, res *http.Response) (interface{}, error) { return f(u, res) } // BodyBytes is the default Transformer applied to all resources. // It takes an *http.Response object and extracts the body // of the response as `[]byte` type BodyBytes struct{} func (BodyBytes) Transform(_ string, res *http.Response) (interface{}, error) { buf, err := io.ReadAll(res.Body) defer res.Body.Close() if err != nil { return nil, fmt.Errorf(`failed to read response body: %w`, err) } return buf, nil } type rqentry struct { fireAt time.Time url string } // entry represents a resource to be fetched over HTTP, // long with optional specifications such as the *http.Client // object to use. type entry struct { mu sync.RWMutex sem chan struct{} lastFetch time.Time // Interval between refreshes are calculated two ways. // 1) You can set an explicit refresh interval by using WithRefreshInterval(). // In this mode, it doesn't matter what the HTTP response says in its // Cache-Control or Expires headers // 2) You can let us calculate the time-to-refresh based on the key's // Cache-Control or Expires headers. // First, the user provides us the absolute minimum interval before // refreshes. We will never check for refreshes before this specified // amount of time. // // Next, max-age directive in the Cache-Control header is consulted. // If `max-age` is not present, we skip the following section, and // proceed to the next option. // If `max-age > user-supplied minimum interval`, then we use the max-age, // otherwise the user-supplied minimum interval is used. // // Next, the value specified in Expires header is consulted. // If the header is not present, we skip the following seciont and // proceed to the next option. // We take the time until expiration `expires - time.Now()`, and // if `time-until-expiration > user-supplied minimum interval`, then // we use the expires value, otherwise the user-supplied minimum interval is used. // // If all of the above fails, we used the user-supplied minimum interval refreshInterval time.Duration minRefreshInterval time.Duration request *fetchRequest transform Transformer data interface{} } func (e *entry) acquireSem() { e.sem <- struct{}{} } func (e *entry) releaseSem() { <-e.sem } func (e *entry) hasBeenFetched() bool { e.mu.RLock() defer e.mu.RUnlock() return !e.lastFetch.IsZero() } // queue is responsible for updating the contents of the storage type queue struct { mu sync.RWMutex registry map[string]*entry windowSize time.Duration fetch Fetcher fetchCond *sync.Cond fetchQueue []*rqentry // list is a sorted list of urls to their expected fire time // when we get a new tick in the RQ loop, we process everything // that can be fired up to the point the tick was called list []*rqentry // clock is really only used by testing clock interface { Now() time.Time } } type clockFunc func() time.Time func (cf clockFunc) Now() time.Time { return cf() } func newQueue(ctx context.Context, window time.Duration, fetch Fetcher, errSink ErrSink) *queue { fetchLocker := &sync.Mutex{} rq := &queue{ windowSize: window, fetch: fetch, fetchCond: sync.NewCond(fetchLocker), registry: make(map[string]*entry), clock: clockFunc(time.Now), } go rq.refreshLoop(ctx, errSink) return rq } func (q *queue) Register(u string, options ...RegisterOption) error { var refreshInterval time.Duration var client HTTPClient var wl Whitelist var transform Transformer = BodyBytes{} minRefreshInterval := 15 * time.Minute for _, option := range options { //nolint:forcetypeassert switch option.Ident() { case identHTTPClient{}: client = option.Value().(HTTPClient) case identRefreshInterval{}: refreshInterval = option.Value().(time.Duration) case identMinRefreshInterval{}: minRefreshInterval = option.Value().(time.Duration) case identTransformer{}: transform = option.Value().(Transformer) case identWhitelist{}: wl = option.Value().(Whitelist) } } q.mu.RLock() rWindow := q.windowSize q.mu.RUnlock() if refreshInterval > 0 && refreshInterval < rWindow { return fmt.Errorf(`refresh interval (%s) is smaller than refresh window (%s): this will not as expected`, refreshInterval, rWindow) } e := entry{ sem: make(chan struct{}, 1), minRefreshInterval: minRefreshInterval, transform: transform, refreshInterval: refreshInterval, request: &fetchRequest{ client: client, url: u, wl: wl, }, } q.mu.Lock() q.registry[u] = &e q.mu.Unlock() return nil } func (q *queue) Unregister(u string) error { q.mu.Lock() defer q.mu.Unlock() _, ok := q.registry[u] if !ok { return fmt.Errorf(`url %q has not been registered`, u) } delete(q.registry, u) return nil } func (q *queue) getRegistered(u string) (*entry, bool) { q.mu.RLock() e, ok := q.registry[u] q.mu.RUnlock() return e, ok } func (q *queue) IsRegistered(u string) bool { _, ok := q.getRegistered(u) return ok } func (q *queue) fetchLoop(ctx context.Context, errSink ErrSink) { for { q.fetchCond.L.Lock() for len(q.fetchQueue) <= 0 { select { case <-ctx.Done(): return default: q.fetchCond.Wait() } } list := make([]*rqentry, len(q.fetchQueue)) copy(list, q.fetchQueue) q.fetchQueue = q.fetchQueue[:0] q.fetchCond.L.Unlock() for _, rq := range list { select { case <-ctx.Done(): return default: } e, ok := q.getRegistered(rq.url) if !ok { continue } if err := q.fetchAndStore(ctx, e); err != nil { if errSink != nil { errSink.Error(&RefreshError{ URL: rq.url, Err: err, }) } } } } } // This loop is responsible for periodically updating the cached content func (q *queue) refreshLoop(ctx context.Context, errSink ErrSink) { // Tick every q.windowSize duration. ticker := time.NewTicker(q.windowSize) go q.fetchLoop(ctx, errSink) defer q.fetchCond.Signal() for { select { case <-ctx.Done(): return case t := <-ticker.C: t = t.Round(time.Second) // To avoid getting stuck here, we just copy the relevant // items, and release the lock within this critical section var list []*rqentry q.mu.Lock() var max int for i, r := range q.list { if r.fireAt.Before(t) || r.fireAt.Equal(t) { max = i list = append(list, r) continue } break } if len(list) > 0 { q.list = q.list[max+1:] } q.mu.Unlock() // release lock if len(list) > 0 { // Now we need to fetch these, but do this elsewhere so // that we don't block this main loop q.fetchCond.L.Lock() q.fetchQueue = append(q.fetchQueue, list...) q.fetchCond.L.Unlock() q.fetchCond.Signal() } } } } func (q *queue) fetchAndStore(ctx context.Context, e *entry) error { e.mu.Lock() defer e.mu.Unlock() // synchronously go fetch e.lastFetch = time.Now() res, err := q.fetch.fetch(ctx, e.request) if err != nil { // Even if the request failed, we need to queue the next fetch q.enqueueNextFetch(nil, e) return fmt.Errorf(`failed to fetch %q: %w`, e.request.url, err) } q.enqueueNextFetch(res, e) data, err := e.transform.Transform(e.request.url, res) if err != nil { return fmt.Errorf(`failed to transform HTTP response for %q: %w`, e.request.url, err) } e.data = data return nil } func (q *queue) Enqueue(u string, interval time.Duration) error { fireAt := q.clock.Now().Add(interval).Round(time.Second) q.mu.Lock() defer q.mu.Unlock() list := q.list ll := len(list) if ll == 0 || list[ll-1].fireAt.Before(fireAt) { list = append(list, &rqentry{ fireAt: fireAt, url: u, }) } else { for i := 0; i < ll; i++ { if i == ll-1 || list[i].fireAt.After(fireAt) { // insert here list = append(list[:i+1], list[i:]...) list[i] = &rqentry{fireAt: fireAt, url: u} break } } } q.list = list return nil } func (q *queue) MarshalJSON() ([]byte, error) { var buf bytes.Buffer buf.WriteString(`{"list":[`) q.mu.RLock() for i, e := range q.list { if i > 0 { buf.WriteByte(',') } fmt.Fprintf(&buf, `{"fire_at":%q,"url":%q}`, e.fireAt.Format(time.RFC3339), e.url) } q.mu.RUnlock() buf.WriteString(`]}`) return buf.Bytes(), nil } func (q *queue) enqueueNextFetch(res *http.Response, e *entry) { dur := calculateRefreshDuration(res, e) // TODO send to error sink _ = q.Enqueue(e.request.url, dur) } func calculateRefreshDuration(res *http.Response, e *entry) time.Duration { if e.refreshInterval > 0 { return e.refreshInterval } if res != nil { if v := res.Header.Get(`Cache-Control`); v != "" { dir, err := httpcc.ParseResponse(v) if err == nil { maxAge, ok := dir.MaxAge() if ok { resDuration := time.Duration(maxAge) * time.Second if resDuration > e.minRefreshInterval { return resDuration } return e.minRefreshInterval } // fallthrough } // fallthrough } if v := res.Header.Get(`Expires`); v != "" { expires, err := http.ParseTime(v) if err == nil { resDuration := time.Until(expires) if resDuration > e.minRefreshInterval { return resDuration } return e.minRefreshInterval } // fallthrough } } // Previous fallthroughs are a little redandunt, but hey, it's all good. return e.minRefreshInterval } type SnapshotEntry struct { URL string `json:"url"` Data interface{} `json:"data"` LastFetched time.Time `json:"last_fetched"` } type Snapshot struct { Entries []SnapshotEntry `json:"entries"` } // Snapshot returns the contents of the cache at the given moment. func (q *queue) snapshot() *Snapshot { q.mu.RLock() list := make([]SnapshotEntry, 0, len(q.registry)) for url, e := range q.registry { list = append(list, SnapshotEntry{ URL: url, LastFetched: e.lastFetch, Data: e.data, }) } q.mu.RUnlock() return &Snapshot{ Entries: list, } } golang-github-lestrrat-go-httprc-1.0.6/queue_test.go000066400000000000000000000033301464732721000225270ustar00rootroot00000000000000package httprc import ( "context" "fmt" "net/http" "net/http/httptest" "testing" "time" "github.com/stretchr/testify/require" ) // Sanity check for the queue portion type dummyFetcher struct { srv *httptest.Server } func (f *dummyFetcher) Fetch(_ context.Context, _ string, _ ...FetchOption) (*http.Response, error) { panic("unimplemented") } // URLs must be for f.srv func (f *dummyFetcher) fetch(_ context.Context, fr *fetchRequest) (*http.Response, error) { //nolint:noctx return f.srv.Client().Get(fr.url) } type noErrorSink struct { t *testing.T } func (s *noErrorSink) Error(err error) { s.t.Errorf(`unexpected error: %s`, err) } func TestQueue(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) })) defer srv.Close() q := newQueue(ctx, 15*time.Minute, &dummyFetcher{srv: srv}, &noErrorSink{t: t}) base := time.Now() q.clock = clockFunc(func() time.Time { return base }) // 0..4 for i := 0; i < 4; i++ { q.Enqueue(fmt.Sprintf("%s/%d", srv.URL, i), time.Duration(i)*time.Second) } // 0..4, 6..9 for i := 7; i < 10; i++ { q.Enqueue(fmt.Sprintf("%s/%d", srv.URL, i), time.Duration(i)*time.Second) } // 0...9 for i := 4; i < 7; i++ { q.Enqueue(fmt.Sprintf("%s/%d", srv.URL, i), time.Duration(i)*time.Second) } var prevTm time.Time for i, rqe := range q.list { require.True(t, prevTm.Before(rqe.fireAt), `entry %d must have fireAt before %s (got %s)`, i, prevTm, rqe.fireAt) u := fmt.Sprintf("%s/%d", srv.URL, i) require.Equal(t, u, rqe.url, `entry %d must have url same as %s (got %s)`, i, u, rqe.url) prevTm = rqe.fireAt } } golang-github-lestrrat-go-httprc-1.0.6/tools/000077500000000000000000000000001464732721000211565ustar00rootroot00000000000000golang-github-lestrrat-go-httprc-1.0.6/tools/autodoc.pl000066400000000000000000000045771464732721000231660ustar00rootroot00000000000000#!perl use strict; use File::Temp; # Accept a list of filenames, and process them # if any of them has a diff, commit it my @files = @ARGV; my @has_diff; for my $filename (@files) { open(my $src, '<', $filename) or die $!; my $output = File::Temp->new(SUFFIX => '.md'); my $skip_until_end; for my $line (<$src>) { if ($line =~ /^$/) { $skip_until_end = 0; } elsif ($skip_until_end) { next; } if ($line !~ /(^)$/) { $output->print($line); next; } $output->print("$1\n"); my $include_filename = $2; my $options = $3; $output->print("```go\n"); my $content = do { open(my $file, '<', $include_filename) or die "failed to include file $include_filename from source file $filename: $!"; local $/; <$file>; }; $content =~ s{^(\t+)}{" " x length($1)}gsme; $output->print($content); $output->print("```\n"); $output->print("source: [$include_filename](https://github.com/lestrrat-go/jwx/blob/$ENV{GITHUB_REF}/$include_filename)\n"); # now we need to skip copying until the end of INCLUDE $skip_until_end = 1; } $output->close(); close($src); if (!$ENV{AUTODOC_DRYRUN}) { rename $output->filename, $filename or die $!; my $diff = `git diff $filename`; if ($diff) { push @has_diff, $filename; } } } if (!$ENV{AUTODOC_DRYRUN}) { if (@has_diff) { # Write multi-line commit message in a file my $commit_message_file = File::Temp->new(SUFFIX => '.txt'); print $commit_message_file "autodoc updates\n\n"; print " - $_\n" for @has_diff; $commit_message_file->close(); system("git", "remote", "set-url", "origin", "https://github-actions:$ENV{GITHUB_TOKEN}\@github.com/$ENV{GITHUB_REPOSITORY}") == 0 or die $!; system("git", "config", "--global", "user.name", "$ENV{GITHUB_ACTOR}") == 0 or die $!; system("git", "config", "--global", "user.email", "$ENV{GITHUB_ACTOR}\@users.noreply.github.com") == 0 or die $!; system("git", "commit", "-F", $commit_message_file->filename, @files) == 0 or die $!; system("git", "push", "origin", "HEAD:$ENV{GITHUB_REF}") == 0 or die $!; } } golang-github-lestrrat-go-httprc-1.0.6/tools/genoptions.sh000077500000000000000000000002661464732721000237060ustar00rootroot00000000000000#!/bin/bash which genoptions > /dev/null if [[ "$?" != "0" ]]; then echo "Please install github.com/lestrrat-go/option/cmd/genoptions" exit 1 fi genoptions -objects=options.yaml golang-github-lestrrat-go-httprc-1.0.6/whitelist.go000066400000000000000000000035671464732721000223740ustar00rootroot00000000000000package httprc import "regexp" // Whitelist is an interface for a set of URL whitelists. When provided // to fetching operations, urls are checked against this object, and // the object must return true for urls to be fetched. type Whitelist interface { IsAllowed(string) bool } // InsecureWhitelist allows any URLs to be fetched. type InsecureWhitelist struct{} func (InsecureWhitelist) IsAllowed(string) bool { return true } // RegexpWhitelist is a httprc.Whitelist object comprised of a list of *regexp.Regexp // objects. All entries in the list are tried until one matches. If none of the // *regexp.Regexp objects match, then the URL is deemed unallowed. type RegexpWhitelist struct { patterns []*regexp.Regexp } func NewRegexpWhitelist() *RegexpWhitelist { return &RegexpWhitelist{} } func (w *RegexpWhitelist) Add(pat *regexp.Regexp) *RegexpWhitelist { w.patterns = append(w.patterns, pat) return w } // IsAlloed returns true if any of the patterns in the whitelist // returns true. func (w *RegexpWhitelist) IsAllowed(u string) bool { for _, pat := range w.patterns { if pat.MatchString(u) { return true } } return false } // MapWhitelist is a httprc.Whitelist object comprised of a map of strings. // If the URL exists in the map, then the URL is allowed to be fetched. type MapWhitelist struct { store map[string]struct{} } func NewMapWhitelist() *MapWhitelist { return &MapWhitelist{store: make(map[string]struct{})} } func (w *MapWhitelist) Add(pat string) *MapWhitelist { w.store[pat] = struct{}{} return w } func (w *MapWhitelist) IsAllowed(u string) bool { _, b := w.store[u] return b } // WhitelistFunc is a httprc.Whitelist object based on a function. // You can perform any sort of check against the given URL to determine // if it can be fetched or not. type WhitelistFunc func(string) bool func (w WhitelistFunc) IsAllowed(u string) bool { return w(u) }