pax_global_header00006660000000000000000000000064147145426040014521gustar00rootroot0000000000000052 comment=0df30c911df3377f7abe897c3c915725b7711cd0 container-library-client-1.4.10/000077500000000000000000000000001471454260400165045ustar00rootroot00000000000000container-library-client-1.4.10/.github/000077500000000000000000000000001471454260400200445ustar00rootroot00000000000000container-library-client-1.4.10/.github/dependabot.yml000066400000000000000000000001771471454260400227010ustar00rootroot00000000000000version: 2 updates: - package-ecosystem: gomod directory: "/" schedule: interval: daily open-pull-requests-limit: 10 container-library-client-1.4.10/.github/workflows/000077500000000000000000000000001471454260400221015ustar00rootroot00000000000000container-library-client-1.4.10/.github/workflows/ci.yml000066400000000000000000000021501471454260400232150ustar00rootroot00000000000000name: ci on: pull_request: push: branches: - main tags: - 'v*.*.*' jobs: build_and_test: name: build_and_test runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 - name: Check for markdown Lint run: | sudo npm install -g markdownlint-cli markdownlint . - name: Set up Go uses: actions/setup-go@v1 with: go-version: '1.22.x' - name: Check go.mod and go.sum are tidy run: | go mod tidy git diff --exit-code -- go.mod go.sum - name: Build Source run: go build ./... - name: Install Lint uses: golangci/golangci-lint-action@v2 with: version: v1.60 skip-pkg-cache: true skip-build-cache: true - name: Run Lint run: | golangci-lint run - name: Run Tests run: go test -coverprofile cover.out -race ./... - name: Upload coverage report uses: codecov/codecov-action@v1 with: files: cover.out flags: unittests name: codecov container-library-client-1.4.10/.gitignore000066400000000000000000000000131471454260400204660ustar00rootroot00000000000000*~ vendor/ container-library-client-1.4.10/.golangci.yml000066400000000000000000000010071471454260400210660ustar00rootroot00000000000000linters: disable-all: true enable: - bidichk - bodyclose - containedctx - contextcheck - decorder - dogsled - errcheck - errchkjson - gochecknoinits # - goconst - gocritic - gocyclo - err113 - gofumpt - goimports - goprintffuncname - gosimple - govet - grouper - ineffassign - ireturn - maintidx - misspell - nakedret - prealloc - revive - staticcheck - stylecheck - tenv - typecheck - unused container-library-client-1.4.10/.markdownlint.yml000066400000000000000000000000151471454260400220120ustar00rootroot00000000000000MD013: false container-library-client-1.4.10/.vscode/000077500000000000000000000000001471454260400200455ustar00rootroot00000000000000container-library-client-1.4.10/.vscode/settings.json000066400000000000000000000000461471454260400226000ustar00rootroot00000000000000{ "go.lintTool": "golangci-lint" }container-library-client-1.4.10/LICENSE.md000066400000000000000000000027321471454260400201140ustar00rootroot00000000000000# LICENSE Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 1. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 1. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. container-library-client-1.4.10/README.md000066400000000000000000000026561471454260400177740ustar00rootroot00000000000000# Container Library Client [![PkgGoDev](https://pkg.go.dev/badge/github.com/apptainer/container-library-client)](https://pkg.go.dev/github.com/apptainer/container-library-client/client) [![Build Status](https://github.com/apptainer/container-library-client/actions/workflows/ci.yml/badge.svg)](https://github.com/apptainer/container-library-client/actions/workflows/ci.yml) [![Code Coverage](https://codecov.io/gh/apptainer/container-library-client/branch/main/graph/badge.svg)](https://codecov.io/gh/apptainer/container-library-client) [![Go Report Card](https://goreportcard.com/badge/github.com/apptainer/container-library-client)](https://goreportcard.com/report/github.com/apptainer/container-library-client) This project provides a Go client to Apptainer for the container library. Forked from [sylabs/scs-library-client](https://github.com/sylabs/scs-library-client). This is maintained as a sub-project by the Apptainer project. To contribute, see the [Apptainer CONTRIBUTING file](https://github.com/apptainer/apptainer/blob/main/CONTRIBUTING.md). ## Go Version Compatibility This module aims to maintain support for the two most recent stable versions of Go. This corresponds to the Go [Release Maintenance Policy](https://github.com/golang/go/wiki/Go-Release-Cycle#release-maintenance) and [Security Policy](https://golang.org/security), ensuring critical bug fixes and security patches are available for all supported language versions. container-library-client-1.4.10/client/000077500000000000000000000000001471454260400177625ustar00rootroot00000000000000container-library-client-1.4.10/client/api.go000066400000000000000000000224601471454260400210660ustar00rootroot00000000000000// Copyright (c) 2018-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( "bytes" "context" "encoding/json" "fmt" "net/http" "net/url" jsonresp "github.com/sylabs/json-resp" ) // getEntity returns the specified entity; returns ErrNotFound if entity is not // found, otherwise error func (c *Client) getEntity(ctx context.Context, entityRef string) (*Entity, error) { entJSON, err := c.apiGet(ctx, "v1/entities/"+entityRef) if err != nil { return nil, err } var res EntityResponse if err := json.Unmarshal(entJSON, &res); err != nil { return nil, fmt.Errorf("error decoding entity: %w", err) } return &res.Data, nil } // getCollection returns the specified collection; returns ErrNotFound if // collection is not found, otherwise error. func (c *Client) getCollection(ctx context.Context, collectionRef string) (*Collection, error) { colJSON, err := c.apiGet(ctx, "v1/collections/"+collectionRef) if err != nil { return nil, err } var res CollectionResponse if err := json.Unmarshal(colJSON, &res); err != nil { return nil, fmt.Errorf("error decoding collection: %w", err) } return &res.Data, nil } // getContainer returns container by ref id; returns ErrNotFound if container // is not found, otherwise error. func (c *Client) getContainer(ctx context.Context, containerRef string) (*Container, error) { conJSON, err := c.apiGet(ctx, "v1/containers/"+containerRef) if err != nil { return nil, err } var res ContainerResponse if err := json.Unmarshal(conJSON, &res); err != nil { return nil, fmt.Errorf("error decoding container: %w", err) } return &res.Data, nil } // createEntity creates an entity (must be authorized) func (c *Client) createEntity(ctx context.Context, name string) (*Entity, error) { e := Entity{ Name: name, Description: "No description", } entJSON, err := c.apiCreate(ctx, "v1/entities", e) if err != nil { return nil, err } var res EntityResponse if err := json.Unmarshal(entJSON, &res); err != nil { return nil, fmt.Errorf("error decoding entity: %w", err) } return &res.Data, nil } // createCollection creates a new collection func (c *Client) createCollection(ctx context.Context, name string, entityID string) (*Collection, error) { newCollection := Collection{ Name: name, Description: "No description", Entity: entityID, } colJSON, err := c.apiCreate(ctx, "v1/collections", newCollection) if err != nil { return nil, err } var res CollectionResponse if err := json.Unmarshal(colJSON, &res); err != nil { return nil, fmt.Errorf("error decoding collection: %w", err) } return &res.Data, nil } // createContainer creates a container in the specified collection func (c *Client) createContainer(ctx context.Context, name string, collectionID string) (*Container, error) { newContainer := Container{ Name: name, Description: "No description", Collection: collectionID, } conJSON, err := c.apiCreate(ctx, "v1/containers", newContainer) if err != nil { return nil, err } var res ContainerResponse if err := json.Unmarshal(conJSON, &res); err != nil { return nil, fmt.Errorf("error decoding container: %w", err) } return &res.Data, nil } // createImage creates a new image func (c *Client) createImage(ctx context.Context, hash string, containerID string, description string) (*Image, error) { i := Image{ Hash: hash, Description: description, Container: containerID, } imgJSON, err := c.apiCreate(ctx, "v1/images", i) if err != nil { return nil, err } var res ImageResponse if err := json.Unmarshal(imgJSON, &res); err != nil { return nil, fmt.Errorf("error decoding image: %w", err) } return &res.Data, nil } // setTags applies tags to the specified container func (c *Client) setTags(ctx context.Context, containerID, imageID string, tags []string) error { // Get existing tags, so we know which will be replaced existingTags, err := c.getTags(ctx, containerID) if err != nil { return err } for _, tag := range tags { c.Logger.Logf("Setting tag %s", tag) if _, ok := existingTags[tag]; ok { c.Logger.Logf("%s replaces an existing tag", tag) } imgTag := ImageTag{ tag, imageID, } err := c.setTag(ctx, containerID, imgTag) if err != nil { return err } } return nil } // getTags returns a tag map for the specified containerID func (c *Client) getTags(ctx context.Context, containerID string) (TagMap, error) { url := fmt.Sprintf("v1/tags/%s", containerID) c.Logger.Logf("getTags calling %s", url) req, err := c.newRequest(ctx, http.MethodGet, url, "", nil) if err != nil { return nil, fmt.Errorf("error creating request to server:\n\t%w", err) } res, err := c.HTTPClient.Do(req) if err != nil { return nil, fmt.Errorf("error making request to server:\n\t%w", err) } defer res.Body.Close() if res.StatusCode != http.StatusOK { err := jsonresp.ReadError(res.Body) if err != nil { return nil, fmt.Errorf("creation did not succeed: %w", err) } return nil, fmt.Errorf("%w: unexpected http status code: %d", errHTTP, res.StatusCode) } var tagRes TagsResponse err = json.NewDecoder(res.Body).Decode(&tagRes) if err != nil { return nil, fmt.Errorf("error decoding tags: %w", err) } return tagRes.Data, nil } // setTag sets tag on specified containerID func (c *Client) setTag(ctx context.Context, containerID string, t ImageTag) error { url := "v1/tags/" + containerID c.Logger.Logf("setTag calling %s", url) s, err := json.Marshal(t) if err != nil { return fmt.Errorf("error encoding object to JSON:\n\t%w", err) } req, err := c.newRequest(ctx, http.MethodPost, url, "", bytes.NewBuffer(s)) if err != nil { return fmt.Errorf("error creating POST request:\n\t%w", err) } res, err := c.HTTPClient.Do(req) if err != nil { return fmt.Errorf("error making request to server:\n\t%w", err) } defer res.Body.Close() if res.StatusCode != http.StatusOK { err := jsonresp.ReadError(res.Body) if err != nil { return fmt.Errorf("creation did not succeed: %w", err) } return fmt.Errorf("%w: creation did not succeed: http status code: %d", errHTTP, res.StatusCode) } return nil } // setTags applies tags to the specified container func (c *Client) setTagsV2(ctx context.Context, containerID, arch string, imageID string, tags []string) error { // Get existing tags, so we know which will be replaced existingTags, err := c.getTagsV2(ctx, containerID) if err != nil { return err } for _, tag := range tags { c.Logger.Logf("Setting tag %s", tag) if _, ok := existingTags[arch][tag]; ok { c.Logger.Logf("%s replaces an existing tag for arch %s", tag, arch) } imgTag := ArchImageTag{ Arch: arch, Tag: tag, ImageID: imageID, } err := c.setTagV2(ctx, containerID, imgTag) if err != nil { return err } } return nil } // getTagsV2 returns a arch->tag map for the specified containerID func (c *Client) getTagsV2(ctx context.Context, containerID string) (ArchTagMap, error) { url := fmt.Sprintf("v2/tags/%s", containerID) c.Logger.Logf("getTagsV2 calling %s", url) req, err := c.newRequest(ctx, http.MethodGet, url, "", nil) if err != nil { return nil, fmt.Errorf("error creating request to server:\n\t%w", err) } res, err := c.HTTPClient.Do(req) if err != nil { return nil, fmt.Errorf("error making request to server:\n\t%w", err) } defer res.Body.Close() if res.StatusCode != http.StatusOK { err := jsonresp.ReadError(res.Body) if err != nil { return nil, fmt.Errorf("creation did not succeed: %w", err) } return nil, fmt.Errorf("%w: unexpected http status code: %d", errHTTP, res.StatusCode) } var tagRes ArchTagsResponse err = json.NewDecoder(res.Body).Decode(&tagRes) if err != nil { return nil, fmt.Errorf("error decoding tags: %w", err) } return tagRes.Data, nil } // setTag sets an arch->tag on specified containerID func (c *Client) setTagV2(ctx context.Context, containerID string, t ArchImageTag) error { url := "v2/tags/" + containerID c.Logger.Logf("setTag calling %s", url) s, err := json.Marshal(t) if err != nil { return fmt.Errorf("error encoding object to JSON:\n\t%w", err) } req, err := c.newRequest(ctx, http.MethodPost, url, "", bytes.NewBuffer(s)) if err != nil { return fmt.Errorf("error creating POST request:\n\t%w", err) } res, err := c.HTTPClient.Do(req) if err != nil { return fmt.Errorf("error making request to server:\n\t%w", err) } defer res.Body.Close() if res.StatusCode != http.StatusOK { err := jsonresp.ReadError(res.Body) if err != nil { return fmt.Errorf("creation did not succeed: %w", err) } return fmt.Errorf("%w: creation did not succeed: http status code: %d", errHTTP, res.StatusCode) } return nil } // GetImage returns the Image object if exists; returns ErrNotFound if image is // not found, otherwise error. func (c *Client) GetImage(ctx context.Context, arch string, imageRef string) (*Image, error) { q := url.Values{} q.Add("arch", arch) apiURL := &url.URL{ Path: "v1/images/" + imageRef, RawQuery: q.Encode(), } imgJSON, err := c.apiGet(ctx, apiURL.String()) if err != nil { return nil, err } var res ImageResponse if err := json.Unmarshal(imgJSON, &res); err != nil { return nil, fmt.Errorf("error decoding image: %w", err) } return &res.Data, nil } container-library-client-1.4.10/client/api_test.go000066400000000000000000000556141471454260400221340ustar00rootroot00000000000000// Copyright (c) 2018-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( "context" "encoding/json" "errors" "net/http" "net/http/httptest" "reflect" "testing" jsonresp "github.com/sylabs/json-resp" ) const ( testToken = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiYWRtaW4iOnRydWUsImlhdCI6MTUxNjIzOTAyMn0.TCYt5XsITJX1CxPCT8yAV-TVkIEq_PbChOMqsLfRoPsnsgw5WEuts01mq-pQy7UJiN5mgRxD-WUcX16dUEMGlv50aqzpqh4Qktb3rk-BuQy72IFLOqV0G_zS245-kronKb78cPN25DGlcTwLtjPAYuNzVBAh4vGHSrQyHUdBBPM" ) var ( signedImage = true unsignedImage = false encryptedImage = true unencryptedImage = false testEntity = Entity{ ID: "5cb9c34d7d960d82f5f5bc4a", Name: "test-user", Description: "A test user", } testCollection = Collection{ ID: "5cb9c34d7d960d82f5f5bc4b", Name: "test-collection", Description: "A test collection", Entity: testEntity.ID, EntityName: testEntity.Name, } testContainer = Container{ ID: "5cb9c34d7d960d82f5f5bc4c", Name: "test-container", Description: "A test container", Entity: testEntity.ID, EntityName: testEntity.Name, Collection: testEntity.ID, CollectionName: testCollection.Name, ImageTags: map[string]string{ "test-tag": "5cb9c34d7d960d82f5f5bc4d", "latest": "5cb9c34d7d960d82f5f5bc4e", }, } archIntel = "amd64" testImage = Image{ ID: "5cb9c34d7d960d82f5f5bc4f", Hash: "sha256.e50a30881ace3d5944f5661d222db7bee5296be9e4dc7c1fcb7604bcae926e88", Entity: testEntity.ID, EntityName: testEntity.Name, Collection: testEntity.ID, CollectionName: testCollection.Name, Container: testContainer.ID, ContainerName: testContainer.Name, Architecture: &archIntel, } archARM = "arm64" testImage2 = Image{ ID: "bf396e3d2de63215e731c11f", Hash: "sha256.d8fb363e56735af5f127a2f12bdba8d7aedf5861c7ef7eb7197f56323d1831f7", Entity: testEntity.ID, EntityName: testEntity.Name, Collection: testEntity.ID, CollectionName: testCollection.Name, Container: testContainer.ID, ContainerName: testContainer.Name, Architecture: &archARM, } testImage3 = Image{ ID: "49a312c677e2e3f3d36ac3d0", Hash: "sha256.b23a9b9f41809a2dfe16a9e3b1d948909dab2efbf1997d6f5a46dea8af5cdb78", Entity: testEntity.ID, EntityName: testEntity.Name, Collection: testEntity.ID, CollectionName: testCollection.Name, Container: testContainer.ID, ContainerName: testContainer.Name, Architecture: &archARM, Signed: &signedImage, } testImage4 = Image{ ID: "2c2e6c834274f031a01f83dc", Hash: "sha256.4f4ba1a6b584001734a33247a06e75d06af2999103139a2e2b767dc873a21b7a", Entity: testEntity.ID, EntityName: testEntity.Name, Collection: testEntity.ID, CollectionName: testCollection.Name, Container: testContainer.ID, ContainerName: testContainer.Name, Architecture: &archARM, Signed: &unsignedImage, } testImage5 = Image{ ID: "1111222223333031a01f83dc", Hash: "sha256.4f4ba1112223331734a33247a06e75d06af2999103139a2e2b767dc873a21b7a", Entity: testEntity.ID, EntityName: testEntity.Name, Collection: testEntity.ID, CollectionName: testCollection.Name, Container: testContainer.ID, ContainerName: testContainer.Name, Signed: &unsignedImage, } testImage6 = Image{ ID: "222233334444031a01f83dc", Hash: "sha256.4f4ba1112223331734a33247a06e75d06af2999103139a2e2b767dc873a21b7a", Entity: testEntity.ID, EntityName: testEntity.Name, Collection: testEntity.ID, CollectionName: testCollection.Name, Container: testContainer.ID, ContainerName: testContainer.Name, Signed: &unsignedImage, Encrypted: &unencryptedImage, } testImage7 = Image{ ID: "333344445555031a01f83dc", Hash: "sha256.4f4ba1112223331734a33247a06e75d06af2999103139a2e2b767dc873a21b7a", Entity: testEntity.ID, EntityName: testEntity.Name, Collection: testEntity.ID, CollectionName: testCollection.Name, Container: testContainer.ID, ContainerName: testContainer.Name, Signed: &unsignedImage, Encrypted: &encryptedImage, } testSearch = SearchResults{ Entities: []Entity{testEntity}, Collections: []Collection{testCollection}, Containers: []Container{testContainer}, Images: []Image{ testImage, testImage2, testImage3, testImage4, testImage5, testImage6, testImage7, }, } ) type mockService struct { t *testing.T code int body interface{} reqCallback func(*http.Request, *testing.T) httpAddr string httpPath string httpServer *httptest.Server baseURI string } func (m *mockService) Run() { mux := http.NewServeMux() mux.HandleFunc(m.httpPath, m.ServeHTTP) m.httpServer = httptest.NewServer(mux) m.httpAddr = m.httpServer.Listener.Addr().String() m.baseURI = "http://" + m.httpAddr } func (m *mockService) Stop() { m.httpServer.Close() } func (m *mockService) ServeHTTP(w http.ResponseWriter, r *http.Request) { if m.reqCallback != nil { m.reqCallback(r, m.t) } w.WriteHeader(m.code) err := json.NewEncoder(w).Encode(&m.body) if err != nil { m.t.Errorf("Error encoding mock response: %v", err) } } func Test_getEntity(t *testing.T) { tests := []struct { description string code int body interface{} reqCallback func(*http.Request, *testing.T) entityRef string expectEntity *Entity expectFound bool expectError bool }{ { description: "NotFound", code: http.StatusNotFound, body: jsonresp.Response{Error: &jsonresp.Error{Code: http.StatusNotFound}}, reqCallback: nil, entityRef: "notthere", expectEntity: nil, expectFound: false, expectError: true, }, { description: "Unauthorized", code: http.StatusUnauthorized, body: jsonresp.Response{Error: &jsonresp.Error{Code: http.StatusUnauthorized}}, reqCallback: nil, entityRef: "notmine", expectEntity: nil, expectFound: false, expectError: true, }, { description: "ValidResponse", code: http.StatusOK, body: EntityResponse{Data: testEntity}, reqCallback: nil, entityRef: "test-user", expectEntity: &testEntity, expectFound: true, expectError: false, }, } // Loop over test cases for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { m := mockService{ t: t, code: tt.code, body: tt.body, reqCallback: tt.reqCallback, httpPath: "/v1/entities/" + tt.entityRef, } m.Run() defer m.Stop() c, err := NewClient(&Config{AuthToken: testToken, BaseURL: m.baseURI}) if err != nil { t.Errorf("Error initializing client: %v", err) } entity, err := c.getEntity(context.Background(), tt.entityRef) if err != nil && !tt.expectError { t.Errorf("Unexpected error: %v", err) } if err == nil && tt.expectError { t.Errorf("Unexpected success. Expected error.") } if err != nil && errors.Is(err, ErrNotFound) && tt.expectFound { t.Errorf("Got found %v - expected %v", !errors.Is(err, ErrNotFound), tt.expectFound) } if !reflect.DeepEqual(entity, tt.expectEntity) { t.Errorf("Got entity %v - expected %v", entity, tt.expectEntity) } }) } } func Test_getCollection(t *testing.T) { tests := []struct { description string code int body interface{} reqCallback func(*http.Request, *testing.T) collectionRef string expectCollection *Collection expectFound bool expectError bool }{ { description: "NotFound", code: http.StatusNotFound, body: jsonresp.Response{Error: &jsonresp.Error{Code: http.StatusNotFound}}, reqCallback: nil, collectionRef: "notthere", expectCollection: nil, expectFound: false, expectError: true, }, { description: "Unauthorized", code: http.StatusUnauthorized, body: jsonresp.Response{Error: &jsonresp.Error{Code: http.StatusUnauthorized}}, reqCallback: nil, collectionRef: "notmine", expectCollection: nil, expectFound: false, expectError: true, }, { description: "ValidResponse", code: http.StatusOK, body: CollectionResponse{Data: testCollection}, reqCallback: nil, collectionRef: "test-entity/test-collection", expectCollection: &testCollection, expectFound: true, expectError: false, }, } // Loop over test cases for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { m := mockService{ t: t, code: tt.code, body: tt.body, reqCallback: tt.reqCallback, httpPath: "/v1/collections/" + tt.collectionRef, } m.Run() defer m.Stop() c, err := NewClient(&Config{AuthToken: testToken, BaseURL: m.baseURI}) if err != nil { t.Errorf("Error initializing client: %v", err) } collection, err := c.getCollection(context.Background(), tt.collectionRef) if err != nil && !tt.expectError { t.Errorf("Unexpected error: %v", err) } if err == nil && tt.expectError { t.Errorf("Unexpected success. Expected error.") } if err != nil && errors.Is(err, ErrNotFound) && tt.expectFound { t.Errorf("Got found %v - expected %v", !errors.Is(err, ErrNotFound), tt.expectFound) } if !reflect.DeepEqual(collection, tt.expectCollection) { t.Errorf("Got entity %v - expected %v", collection, tt.expectCollection) } }) } } func Test_getContainer(t *testing.T) { tests := []struct { description string code int body interface{} reqCallback func(*http.Request, *testing.T) containerRef string expectContainer *Container expectFound bool expectError bool }{ { description: "NotFound", code: http.StatusNotFound, body: jsonresp.Response{Error: &jsonresp.Error{Code: http.StatusNotFound}}, reqCallback: nil, containerRef: "notthere", expectContainer: nil, expectFound: false, expectError: true, }, { description: "Unauthorized", code: http.StatusUnauthorized, body: jsonresp.Response{Error: &jsonresp.Error{Code: http.StatusUnauthorized}}, reqCallback: nil, containerRef: "notmine", expectContainer: nil, expectFound: false, expectError: true, }, { description: "ValidResponse", code: http.StatusOK, body: ContainerResponse{Data: testContainer}, reqCallback: nil, containerRef: "test-entity/test-collection/test-container", expectContainer: &testContainer, expectFound: true, expectError: false, }, } // Loop over test cases for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { m := mockService{ t: t, code: tt.code, body: tt.body, reqCallback: tt.reqCallback, httpPath: "/v1/containers/" + tt.containerRef, } m.Run() defer m.Stop() c, err := NewClient(&Config{AuthToken: testToken, BaseURL: m.baseURI}) if err != nil { t.Errorf("Error initializing client: %v", err) } container, err := c.getContainer(context.Background(), tt.containerRef) if err != nil && !tt.expectError { t.Errorf("Unexpected error: %v", err) } if err == nil && tt.expectError { t.Errorf("Unexpected success. Expected error.") } if err != nil && !errors.Is(err, ErrNotFound) && tt.expectFound { t.Errorf("Got found %v - expected %v", !errors.Is(err, ErrNotFound), tt.expectFound) } if !reflect.DeepEqual(container, tt.expectContainer) { t.Errorf("Got container %v - expected %v", container, tt.expectContainer) } }) } } func Test_getImage(t *testing.T) { tests := []struct { description string code int body interface{} reqCallback func(*http.Request, *testing.T) arch string imageRef string expectImage *Image expectFound bool expectError bool }{ { description: "NotFound", code: http.StatusNotFound, body: jsonresp.Response{Error: &jsonresp.Error{Code: http.StatusNotFound}}, reqCallback: nil, arch: archIntel, imageRef: "notthere", expectImage: nil, expectFound: false, expectError: true, }, { description: "Unauthorized", code: http.StatusUnauthorized, body: jsonresp.Response{Error: &jsonresp.Error{Code: http.StatusUnauthorized}}, reqCallback: nil, arch: archIntel, imageRef: "notmine", expectImage: nil, expectFound: false, expectError: true, }, { description: "ValidResponse", code: http.StatusOK, body: ImageResponse{Data: testImage}, reqCallback: nil, arch: archIntel, imageRef: "test", expectImage: &testImage, expectFound: true, expectError: false, }, } // Loop over test cases for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { m := mockService{ t: t, code: tt.code, body: tt.body, reqCallback: tt.reqCallback, httpPath: "/v1/images/" + tt.imageRef, } m.Run() defer m.Stop() c, err := NewClient(&Config{AuthToken: testToken, BaseURL: m.baseURI}) if err != nil { t.Errorf("Error initializing client: %v", err) } image, err := c.GetImage(context.Background(), tt.arch, tt.imageRef) if err != nil && !tt.expectError { t.Errorf("Unexpected error: %v", err) } if err == nil && tt.expectError { t.Errorf("Unexpected success. Expected error.") } if err != nil && !errors.Is(err, ErrNotFound) && tt.expectFound { t.Errorf("Got found %v - expected %v", !errors.Is(err, ErrNotFound), tt.expectFound) } if !reflect.DeepEqual(image, tt.expectImage) { t.Errorf("Got image %v - expected %v", image, tt.expectImage) } }) } } func Test_createEntity(t *testing.T) { tests := []struct { description string code int body interface{} reqCallback func(*http.Request, *testing.T) entityRef string expectEntity *Entity expectError bool }{ { description: "Valid Request", code: http.StatusOK, body: EntityResponse{Data: testEntity}, entityRef: "test", expectEntity: &testEntity, expectError: false, }, { description: "Error response", code: http.StatusInternalServerError, body: Entity{}, entityRef: "test", expectEntity: nil, expectError: true, }, } // Loop over test cases for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { m := mockService{ t: t, code: tt.code, body: tt.body, reqCallback: tt.reqCallback, httpPath: "/v1/entities/", } m.Run() defer m.Stop() c, err := NewClient(&Config{AuthToken: testToken, BaseURL: m.baseURI}) if err != nil { t.Errorf("Error initializing client: %v", err) } entity, err := c.createEntity(context.Background(), tt.entityRef) if err != nil && !tt.expectError { t.Errorf("Unexpected error: %v", err) } if err == nil && tt.expectError { t.Errorf("Unexpected success. Expected error.") } if !reflect.DeepEqual(entity, tt.expectEntity) { t.Errorf("Got created entity %v - expected %v", entity, tt.expectEntity) } }) } } func Test_createCollection(t *testing.T) { tests := []struct { description string code int body interface{} reqCallback func(*http.Request, *testing.T) collectionRef string expectCollection *Collection expectError bool }{ { description: "Valid Request", code: http.StatusOK, body: CollectionResponse{Data: Collection{Name: "test"}}, collectionRef: "test", expectCollection: &Collection{Name: "test"}, expectError: false, }, } // Loop over test cases for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { m := mockService{ t: t, code: tt.code, body: tt.body, reqCallback: tt.reqCallback, httpPath: "/v1/collections/", } m.Run() defer m.Stop() c, err := NewClient(&Config{AuthToken: testToken, BaseURL: m.baseURI}) if err != nil { t.Errorf("Error initializing client: %v", err) } collection, err := c.createCollection(context.Background(), tt.collectionRef, "5cb9c34d7d960d82f5f5bc50") if err != nil && !tt.expectError { t.Errorf("Unexpected error: %v", err) } if err == nil && tt.expectError { t.Errorf("Unexpected success. Expected error.") } if !reflect.DeepEqual(collection, tt.expectCollection) { t.Errorf("Got created collection %v - expected %v", collection, tt.expectCollection) } }) } } func Test_createContainer(t *testing.T) { tests := []struct { description string code int body interface{} reqCallback func(*http.Request, *testing.T) containerRef string expectContainer *Container expectError bool }{ { description: "Valid Request", code: http.StatusOK, body: ContainerResponse{Data: Container{Name: "test"}}, containerRef: "test", expectContainer: &Container{Name: "test"}, expectError: false, }, } // Loop over test cases for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { m := mockService{ t: t, code: tt.code, body: tt.body, reqCallback: tt.reqCallback, httpPath: "/v1/containers/", } m.Run() defer m.Stop() c, err := NewClient(&Config{AuthToken: testToken, BaseURL: m.baseURI}) if err != nil { t.Errorf("Error initializing client: %v", err) } container, err := c.createContainer(context.Background(), tt.containerRef, "5cb9c34d7d960d82f5f5bc51") if err != nil && !tt.expectError { t.Errorf("Unexpected error: %v", err) } if err == nil && tt.expectError { t.Errorf("Unexpected success. Expected error.") } if !reflect.DeepEqual(container, tt.expectContainer) { t.Errorf("Got created collection %v - expected %v", container, tt.expectContainer) } }) } } func Test_createImage(t *testing.T) { tests := []struct { description string code int body interface{} reqCallback func(*http.Request, *testing.T) imageRef string expectImage *Image expectError bool }{ { description: "Valid Request", code: http.StatusOK, body: ImageResponse{Data: Image{Hash: "sha256.e50a30881ace3d5944f5661d222db7bee5296be9e4dc7c1fcb7604bcae926e88"}}, imageRef: "test", expectImage: &Image{Hash: "sha256.e50a30881ace3d5944f5661d222db7bee5296be9e4dc7c1fcb7604bcae926e88"}, expectError: false, }, { description: "Error response", code: http.StatusInternalServerError, body: Image{}, imageRef: "test", expectImage: nil, expectError: true, }, } // Loop over test cases for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { m := mockService{ t: t, code: tt.code, body: tt.body, reqCallback: tt.reqCallback, httpPath: "/v1/images/", } m.Run() defer m.Stop() c, err := NewClient(&Config{AuthToken: testToken, BaseURL: m.baseURI}) if err != nil { t.Errorf("Error initializing client: %v", err) } image, err := c.createImage(context.Background(), tt.imageRef, "5cb9c34d7d960d82f5f5bc52", "No Description") if err != nil && !tt.expectError { t.Errorf("Unexpected error: %v", err) } if err == nil && tt.expectError { t.Errorf("Unexpected success. Expected error.") } if !reflect.DeepEqual(image, tt.expectImage) { t.Errorf("Got created collection %v - expected %v", image, tt.expectImage) } }) } } func Test_setTags(t *testing.T) { tests := []struct { description string code int reqCallback func(*http.Request, *testing.T) containerRef string imageRef string tags []string expectError bool }{ { description: "Valid Request", code: http.StatusOK, containerRef: "test", imageRef: "5cb9c34d7d960d82f5f5bc53", tags: []string{"tag1", "tag2", "tag3"}, expectError: false, }, { description: "Error response", code: http.StatusInternalServerError, containerRef: "test", imageRef: "5cb9c34d7d960d82f5f5bc54", tags: []string{"tag1", "tag2", "tag3"}, expectError: true, }, } // Loop over test cases for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { m := mockService{ t: t, code: tt.code, reqCallback: tt.reqCallback, httpPath: "/v1/tags/" + tt.containerRef, } m.Run() defer m.Stop() c, err := NewClient(&Config{AuthToken: testToken, BaseURL: m.baseURI}) if err != nil { t.Errorf("Error initializing client: %v", err) } err = c.setTags(context.Background(), tt.containerRef, tt.imageRef, tt.tags) if err != nil && !tt.expectError { t.Errorf("Unexpected error: %v", err) } if err == nil && tt.expectError { t.Errorf("Unexpected success. Expected error.") } }) } } func Test_setTagsV2(t *testing.T) { tests := []struct { description string code int reqCallback func(*http.Request, *testing.T) containerRef string imageRef string arch string tags []string expectError bool }{ { description: "Valid Request", code: http.StatusOK, containerRef: "test", imageRef: "5cb9c34d7d960d82f5f5bc53", arch: archIntel, tags: []string{"tag1", "tag2", "tag3"}, expectError: false, }, { description: "Error response", code: http.StatusInternalServerError, containerRef: "test", imageRef: "5cb9c34d7d960d82f5f5bc54", arch: archIntel, tags: []string{"tag1", "tag2", "tag3"}, expectError: true, }, } // Loop over test cases for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { m := mockService{ t: t, code: tt.code, reqCallback: tt.reqCallback, httpPath: "/v2/tags/" + tt.containerRef, } m.Run() defer m.Stop() c, err := NewClient(&Config{AuthToken: testToken, BaseURL: m.baseURI}) if err != nil { t.Errorf("Error initializing client: %v", err) } err = c.setTagsV2(context.Background(), tt.containerRef, tt.imageRef, tt.arch, tt.tags) if err != nil && !tt.expectError { t.Errorf("Unexpected error: %v", err) } if err == nil && tt.expectError { t.Errorf("Unexpected success. Expected error.") } }) } } container-library-client-1.4.10/client/client.go000066400000000000000000000065031471454260400215730ustar00rootroot00000000000000// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the LICENSE.md file // distributed with the sources of this project regarding your rights to use or distribute this // software. package client import ( "context" "fmt" "io" "net/http" "net/url" "strings" "github.com/go-log/log" ) // Config contains the client configuration. type Config struct { // Base URL of the service. BaseURL string // Auth token to include in the Authorization header of each request (if supplied). AuthToken string // User agent to include in each request (if supplied). UserAgent string // HTTPClient to use to make HTTP requests (if supplied). HTTPClient *http.Client // Logger to be used when output is generated Logger log.Logger } // DefaultConfig is a configuration that uses default values. var DefaultConfig = &Config{} // Client describes the client details. type Client struct { // Base URL of the service. BaseURL *url.URL // Auth token to include in the Authorization header of each request (if supplied). AuthToken string // User agent to include in each request (if supplied). UserAgent string // HTTPClient to use to make HTTP requests. HTTPClient *http.Client // Logger to be used when output is generated Logger log.Logger } const defaultBaseURL = "" // NewClient sets up a new Cloud-Library Service client with the specified base URL and auth token. func NewClient(cfg *Config) (*Client, error) { if cfg == nil { cfg = DefaultConfig } // Determine base URL bu := defaultBaseURL if cfg.BaseURL != "" { bu = cfg.BaseURL } if bu == "" { return nil, errNoBaseURL } // If baseURL has a path component, ensure it is terminated with a separator, to prevent // url.ResolveReference from stripping the final component of the path when constructing // request URL. if !strings.HasSuffix(bu, "/") { bu += "/" } baseURL, err := url.Parse(bu) if err != nil { return nil, err } if baseURL.Scheme != "http" && baseURL.Scheme != "https" { return nil, fmt.Errorf("%w: unsupported protocol scheme %q", errHTTP, baseURL.Scheme) } c := &Client{ BaseURL: baseURL, AuthToken: cfg.AuthToken, UserAgent: cfg.UserAgent, } // Set HTTP client if cfg.HTTPClient != nil { c.HTTPClient = cfg.HTTPClient } else { c.HTTPClient = http.DefaultClient } if cfg.Logger != nil { c.Logger = cfg.Logger } else { c.Logger = log.DefaultLogger } return c, nil } // newRequestWithURL returns a new Request given a method, url, and (optional) body. func (c *Client) newRequestWithURL(ctx context.Context, method, url string, body io.Reader) (*http.Request, error) { r, err := http.NewRequestWithContext(ctx, method, url, body) if err != nil { return nil, err } if v := c.AuthToken; v != "" { if err := (bearerTokenCredentials{authToken: v}).ModifyRequest(r); err != nil { return nil, err } } if v := c.UserAgent; v != "" { r.Header.Set("User-Agent", v) } return r, nil } // newRequest returns a new Request given a method, relative path, rawQuery, and (optional) body. func (c *Client) newRequest(ctx context.Context, method, path, rawQuery string, body io.Reader) (*http.Request, error) { u := c.BaseURL.ResolveReference(&url.URL{ Path: path, RawQuery: rawQuery, }) return c.newRequestWithURL(ctx, method, u.String(), body) } container-library-client-1.4.10/client/client_test.go000066400000000000000000000146221471454260400226330ustar00rootroot00000000000000// Copyright (c) 2019-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the LICENSE.md file // distributed with the sources of this project regarding your rights to use or distribute this // software. package client import ( "context" "io" "net/http" "strings" "testing" ) const testBaseURL = "https://library.example.io" func TestNewClient(t *testing.T) { httpClient := &http.Client{} tests := []struct { name string cfg *Config wantErr bool wantURL string wantAuthToken string wantUserAgent string wantHTTPClient *http.Client }{ {"NilConfig", nil, true, "", "", "", http.DefaultClient}, {"HTTPBaseURL", &Config{ BaseURL: "http://library.staging.sylabs.io", }, false, "http://library.staging.sylabs.io/", "", "", http.DefaultClient}, {"HTTPAlternateBaseURL", &Config{ BaseURL: "http://staging.sylabs.io/library", }, false, "http://staging.sylabs.io/library/", "", "", http.DefaultClient}, {"HTTPSBaseURL", &Config{ BaseURL: "https://library.staging.sylabs.io", }, false, "https://library.staging.sylabs.io/", "", "", http.DefaultClient}, {"HTTPSAlternateBaseURL", &Config{ BaseURL: "https://staging.sylabs.io/library", }, false, "https://staging.sylabs.io/library/", "", "", http.DefaultClient}, {"UnsupportedBaseURL", &Config{ BaseURL: "bad:", }, true, "", "", "", nil}, {"BadBaseURL", &Config{ BaseURL: ":", }, true, "", "", "", nil}, {"AuthToken", &Config{ AuthToken: "blah", BaseURL: testBaseURL, }, false, testBaseURL + "/", "blah", "", http.DefaultClient}, {"UserAgent", &Config{ UserAgent: "Secret Agent Man", BaseURL: testBaseURL, }, false, testBaseURL + "/", "", "Secret Agent Man", http.DefaultClient}, {"HTTPClient", &Config{ HTTPClient: httpClient, BaseURL: testBaseURL, }, false, testBaseURL + "/", "", "", httpClient}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c, err := NewClient(tt.cfg) if (err != nil) != tt.wantErr { t.Fatalf("got err %v, want %v", err, tt.wantErr) } if err == nil { if got, want := c.BaseURL.String(), tt.wantURL; got != want { t.Errorf("got host %v, want %v", got, want) } if got, want := c.AuthToken, tt.wantAuthToken; got != want { t.Errorf("got auth token %v, want %v", got, want) } if got, want := c.UserAgent, tt.wantUserAgent; got != want { t.Errorf("got user agent %v, want %v", got, want) } if got, want := c.HTTPClient, tt.wantHTTPClient; got != want { t.Errorf("got HTTP client %v, want %v", got, want) } } }) } } func TestNewRequest(t *testing.T) { testConfig := &Config{ BaseURL: testBaseURL, } tests := []struct { name string cfg *Config method string path string rawQuery string body string wantErr bool wantURL string wantAuthBearer string wantUserAgent string }{ {"BadMethod", testConfig, "b@d ", "", "", "", true, "", "", ""}, {"NilConfigGet", testConfig, http.MethodGet, "/path", "", "", false, testBaseURL + "/path", "", ""}, {"NilConfigPost", testConfig, http.MethodPost, "/path", "", "", false, testBaseURL + "/path", "", ""}, {"NilConfigPostRawQuery", testConfig, http.MethodPost, "/path", "a=b", "", false, testBaseURL + "/path?a=b", "", ""}, {"NilConfigPostBody", testConfig, http.MethodPost, "/path", "", "body", false, testBaseURL + "/path", "", ""}, {"HTTPBaseURL", &Config{ BaseURL: "http://library.staging.sylabs.io", }, http.MethodGet, "path", "", "", false, "http://library.staging.sylabs.io/path", "", ""}, {"HTTPAlternateBaseURL", &Config{ BaseURL: "http://staging.sylabs.io/library", }, http.MethodGet, "path", "", "", false, "http://staging.sylabs.io/library/path", "", ""}, {"HTTPSBaseURL", &Config{ BaseURL: "https://library.staging.sylabs.io", }, http.MethodGet, "path", "", "", false, "https://library.staging.sylabs.io/path", "", ""}, {"HTTPSAlternateBaseURL", &Config{ BaseURL: "https://staging.sylabs.io/library", }, http.MethodGet, "path", "", "", false, "https://staging.sylabs.io/library/path", "", ""}, {"BaseURLWithPath", &Config{ BaseURL: "https://library.staging.sylabs.io/path1", }, http.MethodGet, "path2", "", "", false, "https://library.staging.sylabs.io/path1/path2", "", ""}, {"AuthToken", &Config{ AuthToken: "blah", BaseURL: testBaseURL, }, http.MethodGet, "/path", "", "", false, testBaseURL + "/path", "BEARER blah", ""}, {"UserAgent", &Config{ UserAgent: "Secret Agent Man", BaseURL: testBaseURL, }, http.MethodGet, "/path", "", "", false, testBaseURL + "/path", "", "Secret Agent Man"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c, err := NewClient(tt.cfg) if err != nil { t.Fatalf("failed to create client: %v", err) } r, err := c.newRequest(context.Background(), tt.method, tt.path, tt.rawQuery, strings.NewReader(tt.body)) if (err != nil) != tt.wantErr { t.Fatalf("got err %v, wantErr %v", err, tt.wantErr) } if err == nil { if got, want := r.Method, tt.method; got != want { t.Errorf("got method %v, want %v", got, want) } if got, want := r.URL.String(), tt.wantURL; got != want { t.Errorf("got URL %v, want %v", got, want) } b, err := io.ReadAll(r.Body) if err != nil { t.Errorf("failed to read body: %v", err) } if got, want := string(b), tt.body; got != want { t.Errorf("got body %v, want %v", got, want) } authBearer, ok := r.Header["Authorization"] if got, want := ok, (tt.wantAuthBearer != ""); got != want { t.Fatalf("presence of auth bearer %v, want %v", got, want) } if ok { if got, want := len(authBearer), 1; got != want { t.Fatalf("got %v auth bearer(s), want %v", got, want) } if got, want := authBearer[0], tt.wantAuthBearer; !strings.EqualFold(got, want) { t.Errorf("got auth bearer %v, want %v", got, want) } } userAgent, ok := r.Header["User-Agent"] if got, want := ok, (tt.wantUserAgent != ""); got != want { t.Fatalf("presence of user agent %v, want %v", got, want) } if ok { if got, want := len(userAgent), 1; got != want { t.Fatalf("got %v user agent(s), want %v", got, want) } if got, want := userAgent[0], tt.wantUserAgent; got != want { t.Errorf("got user agent %v, want %v", got, want) } } } }) } } container-library-client-1.4.10/client/delete.go000066400000000000000000000005341471454260400215550ustar00rootroot00000000000000package client import ( "context" "net/url" ) // DeleteImage deletes requested imageRef. func (c *Client) DeleteImage(ctx context.Context, imageRef, arch string) error { if imageRef == "" || arch == "" { return errImageRefArchRequired } _, err := c.doDeleteRequest(ctx, "v1/images/"+imageRef+"?arch="+url.QueryEscape(arch)) return err } container-library-client-1.4.10/client/delete_test.go000066400000000000000000000027071471454260400226200ustar00rootroot00000000000000package client import ( "context" "fmt" "net/http" "testing" ) func Test_DeleteImage(t *testing.T) { tests := []struct { name string imageRef string arch string expectError bool code int callback func(*http.Request, *testing.T) }{ { name: "MissingImageRefAndArch", expectError: true, }, { name: "MissingImageRef", arch: archIntel, expectError: true, }, { name: "MissingArch", imageRef: "test:v0.0.1", expectError: true, }, { name: "ValidateQueryStringArch", imageRef: "test", arch: archIntel, code: http.StatusOK, callback: func(r *http.Request, t *testing.T) { // ensure arch specified in query string is as expected queryArch := r.URL.Query().Get("arch") if queryArch != archIntel { t.Errorf("got arch %v, want %v", queryArch, archIntel) } }, }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { m := mockService{ t: t, code: tt.code, httpPath: fmt.Sprintf("/v1/images/%s", tt.imageRef), reqCallback: tt.callback, } m.Run() defer m.Stop() c, err := NewClient(&Config{BaseURL: m.baseURI}) if err != nil { t.Errorf("Error initializing client: %v", err) } err = c.DeleteImage(context.Background(), tt.imageRef, tt.arch) if !tt.expectError && err != nil { t.Errorf("Unexpected err: %s", err) } }) } } container-library-client-1.4.10/client/downloader.go000066400000000000000000000072001471454260400224460ustar00rootroot00000000000000// Copyright (c) 2021-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( "context" "fmt" "io" "net/http" "strconv" "strings" "golang.org/x/sync/errgroup" ) // filePartDescriptor defines one part of multipart download. type filePartDescriptor struct { start int64 end int64 cur int64 w io.WriterAt } // Write writes buffer 'p' at offset 'start' using 'WriteAt()' to atomically seek and write. // Returns bytes written func (ps *filePartDescriptor) Write(p []byte) (n int, err error) { n, err = ps.w.WriteAt(p, ps.start+ps.cur) ps.cur += int64(n) return } // minInt64 returns minimum value of two arguments func minInt64(a, b int64) int64 { if a < b { return a } return b } // Download performs download of contents at url by writing 'size' bytes to 'dst' using credentials 'c'. func (c *Client) multipartDownload(ctx context.Context, u string, creds credentials, w io.WriterAt, size int64, spec *Downloader, pb ProgressBar) error { if size <= 0 { return fmt.Errorf("%w: invalid image size (%v)", errBadRequest, size) } // Initialize the progress bar using passed size pb.Init(size) // Clean up (remove) progress bar after download defer pb.Wait() // Calculate # of parts parts := uint(1 + (size-1)/spec.PartSize) c.Logger.Logf("size: %d, parts: %d, streams: %d, partsize: %d", size, parts, spec.Concurrency, spec.PartSize) g, ctx := errgroup.WithContext(ctx) // Allocate channel for file part requests ch := make(chan filePartDescriptor, parts) // Create download part workers for n := uint(0); n < spec.Concurrency; n++ { g.Go(c.ociDownloadWorker(ctx, u, creds, ch, pb)) } // Add part download requests for n := uint(0); n < parts; n++ { partSize := minInt64(spec.PartSize, size-int64(n)*spec.PartSize) ch <- filePartDescriptor{start: int64(n) * spec.PartSize, end: int64(n)*spec.PartSize + partSize - 1, w: w} } // Close worker queue after submitting all requests close(ch) // Wait for workers to complete return g.Wait() } func (c *Client) ociDownloadWorker(ctx context.Context, u string, creds credentials, ch chan filePartDescriptor, pb ProgressBar) func() error { return func() error { // Iterate on channel 'ch' to handle download part requests for ps := range ch { written, err := c.ociDownloadBlobPart(ctx, creds, u, &ps) if err != nil { // Cleanly abort progress bar on error pb.Abort(true) return err } // Increase progress bar by number of bytes downloaded/written pb.IncrBy(int(written)) } return nil } } func (c *Client) ociDownloadBlobPart(ctx context.Context, creds credentials, u string, ps *filePartDescriptor) (int64, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) if err != nil { return 0, err } if creds != nil { if err := creds.ModifyRequest(req); err != nil { return 0, err } } req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", ps.start, ps.end)) res, err := c.HTTPClient.Do(req) if err != nil { return 0, err } defer res.Body.Close() return io.Copy(ps, res.Body) } // parseContentRange parses "Content-Range" header (eg. "Content-Range: bytes 0-1000/2000") and returns size func parseContentRange(val string) (int64, error) { e := strings.Split(val, " ") if !strings.EqualFold(e[0], "bytes") { return 0, errUnexpectedMalformedValue } rangeElems := strings.Split(e[1], "/") if len(rangeElems) != 2 { return 0, errUnexpectedMalformedValue } return strconv.ParseInt(rangeElems[1], 10, 0) } container-library-client-1.4.10/client/downloader_test.go000066400000000000000000000074451471454260400235200ustar00rootroot00000000000000// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( "bytes" "context" "fmt" "io" "log" "net/http" "net/http/httptest" "strconv" "strings" "sync" "testing" ) type inMemoryBuffer struct { m sync.Mutex buf []byte } func (f *inMemoryBuffer) WriteAt(p []byte, ofs int64) (n int, err error) { f.m.Lock() defer f.m.Unlock() n = copy(f.buf[ofs:], p) return } func (f *inMemoryBuffer) Bytes() []byte { f.m.Lock() defer f.m.Unlock() return f.buf } type stdLogger struct{} func (l *stdLogger) Log(v ...interface{}) { log.Print(v...) } func (l *stdLogger) Logf(f string, v ...interface{}) { log.Printf(f, v...) } func parseRangeHeader(t *testing.T, val string) (int64, int64) { t.Helper() if val == "" { return 0, 0 } var start, end int64 e := strings.SplitN(val, "=", 2) byteRange := strings.Split(e[1], "-") start, _ = strconv.ParseInt(byteRange[0], 10, 0) end, _ = strconv.ParseInt(byteRange[1], 10, 0) return start, end } const ( basicAuthUsername = "user" basicAuthPassword = "password" ) var ( testLogger = &stdLogger{} creds = &basicCredentials{username: basicAuthUsername, password: basicAuthPassword} ) func TestMultistreamDownloader(t *testing.T) { const src = "123456789012345678901234567890" size := int64(len(src)) defaultSpec := &Downloader{Concurrency: 10, PartSize: 3} tests := []struct { name string size int64 spec *Downloader expectErr bool }{ {"Basic", size, defaultSpec, false}, {"WithoutSize", 0, defaultSpec, true}, {"SingleStream", size, &Downloader{Concurrency: 1, PartSize: 1}, false}, {"SingleStreamWithoutSize", 0, &Downloader{Concurrency: 1, PartSize: 1}, true}, {"ManyStreams", size, &Downloader{Concurrency: uint(size), PartSize: 1}, false}, {"ManyStreamsWithoutSize", 0, &Downloader{Concurrency: uint(size), PartSize: 1}, true}, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() // Create test http server for serving "file" srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { start, end := parseRangeHeader(t, r.Header.Get("Range")) if username, password, ok := r.BasicAuth(); ok { if got, want := username, basicAuthUsername; got != want { t.Fatalf("unexpected basic auth username: got %v, want %v", got, want) } if got, want := password, basicAuthPassword; got != want { t.Fatalf("unexpected basic auth password: got %v, want %v", got, want) } } w.Header().Set("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, end+1, size)) w.Header().Set("Content-Length", fmt.Sprintf("%v", end-start+1)) w.WriteHeader(http.StatusPartialContent) if _, err := io.Copy(w, bytes.NewReader([]byte(src[start:end+1]))); err != nil { t.Fatalf("unexpected error writing http response: %v", err) } })) defer srv.Close() c, err := NewClient(&Config{BaseURL: srv.URL, Logger: testLogger}) if err != nil { t.Fatalf("error initializing client: %v", err) } // Preallocate sink for downloaded file dst := &inMemoryBuffer{buf: make([]byte, size)} // Start download err = c.multipartDownload(context.Background(), srv.URL, creds, dst, tt.size, tt.spec, &NoopProgressBar{}) if tt.expectErr && err == nil { t.Fatal("unexpected success") } if !tt.expectErr && err != nil { t.Fatalf("unexpected error: %v", err) } if err != nil { return } // Compare results with expectations if got, want := string(dst.Bytes()), src; got != want { t.Fatalf("unexpected data: got %v, want %v", got, want) } }) } } container-library-client-1.4.10/client/errors.go000066400000000000000000000041271471454260400216310ustar00rootroot00000000000000package client import "errors" var ( errHTTP = errors.New("http error") // ErrUnauthorized represents HTTP status "401 Unauthorized" ErrUnauthorized = errors.New("unauthorized") errImageRefArchRequired = errors.New("imageRef and arch are required") errBadRequest = errors.New("bad request") errUnexpectedMalformedValue = errors.New("unexpected/malformed value") errOCIRegistry = errors.New("OCI registry error") errArchNotSpecified = errors.New("architecture not specified") errInvalidAuthHeader = errors.New("invalid auth header") errResetHTTPBody = errors.New("unable to reset HTTP request body") errArchitectureNotPresent = errors.New("architecture not present") errDigestNotVerified = errors.New("digest not verified") errOCIDownloadNotSupported = errors.New("not supported") errGettingPresignedURL = errors.New("error getting presigned URL") errParsingPresignedURL = errors.New("error parsing presigned URL") errNoBaseURL = errors.New("no BaseURL supplied") // ErrRefSchemeNotValid represents a ref with an invalid scheme. ErrRefSchemeNotValid = errors.New("library: ref scheme not valid") // ErrRefUserNotPermitted represents a ref with an invalid user. ErrRefUserNotPermitted = errors.New("library: user not permitted in ref") // ErrRefQueryNotPermitted represents a ref with an invalid query. ErrRefQueryNotPermitted = errors.New("library: query not permitted in ref") // ErrRefFragmentNotPermitted represents a ref with an invalid fragment. ErrRefFragmentNotPermitted = errors.New("library: fragment not permitted in ref") // ErrRefPathNotValid represents a ref with an invalid path. ErrRefPathNotValid = errors.New("library: ref path not valid") // ErrRefTagsNotValid represents a ref with invalid tags. ErrRefTagsNotValid = errors.New("library: ref tags not valid") // ErrNotFound is returned by when a resource is not found (http status 404) ErrNotFound = errors.New("not found") errQueryValueMustBeSpecified = errors.New("search query ('value') must be specified") ) container-library-client-1.4.10/client/models.go000066400000000000000000000203421471454260400215750ustar00rootroot00000000000000// Copyright (c) 2018, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( "sort" "strings" "time" ) // LibraryModels lists names of valid models in the database var LibraryModels = []string{"Entity", "Collection", "Container", "Image", "Blob"} // ModelManager - Generic interface for models which must have a bson ObjectID type ModelManager interface { GetID() string } // BaseModel - has an ID, soft deletion marker, and Audit struct type BaseModel struct { ModelManager `json:",omitempty"` Deleted bool `json:"deleted"` CreatedBy string `json:"createdBy"` CreatedAt time.Time `json:"createdAt"` UpdatedBy string `json:"updatedBy,omitempty"` UpdatedAt time.Time `json:"updatedAt,omitempty"` DeletedBy string `json:"deletedBy,omitempty"` DeletedAt time.Time `json:"deletedAt,omitempty"` Owner string `json:"owner,omitempty"` } // IsDeleted - Convenience method to check soft deletion state if working with // an interface func (m BaseModel) IsDeleted() bool { return m.Deleted } // GetCreated - Convenience method to get creation stamps if working with an // interface func (m BaseModel) GetCreated() (auditUser string, auditTime time.Time) { return m.CreatedBy, m.CreatedAt } // GetUpdated - Convenience method to get update stamps if working with an // interface func (m BaseModel) GetUpdated() (auditUser string, auditTime time.Time) { return m.UpdatedBy, m.UpdatedAt } // GetDeleted - Convenience method to get deletino stamps if working with an // interface func (m BaseModel) GetDeleted() (auditUser string, auditTime time.Time) { return m.DeletedBy, m.DeletedAt } // Check BaseModel implements ModelManager at compile time var _ ModelManager = (*BaseModel)(nil) // Entity - Top level entry in the library, contains collections of images // for a user or group type Entity struct { BaseModel ID string `json:"id"` Name string `json:"name"` Description string `json:"description"` Collections []string `json:"collections"` Size int64 `json:"size"` Quota int64 `json:"quota"` // DefaultPrivate set true will make any new Collections in this entity // private at the time of creation. DefaultPrivate bool `json:"defaultPrivate"` // CustomData can hold a user-provided string for integration purposes // not used by the library itself. CustomData string `json:"customData"` } // GetID - Convenience method to get model ID if working with an interface func (e Entity) GetID() string { return e.ID } // LibraryURI - library:// URI to the entity func (e Entity) LibraryURI() string { return "library://" + e.Name } // Collection - Second level in the library, holds a collection of containers type Collection struct { BaseModel ID string `json:"id"` Name string `json:"name"` Description string `json:"description"` Entity string `json:"entity"` Containers []string `json:"containers"` Size int64 `json:"size"` Private bool `json:"private"` // CustomData can hold a user-provided string for integration purposes // not used by the library itself. CustomData string `json:"customData"` // Computed fields that will not be stored - JSON response use only EntityName string `json:"entityName,omitempty"` } // GetID - Convenience method to get model ID if working with an interface func (c Collection) GetID() string { return c.ID } // LibraryURI - library:// URI to the collection func (c Collection) LibraryURI() string { return "library://" + c.EntityName + "/" + c.Name } // Container - Third level of library. Inside a collection, holds images for // a particular container type Container struct { BaseModel ID string `json:"id"` Name string `json:"name"` Description string `json:"description"` FullDescription string `json:"fullDescription"` Collection string `json:"collection"` Images []string `json:"images"` // This base TagMap without architecture support is for old clients only // (Singularity <=3.3) to preserve non-architecture-aware behavior ImageTags TagMap `json:"imageTags"` // We now have a 2 level map for new clients, keeping tags per architecture ArchTags ArchTagMap `json:"archTags"` Size int64 `json:"size"` DownloadCount int64 `json:"downloadCount"` Stars int `json:"stars"` Private bool `json:"private"` ReadOnly bool `json:"readOnly"` // CustomData can hold a user-provided string for integration purposes // not used by the library itself. CustomData string `json:"customData"` // Computed fields that will not be stored - JSON response use only Entity string `json:"entity,omitempty"` EntityName string `json:"entityName,omitempty"` CollectionName string `json:"collectionName,omitempty"` } // GetID - Convenience method to get model ID if working with an interface func (c Container) GetID() string { return c.ID } // LibraryURI - library:// URI to the container func (c Container) LibraryURI() string { return "library://" + c.EntityName + "/" + c.CollectionName + "/" + c.Name } // TagList - return a sorted space delimited list of tags func (c Container) TagList() string { var taglist sort.StringSlice for tag := range c.ImageTags { taglist = append(taglist, tag) } taglist.Sort() return strings.Join(taglist, " ") } // Image - Represents a Singularity image held by the library for a particular // Container type Image struct { BaseModel ID string `json:"id"` Hash string `json:"hash"` Description string `json:"description"` Container string `json:"container"` Blob string `json:"blob,omitempty"` Size int64 `json:"size"` Uploaded bool `json:"uploaded"` Signed *bool `json:"signed,omitempty"` Architecture *string `json:"arch,omitempty"` Fingerprints []string `json:"fingerprints,omitempty"` Encrypted *bool `json:"encrypted,omitempty"` // CustomData can hold a user-provided string for integration purposes // not used by the library itself. CustomData string `json:"customData"` // Computed fields that will not be stored - JSON response use only Entity string `json:"entity,omitempty"` EntityName string `json:"entityName,omitempty"` Collection string `json:"collection,omitempty"` CollectionName string `json:"collectionName,omitempty"` ContainerName string `json:"containerName,omitempty"` Tags []string `json:"tags,omitempty"` ContainerDescription string `json:"containerDescription,omitempty"` ContainerStars int `json:"containerStars"` ContainerDownloads int64 `json:"containerDownloads"` } // GetID - Convenience method to get model ID if working with an interface func (img Image) GetID() string { return img.ID } // Blob - Binary data object (e.g. container image file) stored in a Backend // Uses object store bucket/key semantics type Blob struct { BaseModel ID string `json:"id"` Bucket string `json:"bucket"` Key string `json:"key"` Size int64 `json:"size"` ContentHash string `json:"contentHash"` Status string `json:"status"` } // GetID - Convenience method to get model ID if working with an interface func (b Blob) GetID() string { return b.ID } // ImageTag - A single mapping from a string to bson ID. Not stored in the DB // but used by API calls setting tags type ImageTag struct { Tag string ImageID string } // TagMap is a mapping of a string tag, to an ObjectID that refers to an Image // e.g. { "latest": 507f1f77bcf86cd799439011 } type TagMap map[string]string // ArchImageTag - A simple mapping from a architecture and tag string to bson // ID. Not stored in the DB but used by API calls setting tags type ArchImageTag struct { Arch string Tag string ImageID string } // ArchTagMap is a mapping of a string architecture to a TagMap, and hence to // Images. // // e.g. { // "amd64": { "latest": 507f1f77bcf86cd799439011 }, // "ppc64le": { "latest": 507f1f77bcf86cd799439012 }, // } type ArchTagMap map[string]TagMap container-library-client-1.4.10/client/oci.go000066400000000000000000000734451471454260400211000ustar00rootroot00000000000000// Copyright (c) 2018-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( "bytes" "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "strconv" "strings" "time" "github.com/apptainer/sif/v2/pkg/sif" "github.com/go-log/log" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/specs-go" v1 "github.com/opencontainers/image-spec/specs-go/v1" ) const mediaTypeSIFLayer = "application/vnd.sylabs.sif.layer.v1.sif" // ociRegistryAuth uses Cloud Library endpoint to determine if artifact can be pulled // directly from OCI registry. // // Returns url, credentials (if applicable) for that url, and mapped name. // // The mapped name can be the same value as 'name' or mapped to a fully-qualified name // (ie. from "alpine" to "library/default/alpine") if supported by cloud library server. // It will never be an empty string ("") func (c *Client) ociRegistryAuth(ctx context.Context, name string, accessTypes []accessType) (*url.URL, *bearerTokenCredentials, string, error) { // Build raw query string to get token for specified namespace and access v := url.Values{} v.Set("namespace", name) // Setting 'mapped' to '1' (true) enables support for mapping short library refs to // fully-qualified name v.Set("mapped", strconv.Itoa(1)) ats := make([]string, 0, len(accessTypes)) for _, at := range accessTypes { ats = append(ats, string(at)) } v.Set("accessTypes", strings.Join(ats, ",")) req, err := c.newRequest(ctx, http.MethodGet, "v1/oci-redirect", v.Encode(), nil) if err != nil { return nil, nil, "", err } if c.UserAgent != "" { req.Header.Set("User-Agent", c.UserAgent) } res, err := c.HTTPClient.Do(req) if err != nil { return nil, nil, "", fmt.Errorf("error determining direct OCI registry access: %w", err) } defer res.Body.Close() if res.StatusCode != http.StatusOK { return nil, nil, "", fmt.Errorf("error determining direct OCI registry access: %w", err) } type ociDownloadRedirectResponse struct { Token string `json:"token"` RegistryURI string `json:"url"` Name string `json:"name"` } var ociArtifactSpec ociDownloadRedirectResponse if err := json.NewDecoder(res.Body).Decode(&ociArtifactSpec); err != nil { return nil, nil, "", fmt.Errorf("error decoding direct OCI registry access response: %w", err) } if ociArtifactSpec.Name != "" && ociArtifactSpec.Name != name { name = ociArtifactSpec.Name } endpoint, err := url.Parse(ociArtifactSpec.RegistryURI) if err != nil { return nil, nil, "", fmt.Errorf("malformed OCI registry URI %v: %w", ociArtifactSpec.RegistryURI, err) } return endpoint, &bearerTokenCredentials{authToken: ociArtifactSpec.Token}, name, nil } const ( mediaTypeSIFConfig = "application/vnd.sylabs.sif.config.v1+json" ) type imageConfig struct { Architecture string `json:"architecture"` OS string `json:"os"` RootFS digest.Digest `json:"rootfs"` Description string `json:"description,omitempty"` Signed bool `json:"signed"` Encrypted bool `json:"encrypted"` } type credentials interface { ModifyRequest(r *http.Request, opts ...modifyRequestOption) error } type basicCredentials struct { username string password string } func (c basicCredentials) ModifyRequest(r *http.Request, _ ...modifyRequestOption) error { r.SetBasicAuth(c.username, c.password) return nil } type bearerTokenCredentials struct { authToken string } func (c bearerTokenCredentials) ModifyRequest(r *http.Request, _ ...modifyRequestOption) error { if c.authToken != "" { r.Header.Set("Authorization", fmt.Sprintf("Bearer %v", c.authToken)) } return nil } type accessType string const ( accessTypePull accessType = "pull" accessTypePush accessType = "push" ) type accessOptions struct { namespace string accessTypes []accessType } type ociRegistry struct { baseURL *url.URL httpClient *http.Client userAgent string logger log.Logger } func (r *ociRegistry) getManifestFromIndex(idx v1.Index, arch string) (digest.Digest, error) { // If arch not supplied, return single manifest or error. if arch == "" { if len(idx.Manifests) != 1 || idx.Manifests[0].MediaType != v1.MediaTypeImageManifest { return "", errArchNotSpecified } return idx.Manifests[0].Digest, nil } // Otherwise, go fish for matching architecture/OS. for _, m := range idx.Manifests { // Only consider image manifests. if m.MediaType != v1.MediaTypeImageManifest { continue } // If arch matches, execute! if m.Platform.Architecture == arch { return m.Digest, nil } } // If we make it here, no matching OS/architecture was found. return "", fmt.Errorf("%w: no matching OS/architecture (%v) found", errOCIRegistry, arch) } func (r *ociRegistry) getImageManifest(ctx context.Context, creds credentials, name, tag, arch string) (digest.Digest, v1.Manifest, error) { if _, idx, err := r.DownloadV1Index(ctx, creds, name, tag); err == nil { // Get manifest from index d, err := r.getManifestFromIndex(idx, arch) if err != nil { return "", v1.Manifest{}, err } tag = d.String() } return r.downloadV1Manifest(ctx, creds, name, tag) } func (r *ociRegistry) getImageDetails(ctx context.Context, creds credentials, name, tag, arch string) (v1.Descriptor, error) { _, m, err := r.getImageManifest(ctx, creds, name, tag, arch) if err != nil { return v1.Descriptor{}, err } if got, want := m.Config.MediaType, mediaTypeSIFConfig; got != want { return v1.Descriptor{}, fmt.Errorf("%w: unexpected media type error (got %v, want %v)", errOCIRegistry, got, want) } // There should always be exactly one layer (the image blob). if n := len(m.Layers); n != 1 { return v1.Descriptor{}, fmt.Errorf("%w: unexpected # of layers: %v", errOCIRegistry, n) } // If architecture was supplied, ensure the image config matches. ic, err := r.getImageConfig(ctx, creds, name, m.Config.Digest) if err != nil { return v1.Descriptor{}, err } // Ensure architecture matches, if supplied. if got, want := ic.Architecture, arch; want != "" && got != want { return v1.Descriptor{}, &unexpectedArchitectureError{got, want} } return m.Layers[0], nil } func (r *ociRegistry) DownloadV1Index(ctx context.Context, creds credentials, name, tag string) (digest.Digest, v1.Index, error) { var idx v1.Index d, err := r.downloadManifest(ctx, creds, name, tag, &idx, v1.MediaTypeImageIndex) return d, idx, err } func (r *ociRegistry) downloadV1Manifest(ctx context.Context, creds credentials, name, tag string) (digest.Digest, v1.Manifest, error) { var m v1.Manifest d, err := r.downloadManifest(ctx, creds, name, tag, &m, v1.MediaTypeImageManifest) return d, m, err } func (r *ociRegistry) newRequest(ctx context.Context, method string, u *url.URL, body io.Reader) (*http.Request, error) { return http.NewRequestWithContext(ctx, method, r.baseURL.ResolveReference(u).String(), body) } type modifyRequestOptions struct { httpClient *http.Client userAgent string authenticateHeader authHeader // Parsed "Www-Authenticate" header accessOptions *accessOptions } type modifyRequestOption func(*modifyRequestOptions) error // withAuthenticateHeader specifies s as the value of the "Www-Authenticate" header. func withAuthenticateHeader(s string) modifyRequestOption { return func(opts *modifyRequestOptions) error { ah, err := parseAuthHeader(s) if err != nil { return err } opts.authenticateHeader = ah return nil } } type authType int const ( authTypeUnknown authType = iota authTypeBasic authTypeBearer ) // parseList parses a comma-separated list of values as described by RFC 2068 and returns list // elements. // // Lifted from https://code.google.com/p/gorilla/source/browse/http/parser/parser.go func parseList(value string) []string { var list []string var escape, quote bool b := bytes.Buffer{} for _, r := range value { switch { case escape: b.WriteRune(r) escape = false case quote: if r == '\\' { escape = true } else { if r == '"' { quote = false } b.WriteRune(r) } case r == ',': list = append(list, strings.TrimSpace(b.String())) b.Reset() case r == '"': quote = true b.WriteRune(r) default: b.WriteRune(r) } } // Append last part. if s := b.String(); s != "" { list = append(list, strings.TrimSpace(s)) } return list } // parsePairs extracts key/value pairs from a comma-separated list of values as described by RFC // 2068 and returns a map[key]value. The resulting values are unquoted. If a list element doesn't // contain a "=", the key is the element itself and the value is an empty string. // // Lifted from https://code.google.com/p/gorilla/source/browse/http/parser/parser.go func parsePairs(value string) map[string]string { m := make(map[string]string) for _, pair := range parseList(strings.TrimSpace(value)) { if i := strings.Index(pair, "="); i < 0 { m[pair] = "" } else { v := pair[i+1:] if v[0] == '"' && v[len(v)-1] == '"' { // Unquote it. v = v[1 : len(v)-1] } m[pair[:i]] = v } } return m } type authHeader struct { at authType realm string service string scope string } type unknownAuthTypeError struct { authType string } func (e *unknownAuthTypeError) Error() string { if e.authType != "" { return fmt.Sprintf("unknown auth type '%v'", e.authType) } return "unknown auth type" } func (e *unknownAuthTypeError) Is(target error) bool { var t *unknownAuthTypeError if errors.As(target, &t) { return t.authType == "" || e.authType == t.authType } return false } func getAuthType(raw string) (authType, error) { switch { case strings.EqualFold(raw, "basic"): return authTypeBasic, nil case strings.EqualFold(raw, "bearer"): return authTypeBearer, nil default: return authTypeUnknown, &unknownAuthTypeError{raw} } } func parseAuthHeader(authenticateHeader string) (authHeader, error) { parts := strings.SplitN(authenticateHeader, " ", 2) if len(parts) != 2 { return authHeader{}, fmt.Errorf("%w: %v", errInvalidAuthHeader, authenticateHeader) } authType, err := getAuthType(parts[0]) if err != nil { return authHeader{}, err } ah := authHeader{at: authType} pairs := parsePairs(parts[1]) if v, ok := pairs["realm"]; ok { ah.realm = v } if v, ok := pairs["service"]; ok { ah.service = v } if v, ok := pairs["scope"]; ok { ah.scope = v } return ah, nil } type noneCreds struct{} func (c *noneCreds) ModifyRequest(r *http.Request, _ ...modifyRequestOption) error { r.Header.Set("Authorization", "none") return nil } // none returns Credentials that set the authorization header to "none". func none() *noneCreds { return &noneCreds{} } func withUserAgent(s string) modifyRequestOption { return func(o *modifyRequestOptions) error { o.userAgent = s return nil } } func withHTTPClient(client *http.Client) modifyRequestOption { return func(o *modifyRequestOptions) error { o.httpClient = client return nil } } func (r *ociRegistry) doRequestWithCredentials(req *http.Request, creds credentials, opts ...modifyRequestOption) (*http.Response, error) { opts = append(opts, withUserAgent(r.userAgent), withHTTPClient(r.httpClient), ) // Modify request to include credentials. if err := creds.ModifyRequest(req, opts...); err != nil { return nil, err } res, err := r.httpClient.Do(req) if err != nil { return nil, err } if code := res.StatusCode; code/100 != 2 { defer res.Body.Close() if code == http.StatusUnauthorized { return nil, ErrUnauthorized } return nil, fmt.Errorf("%w: unexpected http status %v", errHTTP, res.StatusCode) } return res, nil } func (r *ociRegistry) retryRequestWithCredentials(req *http.Request, creds credentials, opts ...modifyRequestOption) (*http.Response, error) { // If the original request contained a body, we need to reset it. if req.Body != nil { if req.GetBody == nil { return nil, errResetHTTPBody } rc, err := req.GetBody() if err != nil { return nil, err } req.Body = rc } return r.doRequestWithCredentials(req, creds, opts...) } func (r *ociRegistry) doRequest(req *http.Request, creds credentials, opts ...modifyRequestOption) (*http.Response, error) { res, err := r.httpClient.Do(req) if err != nil { return nil, err } if code := res.StatusCode; code/100 != 2 { defer res.Body.Close() // If authorization required, re-attempt request using credentials (if supplied) according // to the contents of the "WWW-Authenticate" header (if present). if code == http.StatusUnauthorized { if creds == nil { // Unauthenticated requests to certain Harbor APIs require an Authorization header, // even if it's set to "none". 🤦 creds = none() } opts = append(opts, withAuthenticateHeader(res.Header.Get("WWW-Authenticate"))) return r.retryRequestWithCredentials(req, creds, opts...) } if code == http.StatusUnauthorized { return nil, ErrUnauthorized } return nil, fmt.Errorf("%w: unexpected http status %v", errHTTP, code) } return res, nil } // withNamespaceAccess specifies that credentials must be procured that are sufficient to grant the // access specified by accessTypes to namespace name. func withNamespaceAccess(name string, accessTypes ...accessType) modifyRequestOption { return func(opts *modifyRequestOptions) error { opts.accessOptions = &accessOptions{ namespace: name, accessTypes: accessTypes, } return nil } } type unexpectedContentTypeError struct { got string want string } func (e *unexpectedContentTypeError) Error() string { return fmt.Sprintf("unexpected content type: got %v, want %v", e.got, e.want) } // downloadManifest downloads the manifest of type contentType associated with name/ref in the // registry, and unmarshals it to v. func (r *ociRegistry) downloadManifest(ctx context.Context, creds credentials, name, tag string, v interface{}, contentType string) (digest.Digest, error) { req, err := r.newRequest(ctx, http.MethodGet, &url.URL{Path: fmt.Sprintf("v2/%v/manifests/%v", name, tag)}, nil) if err != nil { return "", err } req.Header.Set("Accept", contentType) res, err := r.doRequest(req, creds, withNamespaceAccess(name, accessTypePull)) if err != nil { return "", err } defer res.Body.Close() // Although we've set the "Accept" header, some registries will return other content types. if got, want := res.Header.Get("Content-Type"), contentType; got != want { return "", &unexpectedContentTypeError{got, want} } d := digest.Digest(res.Header.Get("Docker-Content-Digest")) if err := d.Validate(); err != nil { return "", err } if err := json.NewDecoder(res.Body).Decode(&v); err != nil { return "", err } return d, nil } func (r *ociRegistry) downloadBlob(ctx context.Context, creds credentials, name string, d digest.Digest, rangeValue string, w io.Writer) (int64, error) { if err := d.Validate(); err != nil { return 0, err } req, err := r.newRequest(ctx, http.MethodGet, &url.URL{Path: fmt.Sprintf("v2/%v/blobs/%v", name, d)}, nil) if err != nil { return 0, err } // Set HTTP Range header, if applicable. if rangeValue != "" { req.Header.Set("Range", rangeValue) } res, err := r.doRequest(req, creds, withNamespaceAccess(name, accessTypePull)) if err != nil { return 0, err } defer res.Body.Close() // Download blob. return io.Copy(w, res.Body) } // validateImageConfig validates ic, and returns an error when ic is invalid. func validateImageConfig(ic imageConfig) error { if ic.Architecture == "" { return errArchitectureNotPresent } return ic.RootFS.Validate() } type unexpectedArchitectureError struct { got string want string } func (e *unexpectedArchitectureError) Error() string { return fmt.Sprintf("unexpected image architecture: got %v, want %v", e.got, e.want) } func (e *unexpectedArchitectureError) Is(target error) bool { t := &unexpectedArchitectureError{} if !errors.As(target, &t) { return false } return (e.got == t.got || t.got == "") && (e.want == t.want || t.want == "") } func (r *ociRegistry) getImageConfig(ctx context.Context, creds credentials, name string, d digest.Digest) (imageConfig, error) { var b bytes.Buffer if _, err := r.downloadBlob(ctx, creds, name, d, "", &b); err != nil { return imageConfig{}, err } if digest.FromBytes(b.Bytes()) != d { return imageConfig{}, errDigestNotVerified } var ic imageConfig if err := json.Unmarshal(b.Bytes(), &ic); err != nil { return imageConfig{}, err } if err := validateImageConfig(ic); err != nil { return imageConfig{}, fmt.Errorf("invalid image config: %w", err) } return ic, nil } // newOCIRegistry returns *ociRegistry, credentials for that registry, and the (optionally) remapped image name func (c *Client) newOCIRegistry(ctx context.Context, name string, accessTypes []accessType) (*ociRegistry, *bearerTokenCredentials, string, error) { // Attempt to obtain (direct) OCI registry auth token originalName := name registryURI, creds, name, err := c.ociRegistryAuth(ctx, name, accessTypes) if err != nil { return nil, nil, "", errOCIDownloadNotSupported } // Download directly from OCI registry c.Logger.Logf("Using OCI registry endpoint %v", registryURI) if name != "" && originalName != name { c.Logger.Logf("OCI artifact name \"%v\" mapped to \"%v\"", originalName, name) } return &ociRegistry{baseURL: registryURI, httpClient: c.HTTPClient, logger: c.Logger}, creds, name, nil } func (c *Client) ociDownloadImage(ctx context.Context, arch, name, tag string, w io.WriterAt, spec *Downloader, pb ProgressBar) error { reg, creds, name, err := c.newOCIRegistry(ctx, name, []accessType{accessTypePull}) if err != nil { return err } // Fetch image manifest to get image details id, err := reg.getImageDetails(ctx, creds, name, tag, arch) if err != nil { return fmt.Errorf("error getting image details: %w", err) } imageURI := reg.baseURL.ResolveReference(&url.URL{Path: fmt.Sprintf("v2/%v/blobs/%v", name, id.Digest)}).String() return c.multipartDownload(ctx, imageURI, creds, w, id.Size, spec, pb) } const sifHeaderSize = 32768 type unexpectedImageDigest struct { got digest.Digest want digest.Digest } func (e *unexpectedImageDigest) Error() string { return fmt.Sprintf("unexpected image digest: %v != %v", e.got, e.want) } func (c *Client) ociUploadImage(ctx context.Context, r io.Reader, size int64, name, _ string, tags []string, description, hash string, callback UploadCallback) error { reg, creds, name, err := c.newOCIRegistry(ctx, name, []accessType{accessTypePull, accessTypePush}) if err != nil { return err } sifHeader := bytes.NewBuffer(make([]byte, 0, sifHeaderSize)) // Convert SIF hash to OCI digest. imageDigest := digest.Digest(strings.ReplaceAll(hash, ".", ":")) if err := imageDigest.Validate(); err != nil { return fmt.Errorf("invalid image hash '%v': %w", hash, err) } // Check if image exists, 'ok' is set correctly if this returns an error. ok, _ := reg.existingImageBlob(ctx, creds, name, imageDigest) var id digest.Digest if !ok { // Construct a reader that tees off a copy of the SIF header into a buffer as the blob is uploaded. r = io.MultiReader( io.TeeReader(io.LimitReader(r, sifHeaderSize), sifHeader), r, ) if callback != nil { callback.InitUpload(size, r) r = callback.GetReader() } var err error id, _, err = reg.uploadImageBlob(ctx, creds, name, size, r) if err != nil { if callback != nil { callback.Terminate() } return fmt.Errorf("upload image blob failed: %w", err) } if callback != nil { callback.Finish() } // Verify image blob matches had expected digest. if got, want := id, imageDigest; got != want { return &unexpectedImageDigest{got, want} } } else { c.Logger.Logf("Skipping image blob upload (matching hash exists)") id = imageDigest if _, err := io.Copy(sifHeader, io.LimitReader(r, sifHeaderSize)); err != nil { return fmt.Errorf("error reading local SIF file header: %w", err) } } // Populate image configuration. ic, err := reg.processImageHeader(id, description, sifHeader.Bytes()) if err != nil { return fmt.Errorf("process image failed: %w", err) } cs, cd, err := reg.uploadimageConfig(ctx, creds, name, ic) if err != nil { return fmt.Errorf("upload image config failed: %w", err) } md, err := reg.uploadImageManifest(ctx, creds, name, hash, cd, id, cs, size) if err != nil { return fmt.Errorf("upload image manifest failed: %w", err) } idx := v1.Index{ Versioned: specs.Versioned{SchemaVersion: 2}, } idx.Manifests = append(idx.Manifests, v1.Descriptor{ MediaType: v1.MediaTypeImageManifest, Digest: md, Platform: &v1.Platform{ Architecture: ic.Architecture, OS: ic.OS, }, }) // Add tags for _, ref := range tags { c.Logger.Logf("Tag: %v", ref) if _, err := reg.uploadManifest(ctx, creds, name, ref, idx, v1.MediaTypeImageIndex); err != nil { return fmt.Errorf("error uploading index: %w", err) } } return nil } func (r *ociRegistry) existingImageBlob(ctx context.Context, creds credentials, name string, d digest.Digest) (bool, error) { u := r.baseURL.ResolveReference(&url.URL{Path: fmt.Sprintf("v2/%v/blobs/%v", name, d.String())}) req, err := http.NewRequestWithContext(ctx, http.MethodHead, u.String(), nil) if err != nil { return false, fmt.Errorf("error checking for existing layer: %w", err) } res, err := r.doRequest(req, creds) if err != nil { return false, err } defer res.Body.Close() // TODO: should we validate 'Content-Length' here? return res.StatusCode == http.StatusOK && d.String() == res.Header.Get("Docker-Content-Digest"), nil } // uploadimageConfig uploads ic into namespace name of the registry, using credentials c. // // On success, the config size and digest are returned. func (r *ociRegistry) uploadimageConfig(ctx context.Context, creds credentials, name string, ic imageConfig) (size int64, d digest.Digest, err error) { b, err := json.Marshal(ic) if err != nil { return 0, "", err } log.Logf("Starting image config upload: name=[%v], size=[%v]", name, len(b)) defer func(t time.Time) { log.Logf("Finished image config upload: took=[%v] digest=[%v] err=[%v]", time.Since(t), d.String(), err) }(time.Now()) d, _, err = r.uploadBlob(ctx, creds, name, int64(len(b)), bytes.NewReader(b)) if err != nil { return 0, "", err } return int64(len(b)), d, err } // uploadImageManifest uploads an image manifest to the registry, naming it name:ref. The // corresponding config blob has digest configDigest of size configSize. The corresponding image // blob has digest imageDigest of size imageSize. // // On success, the manifest digest is returned. func (r *ociRegistry) uploadImageManifest(ctx context.Context, creds credentials, name, ref string, configDigest, imageDigest digest.Digest, configSize, imageSize int64) (d digest.Digest, err error) { r.logger.Logf("Starting image manifest upload: name=[%v], ref=[%v]", name, ref) defer func(t time.Time) { r.logger.Logf("Finished image manifest upload: took=[%v] digest=[%v], err=[%v]", time.Since(t), d.String(), err) }(time.Now()) m := v1.Manifest{ Versioned: specs.Versioned{SchemaVersion: 2}, Config: v1.Descriptor{ MediaType: mediaTypeSIFConfig, Digest: configDigest, Size: configSize, }, Layers: []v1.Descriptor{ { MediaType: mediaTypeSIFLayer, Digest: imageDigest, Size: imageSize, }, }, } return r.uploadV1Manifest(ctx, creds, name, ref, m) } func (r *ociRegistry) uploadImageBlob(ctx context.Context, creds credentials, name string, size int64, rd io.Reader) (digest.Digest, int64, error) { return r.uploadBlob(ctx, creds, name, size, rd) } const maxChunkSize int64 = 5 * 1024 * 1024 func (r *ociRegistry) uploadBlob(ctx context.Context, creds credentials, name string, size int64, rd io.Reader) (digest.Digest, int64, error) { u, creds, err := r.openUploadBlobSession(ctx, creds, name) if err != nil { return "", 0, err } // Accumulate digest as we upload chunks. h := digest.Canonical.Hash() tee := io.TeeReader(rd, h) var totalBytesUploaded int64 // Send chunks. for offset := int64(0); offset < size; offset += maxChunkSize { chunkSize := maxChunkSize if offset+chunkSize > size { chunkSize = size - offset // last chunk } if u, err = r.uploadBlobPart(ctx, creds, u, tee, chunkSize, offset); err != nil { return "", 0, err } totalBytesUploaded += chunkSize } d := digest.NewDigest(digest.Canonical, h) if err := r.closeUploadBlobSession(ctx, creds, u, d); err != nil { return "", 0, err } return d, totalBytesUploaded, nil } func (r *ociRegistry) openUploadBlobSession(ctx context.Context, creds credentials, name string) (*url.URL, *bearerTokenCredentials, error) { u := &url.URL{Path: fmt.Sprintf("v2/%v/blobs/uploads/", name)} req, err := r.newRequest(ctx, http.MethodPost, u, nil) if err != nil { return nil, nil, err } res, err := r.doRequest(req, creds, withNamespaceAccess(name, accessTypePush)) if err != nil { return nil, nil, err } defer res.Body.Close() if u, err = getRelativeLocation(res); err != nil { return nil, nil, err } // Strip prefix from Authorization header parts := strings.SplitN(req.Header.Get("Authorization"), " ", 2) if len(parts) != 2 { return nil, nil, fmt.Errorf("%w malformed Authorization header (%v)", errHTTP, req.Header.Get("Authorization")) } return u, &bearerTokenCredentials{authToken: parts[1]}, nil } // closeUploadBlobSession closes a blob upload session using relative URL u, including digest d. func (r *ociRegistry) closeUploadBlobSession(ctx context.Context, creds credentials, u *url.URL, d digest.Digest) error { q := u.Query() q.Set("digest", d.String()) u.RawQuery = q.Encode() req, err := r.newRequest(ctx, http.MethodPut, u, nil) if err != nil { return err } res, err := r.doRequestWithCredentials(req, creds) if err != nil { return err } defer res.Body.Close() return nil } // uploadBlobPart uploads a chunk of a blob read from rd using relative URL u. The chunk is located // at offset and is of size chunkSize. func (r *ociRegistry) uploadBlobPart(ctx context.Context, creds credentials, u *url.URL, rd io.Reader, chunkSize, offset int64) (*url.URL, error) { req, err := r.newRequest(ctx, http.MethodPatch, u, io.LimitReader(rd, chunkSize)) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/octet-stream") req.Header.Set("Content-Range", fmt.Sprintf("%v-%v", offset, offset+chunkSize-1)) req.Header.Set("Content-Length", strconv.FormatInt(chunkSize, 10)) res, err := r.doRequestWithCredentials(req, creds) if err != nil { return nil, err } defer res.Body.Close() return getRelativeLocation(res) } // getRelativeLocation returns the relative URL contained in the `Location` header of res. func getRelativeLocation(res *http.Response) (*url.URL, error) { u, err := url.Parse(res.Header.Get("Location")) if err != nil { return nil, err } u.Path = strings.TrimPrefix(u.Path, "/") return u, nil } func getSigned(f *sif.FileImage) (bool, error) { sigs, err := f.GetDescriptors(sif.WithDataType(sif.DataSignature)) if err != nil { return false, err } return len(sigs) > 0, nil } // getEncrypted returns a boolean indicating whether the primary system partition in f is // encrypted. func getEncrypted(f *sif.FileImage) (bool, error) { od, err := f.GetDescriptor(sif.WithPartitionType(sif.PartPrimSys)) if err != nil { return false, err } t, _, _, err := od.PartitionMetadata() if err != nil { return false, err } return (t == sif.FsEncryptedSquashfs), nil } // processImageHeader creates an imageConfig using the supplied hash, description, and SIF header // contained in b. func (r *ociRegistry) processImageHeader(rootFS digest.Digest, description string, b []byte) (imageConfig, error) { f, err := sif.LoadContainer(sif.NewBuffer(b)) if err != nil { return imageConfig{}, err } defer func() { if err := f.UnloadContainer(); err != nil { r.logger.Logf("Failed to unload container: %v", err) } }() signed, err := getSigned(f) if err != nil { return imageConfig{}, err } encrypted, err := getEncrypted(f) if err != nil { return imageConfig{}, err } ic := imageConfig{ Architecture: f.PrimaryArch(), OS: "linux", RootFS: rootFS, Description: description, Signed: signed, Encrypted: encrypted, } return ic, nil } // manifestURL returns the relative URL associated with name/ref. func manifestURL(name, ref string) *url.URL { return &url.URL{ Path: fmt.Sprintf("v2/%v/manifests/%v", name, ref), } } // uploadManifest uploads manifest v of type contentType to the registry, and associates it with // name/ref. If ref is empty, the manifest digest is used. func (r *ociRegistry) uploadManifest(ctx context.Context, creds credentials, name, ref string, v interface{}, contentType string) (digest.Digest, error) { b, err := json.Marshal(v) if err != nil { return "", err } d := digest.FromBytes(b) if ref == "" { ref = d.String() } req, err := r.newRequest(ctx, http.MethodPut, manifestURL(name, ref), bytes.NewReader(b)) if err != nil { return "", err } req.Header.Set("Content-Type", contentType) res, err := r.doRequest(req, creds, withNamespaceAccess(name, accessTypePush)) if err != nil { return "", err } defer res.Body.Close() return d, nil } // UploadV1Index uploads image index idx to the registry, and associates it with name/ref. If ref // is empty, the image index digest is used. func (r *ociRegistry) UploadV1Index(ctx context.Context, creds credentials, name, ref string, idx v1.Index) (digest.Digest, error) { return r.uploadManifest(ctx, creds, name, ref, idx, v1.MediaTypeImageIndex) } // uploadV1Manifest uploads manifest m to the registry, and associates it with name/ref. If ref is // empty, the manifest digest is used. func (r *ociRegistry) uploadV1Manifest(ctx context.Context, creds credentials, name, ref string, m v1.Manifest) (digest.Digest, error) { return r.uploadManifest(ctx, creds, name, ref, m, v1.MediaTypeImageManifest) } container-library-client-1.4.10/client/oci_test.go000066400000000000000000000053211471454260400221230ustar00rootroot00000000000000// Copyright (c) 2022-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( "context" "encoding/json" "net/http" "net/http/httptest" "testing" ) func TestOciRegistryAuth(t *testing.T) { const ociRegistryURI = "https://registry" tests := []struct { name string directOciDownloadSupported bool ref string mappedRef string }{ {"Basic", true, "entity/collection/container", "entity/collection/container"}, {"TwoElements", true, "entity/container", "entity/container"}, {"ShortName", true, "alpine", "library/default/alpine"}, {"NotSupported", false, "", ""}, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() testShimSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if !tt.directOciDownloadSupported { w.WriteHeader(http.StatusNotFound) return } response := struct { Token string `json:"token"` RegistryURI string `json:"url"` Name string `json:"name"` }{ Token: "xxx", RegistryURI: ociRegistryURI, Name: tt.mappedRef, } if v := r.URL.Query().Get("namespace"); v == "" { t.Fatal("Query string \"namespace\" not set") } if v := r.URL.Query().Get("accessTypes"); v == "" { t.Fatalf("Query string \"accessTypes\" not set") } if err := json.NewEncoder(w).Encode(&response); err != nil { t.Fatalf("error JSON encoding: %v", err) } })) defer testShimSrv.Close() clientCfg := &Config{ BaseURL: testShimSrv.URL, Logger: &stdLogger{}, UserAgent: "scs-library-client-unit-tests/1.0", } c, err := NewClient(clientCfg) if err != nil { t.Fatalf("error initializing client: %v", err) } u, creds, name, err := c.ociRegistryAuth(context.Background(), tt.ref, []accessType{accessTypePull}) if tt.directOciDownloadSupported && err != nil { t.Fatalf("error getting OCI registry credentials: %v", err) } else if !tt.directOciDownloadSupported && err == nil { t.Fatal("unexpected success") } if !tt.directOciDownloadSupported { return } if got, want := name, tt.mappedRef; got != want { t.Fatalf("unexpected OCI artifact name: got %v, want %v", got, want) } if got, want := u.String(), ociRegistryURI; got != want { t.Fatalf("unexpected OCI registry URI: got %v, want %v", got, want) } if creds == nil { t.Fatal("expecting bearer token credential") } }) } } container-library-client-1.4.10/client/pull.go000066400000000000000000000211411471454260400212640ustar00rootroot00000000000000// Copyright (c) 2018-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( "context" "errors" "fmt" "io" "net/http" "net/url" "os" "strconv" "strings" jsonresp "github.com/sylabs/json-resp" ) var errRequestedImageNotFound = fmt.Errorf("requested image was not found in the library") // DownloadImage will retrieve an image from the Container Library, saving it // into the specified io.Writer. The timeout value for this operation is set // within the context. It is recommended to use a large value (ie. 1800 seconds) // to prevent timeout when downloading large images. func (c *Client) DownloadImage(ctx context.Context, w io.Writer, arch, path, tag string, callback func(int64, io.Reader, io.Writer) error) error { if arch != "" && !c.apiAtLeast(ctx, APIVersionV2ArchTags) { c.Logger.Log("This library does not support architecture specific tags") c.Logger.Log("The image returned may not be the requested architecture") } if strings.Contains(path, ":") { return fmt.Errorf("%w: malformed image path: %s", errBadRequest, path) } if tag == "" { tag = "latest" } apiPath := fmt.Sprintf("v1/imagefile/%s:%s", strings.TrimPrefix(path, "/"), tag) q := url.Values{} q.Add("arch", arch) c.Logger.Logf("Pulling from URL: %s", apiPath) req, err := c.newRequest(ctx, http.MethodGet, apiPath, q.Encode(), nil) if err != nil { return err } res, err := c.HTTPClient.Do(req) if err != nil { return err } defer res.Body.Close() if res.StatusCode == http.StatusNotFound { return errRequestedImageNotFound } if res.StatusCode != http.StatusOK { err := jsonresp.ReadError(res.Body) if err != nil { return fmt.Errorf("download did not succeed: %w", err) } if res.StatusCode == http.StatusUnauthorized { return ErrUnauthorized } return fmt.Errorf("%w: unexpected http status code: %d", errHTTP, res.StatusCode) } c.Logger.Logf("OK response received, beginning body download") if callback != nil { err = callback(res.ContentLength, res.Body, w) } else { _, err = io.Copy(w, res.Body) } if err != nil { return err } c.Logger.Logf("Download complete") return nil } // Downloader defines concurrency (# of requests) and part size for download operation. type Downloader struct { // Concurrency defines concurrency for multi-part downloads. Concurrency uint // PartSize specifies size of part for multi-part downloads. Default is 5 MiB. PartSize int64 // BufferSize specifies buffer size used for multi-part downloader routine. // Default is 32 KiB. // Deprecated: this value will be ignored. It is retained for backwards compatibility. BufferSize int64 } // NoopProgressBar implements ProgressBarInterface to allow disabling the progress bar type NoopProgressBar struct{} // Init is a no-op func (*NoopProgressBar) Init(int64) {} // ProxyReader is a no-op func (*NoopProgressBar) ProxyReader(r io.Reader) io.ReadCloser { return io.NopCloser(r) } // IncrBy is a no-op func (*NoopProgressBar) IncrBy(int) {} // Abort is a no-op func (*NoopProgressBar) Abort(bool) {} // Wait is a no-op func (*NoopProgressBar) Wait() {} // ProgressBar provides a minimal interface for interacting with a progress bar. // Init is called prior to concurrent download operation. type ProgressBar interface { // Initialize progress bar. Argument is size of file to set progress bar limit. Init(int64) // ProxyReader wraps r with metrics required for progress tracking. Only useful for // single stream downloads. ProxyReader(io.Reader) io.ReadCloser // IncrBy increments the progress bar. It is called after each concurrent // buffer transfer. IncrBy(int) // Abort terminates the progress bar. Abort(bool) // Wait waits for the progress bar to complete. Wait() } // ConcurrentDownloadImage implements a multi-part (concurrent) downloader for // Cloud Library images. spec is used to define transfer parameters. pb is an // optional progress bar interface. If pb is nil, NoopProgressBar is used. // // The downloader will handle source files of all sizes and is not limited to // only files larger than Downloader.PartSize. It will automatically adjust the // concurrency for source files that do not meet minimum size for multi-part // downloads. func (c *Client) ConcurrentDownloadImage(ctx context.Context, dst *os.File, arch, path, tag string, spec *Downloader, pb ProgressBar) error { if pb == nil { pb = &NoopProgressBar{} } if strings.Contains(path, ":") { return fmt.Errorf("%w: malformed image path: %s", errBadRequest, path) } name := strings.TrimPrefix(path, "/") if tag == "" { tag = "latest" } // Check for direct OCI registry access if err := c.ociDownloadImage(ctx, arch, name, tag, dst, spec, pb); err != nil { if !errors.Is(err, errOCIDownloadNotSupported) { // Return OCI download error or fallback to legacy download return err } c.Logger.Log("Fallback to (legacy) library download") return c.legacyDownloadImage(ctx, arch, name, tag, dst, spec, pb) } return nil } func (c *Client) legacyDownloadImage(ctx context.Context, arch, name, tag string, dst io.WriterAt, spec *Downloader, pb ProgressBar) error { if arch != "" && !c.apiAtLeast(ctx, APIVersionV2ArchTags) { c.Logger.Log("This library does not support architecture specific tags") c.Logger.Log("The image returned may not be the requested architecture") } apiPath := fmt.Sprintf("v1/imagefile/%v:%v", name, tag) q := url.Values{} q.Add("arch", arch) c.Logger.Logf("Pulling from URL: %s", apiPath) customHTTPClient := &http.Client{ Transport: c.HTTPClient.Transport, CheckRedirect: func(req *http.Request, via []*http.Request) error { if req.Response.StatusCode == http.StatusSeeOther { return http.ErrUseLastResponse } maxRedir := 10 if len(via) >= maxRedir { return fmt.Errorf("%w: stopped after %d redirects", errHTTP, maxRedir) } return nil }, Jar: c.HTTPClient.Jar, Timeout: c.HTTPClient.Timeout, } req, err := c.newRequest(ctx, http.MethodGet, apiPath, q.Encode(), nil) if err != nil { return err } res, err := customHTTPClient.Do(req) if err != nil { return err } defer res.Body.Close() if res.StatusCode == http.StatusNotFound { return errRequestedImageNotFound } if res.StatusCode == http.StatusOK { // Library endpoint does not provide HTTP redirection response, treat as single stream download c.Logger.Log("Library endpoint does not support concurrent downloads; reverting to single stream") size, err := parseContentLengthHeader(res.Header.Get("Content-Length")) if err != nil { return err } return c.download(ctx, dst, res.Body, size, pb) } if res.StatusCode != http.StatusSeeOther { if res.StatusCode == http.StatusUnauthorized { return ErrUnauthorized } return fmt.Errorf("%w: unexpected http status %d", errHTTP, res.StatusCode) } // Get image metadata to determine image size img, err := c.GetImage(ctx, arch, fmt.Sprintf("%v:%v", name, tag)) if err != nil { return err } redirectURL, err := url.Parse(res.Header.Get("Location")) if err != nil { return err } var creds credentials if c.AuthToken != "" && samehost(c.BaseURL, redirectURL) { // Only include credentials if redirected to same host as base URL creds = bearerTokenCredentials{authToken: c.AuthToken} } // Use redirect URL to download artifact return c.multipartDownload(ctx, redirectURL.String(), creds, dst, img.Size, spec, pb) } // samehost returns true if host1 and host2 are, in fact, the same host by // comparing scheme (https == https) and host, including port. // // Hosts will be treated as dissimilar if one host includes domain suffix // and the other does not, even if the host names match. func samehost(host1, host2 *url.URL) bool { return strings.EqualFold(host1.Scheme, host2.Scheme) && strings.EqualFold(host1.Host, host2.Host) } func parseContentLengthHeader(val string) (int64, error) { if val == "" { return int64(-1), nil } size, err := strconv.ParseInt(val, 10, 64) if err != nil { return -1, fmt.Errorf("parsing Content-Length header %v: %w", val, err) } return size, nil } // download implements a simple, single stream downloader func (c *Client) download(_ context.Context, w io.WriterAt, r io.Reader, size int64, pb ProgressBar) error { pb.Init(size) defer pb.Wait() proxyReader := pb.ProxyReader(r) defer proxyReader.Close() written, err := io.Copy(&filePartDescriptor{start: 0, end: size - 1, w: w}, proxyReader) if err != nil { pb.Abort(true) return err } c.Logger.Logf("Downloaded %v byte(s)", written) return nil } container-library-client-1.4.10/client/pull_test.go000066400000000000000000000216701471454260400223320ustar00rootroot00000000000000// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( "bufio" "bytes" "context" "crypto/sha256" "encoding/binary" "fmt" "io" "net/http" "net/http/httptest" "net/url" "os" "reflect" "strings" "testing" crypto_rand "crypto/rand" math_rand "math/rand" ) type mockRawService struct { t *testing.T code int testFile string reqCallback func(*http.Request, *testing.T) httpAddr string httpPath string httpServer *httptest.Server baseURI string } func (m *mockRawService) Run() { mux := http.NewServeMux() mux.HandleFunc(m.httpPath, m.ServeHTTP) m.httpServer = httptest.NewServer(mux) m.httpAddr = m.httpServer.Listener.Addr().String() m.baseURI = "http://" + m.httpAddr } func (m *mockRawService) Stop() { m.httpServer.Close() } func (m *mockRawService) ServeHTTP(w http.ResponseWriter, r *http.Request) { if m.reqCallback != nil { m.reqCallback(r, m.t) } w.WriteHeader(m.code) inFile, err := os.Open(m.testFile) if err != nil { m.t.Errorf("error opening file %v:", err) } defer inFile.Close() _, err = io.Copy(w, bufio.NewReader(inFile)) if err != nil { m.t.Errorf("Test HTTP server unable to output file: %v", err) } } func Test_DownloadImage(t *testing.T) { f, err := os.CreateTemp("", "test") if err != nil { t.Fatalf("Error creating a temporary file for testing") } tempFile := f.Name() f.Close() os.Remove(tempFile) tests := []struct { name string arch string path string tag string outFile string code int testFile string tokenFile string checkContent bool expectError bool }{ {"Bad library ref", "amd64", "entity/collection/im,age", "tag", tempFile, http.StatusBadRequest, "test_data/test_sha256", "test_data/test_token", false, true}, {"Server error", "amd64", "entity/collection/image", "tag", tempFile, http.StatusInternalServerError, "test_data/test_sha256", "test_data/test_token", false, true}, {"Tags in path", "amd64", "entity/collection/image:tag", "anothertag", tempFile, http.StatusOK, "test_data/test_sha256", "test_data/test_token", false, true}, {"Good Download", "amd64", "entity/collection/image", "tag", tempFile, http.StatusOK, "test_data/test_sha256", "test_data/test_token", true, false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { m := mockRawService{ t: t, code: tt.code, testFile: tt.testFile, httpPath: fmt.Sprintf("/v1/imagefile/%s:%s", tt.path, tt.tag), } m.Run() defer m.Stop() c, err := NewClient(&Config{AuthToken: tt.tokenFile, BaseURL: m.baseURI}) if err != nil { t.Errorf("Error initializing client: %v", err) } out, err := os.OpenFile(tt.outFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o777) if err != nil { t.Errorf("Error opening file %s for writing: %v", tt.outFile, err) } err = c.DownloadImage(context.Background(), out, tt.arch, tt.path, tt.tag, nil) out.Close() if err != nil && !tt.expectError { t.Errorf("Unexpected error: %v", err) } if err == nil && tt.expectError { t.Errorf("Unexpected success. Expected error.") } if tt.checkContent { fileContent, err := os.ReadFile(tt.outFile) if err != nil { t.Errorf("Error reading test output file: %v", err) } testContent, err := os.ReadFile(tt.testFile) if err != nil { t.Errorf("Error reading test file: %v", err) } if !bytes.Equal(fileContent, testContent) { t.Errorf("File contains '%v' - expected '%v'", fileContent, testContent) } } }) } } func TestParseContentRange(t *testing.T) { const hdr = "bytes 0-1000/1000" size, err := parseContentRange(hdr) if err != nil { t.Fatalf("unexpected error: %v", err) } if got, want := size, int64(1000); got != want { t.Fatalf("unexpected content length: got %v, want %v", got, want) } } func TestParseContentLengthHeader(t *testing.T) { t.Parallel() tests := []struct { name string headerValue string expectedResult int64 expectError bool }{ {"ValidValue", "1234", 1234, false}, {"InvalidValue", "xxxx", 0, true}, {"EmptyValue", "", -1, false}, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { result, err := parseContentLengthHeader(tt.headerValue) if !tt.expectError && err != nil { t.Fatalf("unexpected error: %v", err) } if tt.expectError && err == nil { t.Fatal("unexpected success") } if got, want := result, tt.expectedResult; err == nil && got != want { t.Fatalf("unexpected result: got %v, want %v", got, want) } }) } } func generateSampleData(t *testing.T) []byte { t.Helper() const maxSampleDataSize = 1 * 1024 * 1024 // 1 MiB size := math_rand.Int63() % maxSampleDataSize sampleBytes := make([]byte, size) if _, err := crypto_rand.Read(sampleBytes); err != nil { t.Fatalf("error generating random bytes: %v", err) } return sampleBytes } func seedRandomNumberGenerator(t *testing.T) { t.Helper() var b [8]byte if _, err := crypto_rand.Read(b[:]); err != nil { t.Fatalf("error seeding random number generator: %v", err) } math_rand.New(math_rand.NewSource(int64(binary.LittleEndian.Uint64(b[:])))) } // mockLibraryServer returns *httptest.Server that mocks Cloud Library server; in particular, // it has handlers for /version, /v1/images, /v1/imagefile, and /v1/imagepart func mockLibraryServer(t *testing.T, sampleBytes []byte, size int64, multistream bool) *httptest.Server { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if strings.HasPrefix(r.URL.Path, "/version") { w.Header().Set("Content-Type", "application/json; charset=utf-8") w.WriteHeader(http.StatusOK) if _, err := w.Write([]byte("{\"data\": {\"apiVersion\": \"1.0.0\"}}")); err != nil { t.Fatalf("error writing /version response: %v", err) } return } if multistream && strings.HasPrefix(r.URL.Path, "/v1/images/") { w.Header().Set("Content-Type", "application/json; charset=utf-8") w.WriteHeader(http.StatusOK) if _, err := w.Write([]byte(fmt.Sprintf("{\"data\": {\"size\": %v}}", size))); err != nil { t.Fatalf("error writing /v1/images response: %v", err) } return } if multistream && strings.HasPrefix(r.URL.Path, "/v1/imagefile/") { redirectURL := &url.URL{ Scheme: "http", Host: r.Host, Path: "/v1/imagepart/" + strings.TrimPrefix(r.URL.Path, "/v1/imagefile/"), } w.Header().Set("Location", redirectURL.String()) w.WriteHeader(http.StatusSeeOther) return } // Handle /v1/imagefile (single stream) or /v1/imagepart (multistream) // Handle Range request for multipart downloads var start, end int64 if val := r.Header.Get("Range"); val != "" { start, end = parseRangeHeader(t, val) } else { start, end = 0, int64(size)-1 } // Set up response headers w.Header().Set("Content-Length", fmt.Sprintf("%v", end-start+1)) w.Header().Set("Content-Type", "application/octet-stream") w.WriteHeader(http.StatusOK) // Write image data if _, err := w.Write(sampleBytes[start : end+1]); err != nil { t.Fatalf("error writing response: %v", err) } })) return srv } // TestLegacyDownloadImage downloads random image data from mock library and compares hash to // ensure download integrity. func TestLegacyDownloadImage(t *testing.T) { tests := []struct { name string multistreamDownload bool }{ {"SingleStream", false}, {"MultiStream", true}, } // Total overkill seeding the random number generator seedRandomNumberGenerator(t) for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { // Generate random bytes to simulate file; upto 256k sampleBytes := generateSampleData(t) size := int64(len(sampleBytes)) testLogger.Logf("Generated %v bytes of mock image data", size) hash := sha256.Sum256(sampleBytes) // Create mock library server that responds to '/version' and '/v1/imagefile' only srv := mockLibraryServer(t, sampleBytes, size, tt.multistreamDownload) defer srv.Close() // Initialize scs-library-client c, err := NewClient(&Config{BaseURL: srv.URL, Logger: testLogger}) if err != nil { t.Fatalf("error initializing client: %v", err) } // Initialize sink for downloaded sample image dst := &inMemoryBuffer{buf: make([]byte, size)} err = c.legacyDownloadImage( context.Background(), "amd64", "entity/collection/container", "tag", dst, &Downloader{Concurrency: 4, PartSize: 64 * 1024}, &NoopProgressBar{}, ) if err != nil { t.Fatalf("unexpected error: %v", err) } // Compare sha256 hash of data sent with hash of data received if got, want := sha256.Sum256(dst.Bytes()), hash; !reflect.DeepEqual(got, want) { t.Fatalf("unexpected hash: got %x, want %v", got, want) } }) } } container-library-client-1.4.10/client/push.go000066400000000000000000000460171471454260400213000ustar00rootroot00000000000000// Copyright (c) 2018-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( "context" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "strings" jsonresp "github.com/sylabs/json-resp" "golang.org/x/sync/errgroup" ) const ( // minimumPartSize is the minimum size of a part in a multipart upload; // this liberty is taken by defining this value on the client-side to // prevent a round-trip to the server. The server will return HTTP status // 400 if the requested multipart upload size is less than 5MiB. minimumPartSize = 64 * 1024 * 1024 // OptionS3Compliant indicates a 100% S3 compatible object store is being used by backend library server OptionS3Compliant = "s3compliant" ) // UploadCallback defines an interface used to perform a call-out to // set up the source file Reader. type UploadCallback interface { // Initializes the callback given a file size and source file Reader InitUpload(int64, io.Reader) // (optionally) can return a proxied Reader GetReader() io.Reader // TerminateUpload is called if the upload operation is interrupted before completion Terminate() // called when the upload operation is complete Finish() } // Default upload callback type defaultUploadCallback struct { r io.Reader } func (c *defaultUploadCallback) InitUpload(_ int64, r io.Reader) { c.r = r } func (c *defaultUploadCallback) GetReader() io.Reader { return c.r } func (c *defaultUploadCallback) Terminate() { } func (c *defaultUploadCallback) Finish() { } // calculateChecksums uses a TeeReader to calculate MD5 and SHA256 // checksums concurrently func calculateChecksums(r io.Reader) (string, string, int64, error) { pr, pw := io.Pipe() tr := io.TeeReader(r, pw) var g errgroup.Group var md5checksum string var sha256checksum string var fileSize int64 // compute MD5 checksum for comparison with S3 checksum g.Go(func() error { // The pipe writer must be closed so sha256 computation gets EOF and will // complete. defer pw.Close() var err error md5checksum, fileSize, err = md5sum(tr) if err != nil { return fmt.Errorf("error calculating MD5 checksum: %w", err) } return nil }) // Compute sha256 g.Go(func() error { var err error sha256checksum, _, err = sha256sum(pr) if err != nil { return fmt.Errorf("error calculating SHA checksum: %w", err) } return nil }) err := g.Wait() return md5checksum, sha256checksum, fileSize, err } // UploadImage will push a specified image from an io.ReadSeeker up to the // Container Library, The timeout value for this operation is set within // the context. It is recommended to use a large value (ie. 1800 seconds) to // prevent timeout when uploading large images. func (c *Client) UploadImage(ctx context.Context, r io.ReadSeeker, path, arch string, tags []string, description string, callback UploadCallback) (*UploadImageComplete, error) { if !IsLibraryPushRef(path) { return nil, fmt.Errorf("%w: malformed image path: %s", errBadRequest, path) } entityName, collectionName, containerName, parsedTags := ParseLibraryPath(path) if len(parsedTags) != 0 { return nil, fmt.Errorf("%w: malformed image path: %s", errBadRequest, path) } // calculate sha256 and md5 checksums md5Checksum, imageHash, fileSize, err := calculateChecksums(r) if err != nil { return nil, fmt.Errorf("error calculating checksums: %w", err) } // rollback to top of file if _, err = r.Seek(0, io.SeekStart); err != nil { return nil, fmt.Errorf("error seeking to start stream: %w", err) } c.Logger.Logf("Image hash computed as %s", imageHash) if err := c.ociUploadImage(ctx, r, fileSize, strings.TrimPrefix(path, "library://"), arch, tags, description, "sha256."+imageHash, callback); err == nil { return nil, nil } else if !errors.Is(err, errOCIDownloadNotSupported) { // Return OCI upload error or fallback to legacy download return nil, err } c.Logger.Log("Fallback to (legacy) library upload") // Find or create entity entity, err := c.getEntity(ctx, entityName) if err != nil { if !errors.Is(err, ErrNotFound) { return nil, err } c.Logger.Logf("Entity %s does not exist in library - creating it.", entityName) entity, err = c.createEntity(ctx, entityName) if err != nil { return nil, err } } // Find or create collection qualifiedCollectionName := fmt.Sprintf("%s/%s", entityName, collectionName) collection, err := c.getCollection(ctx, qualifiedCollectionName) if err != nil { if !errors.Is(err, ErrNotFound) { return nil, err } // create collection c.Logger.Logf("Collection %s does not exist in library - creating it.", collectionName) collection, err = c.createCollection(ctx, collectionName, entity.ID) if err != nil { return nil, err } } // Find or create container computedName := fmt.Sprintf("%s/%s", qualifiedCollectionName, containerName) container, err := c.getContainer(ctx, computedName) if err != nil { if !errors.Is(err, ErrNotFound) { return nil, err } // Create container c.Logger.Logf("Container %s does not exist in library - creating it.", containerName) container, err = c.createContainer(ctx, containerName, collection.ID) if err != nil { return nil, err } } // Find or create image image, err := c.GetImage(ctx, arch, computedName+":"+"sha256."+imageHash) if err != nil { if !errors.Is(err, ErrNotFound) { return nil, err } // Create image c.Logger.Logf("Image %s does not exist in library - creating it.", imageHash) image, err = c.createImage(ctx, "sha256."+imageHash, container.ID, description) if err != nil { return nil, err } } var res *UploadImageComplete if !image.Uploaded { // upload image if callback == nil { // use default (noop) upload callback callback = &defaultUploadCallback{r: r} } metadata := map[string]string{ "sha256sum": imageHash, "md5sum": md5Checksum, } res, err = c.postFileWrapper(ctx, r, fileSize, image.ID, callback, metadata) if err != nil { return nil, err } } else { c.Logger.Logf("Image is already present in the library - not uploading.") } // set tags on image c.Logger.Logf("Setting tags against uploaded image") if c.apiAtLeast(ctx, APIVersionV2ArchTags) { if err := c.setTagsV2(ctx, container.ID, arch, image.ID, append(tags, parsedTags...)); err != nil { return nil, err } return res, nil } c.Logger.Logf("This library does not support multiple architectures per tag.") c.Logger.Logf("This tag will replace any already uploaded with the same name.") if err := c.setTags(ctx, container.ID, image.ID, append(tags, parsedTags...)); err != nil { return nil, err } return res, nil } func (c *Client) postFileWrapper(ctx context.Context, r io.ReadSeeker, fileSize int64, imageID string, callback UploadCallback, metadata map[string]string) (*UploadImageComplete, error) { var err error // use callback to set up source file reader callback.InitUpload(fileSize, r) var res *UploadImageComplete c.Logger.Log("Now uploading to the library") if c.apiAtLeast(ctx, APIVersionV2Upload) { // use v2 post file api. Send both md5 and sha256 checksums. If the // remote does not support sha256, it will be ignored and fallback // to md5. If the remote is aware of sha256, will be used and md5 // will be ignored. res, err = c.postFileV2(ctx, r, fileSize, imageID, callback, metadata) } else { // fallback to legacy upload res, err = c.postFile(ctx, fileSize, imageID, callback) } if err != nil { callback.Terminate() c.Logger.Log("Upload terminated due to error") } else { callback.Finish() c.Logger.Log("Upload completed OK") } return res, err } func (c *Client) postFile(ctx context.Context, fileSize int64, imageID string, callback UploadCallback) (*UploadImageComplete, error) { postURL := "v1/imagefile/" + imageID c.Logger.Logf("postFile calling %s", postURL) // Make an upload request req, err := c.newRequest(ctx, http.MethodPost, postURL, "", callback.GetReader()) if err != nil { return nil, err } // Content length is required by the API req.ContentLength = fileSize res, err := c.HTTPClient.Do(req) if err != nil { return nil, fmt.Errorf("error uploading file to server: %w", err) } defer res.Body.Close() if res.StatusCode != http.StatusOK { if err := jsonresp.ReadError(res.Body); err != nil { return nil, fmt.Errorf("sending file did not succeed: %w", err) } return nil, fmt.Errorf("%w: sending file did not succeed: http status code %d", errHTTP, res.StatusCode) } return nil, nil } // postFileV2 uses V2 API to upload images to SCS library server. This is // a three step operation: "create" upload image request, which returns a // URL to issue an http PUT operation against, and then finally calls the // completion endpoint once upload is complete. func (c *Client) postFileV2(ctx context.Context, r io.ReadSeeker, fileSize int64, imageID string, callback UploadCallback, metadata map[string]string) (*UploadImageComplete, error) { if fileSize > minimumPartSize { // only attempt multipart upload if size greater than S3 minimum c.Logger.Log("Attempting to use multipart uploader") var err error var res *UploadImageComplete res, err = c.postFileV2Multipart(ctx, r, fileSize, imageID, callback) if err != nil { // if the error is anything other than ErrNotFound, fallback to legacy (single part) // uploader. if !errors.Is(err, ErrNotFound) { return nil, err } // fallthrough to legacy (single part) uploader } else { // multipart upload successful return res, nil } } // fallback to legacy uploader c.Logger.Log("Using legacy (single part) uploader") return c.legacyPostFileV2(ctx, fileSize, imageID, callback, metadata) } // uploadManager contains common params for multipart part function type uploadManager struct { Source io.ReadSeeker Size int64 ImageID string UploadID string } func (c *Client) postFileV2Multipart(ctx context.Context, r io.ReadSeeker, fileSize int64, imageID string, callback UploadCallback) (*UploadImageComplete, error) { // initiate multipart upload with backend to determine number of expected // parts and part size response, err := c.startMultipartUpload(ctx, fileSize, imageID) if err != nil { c.Logger.Logf("Error starting multipart upload: %v", err) return nil, err } c.Logger.Logf("Multi-part upload: ID=[%s] totalParts=[%d] partSize=[%d]", response.UploadID, response.TotalParts, fileSize) // Enable S3 compliance mode by default val := response.Options[OptionS3Compliant] s3Compliant := val == "" || val == "true" c.Logger.Logf("S3 compliant option: %v", s3Compliant) // maintain list of completed parts which will be passed to the completion function completedParts := []CompletedPart{} bytesRemaining := fileSize for nPart := 1; nPart <= response.TotalParts; nPart++ { partSize := getPartSize(bytesRemaining, response.PartSize) c.Logger.Logf("Uploading part %d (%d bytes)", nPart, partSize) mgr := &uploadManager{ Source: r, Size: partSize, ImageID: imageID, UploadID: response.UploadID, } // include "X-Amz-Content-Sha256" header only if object store is 100% S3 compatible etag, err := c.multipartUploadPart(ctx, nPart, mgr, callback, s3Compliant) if err != nil { // error uploading part c.Logger.Logf("Error uploading part %d: %v", nPart, err) if err := c.abortMultipartUpload(ctx, mgr); err != nil { c.Logger.Logf("Error aborting multipart upload: %v", err) } return nil, err } // append completed part info to list completedParts = append(completedParts, CompletedPart{PartNumber: nPart, Token: etag}) // decrement upload bytes remaining bytesRemaining -= partSize } c.Logger.Logf("Uploaded %d parts", response.TotalParts) return c.completeMultipartUpload(ctx, &completedParts, &uploadManager{ ImageID: imageID, UploadID: response.UploadID, }) } // getPartSize returns number of bytes to read for "next" part. This value will // never exceed the S3 maximum func getPartSize(bytesRemaining int64, partSize int64) int64 { if bytesRemaining > int64(partSize) { return partSize } return bytesRemaining } func (c *Client) startMultipartUpload(ctx context.Context, fileSize int64, imageID string) (MultipartUpload, error) { // attempt to initiate multipart upload postURL := fmt.Sprintf("v2/imagefile/%s/_multipart", imageID) c.Logger.Logf("startMultipartUpload calling %s", postURL) body := MultipartUploadStartRequest{ Size: fileSize, } objJSON, err := c.apiCreate(ctx, postURL, body) if err != nil { return MultipartUpload{}, err } var res MultipartUploadStartResponse if err := json.Unmarshal(objJSON, &res); err != nil { return MultipartUpload{}, err } return res.Data, nil } // remoteSHA256ChecksumSupport parses the 'X-Amz-SignedHeaders' from the // presigned PUT URL query string to determine if 'x-amz-content-sha256' is // present. If 'x-amz-content-sha256' is present, the remote is expecting the // SHA256 checksum in the headers of the presigned PUT URL request. func remoteSHA256ChecksumSupport(u *url.URL) bool { hdr := u.Query()["X-Amz-SignedHeaders"] if len(hdr) < 1 { return false } for _, h := range strings.Split(hdr[0], ";") { if h == "x-amz-content-sha256" { return true } } return false } func (c *Client) legacyPostFileV2(ctx context.Context, fileSize int64, imageID string, callback UploadCallback, metadata map[string]string) (*UploadImageComplete, error) { postURL := fmt.Sprintf("v2/imagefile/%s", imageID) c.Logger.Logf("legacyPostFileV2 calling %s", postURL) // issue upload request (POST) to obtain presigned S3 URL body := UploadImageRequest{ Size: fileSize, SHA256Checksum: metadata["sha256sum"], MD5Checksum: metadata["md5sum"], } objJSON, err := c.apiCreate(ctx, postURL, body) if err != nil { return nil, err } var res UploadImageResponse if err := json.Unmarshal(objJSON, &res); err != nil { return nil, err } // upload (PUT) directly to S3 presigned URL provided above presignedURL := res.Data.UploadURL if presignedURL == "" { return nil, errGettingPresignedURL } parsedURL, err := url.Parse(presignedURL) if err != nil { return nil, errParsingPresignedURL } // parse presigned URL to determine if we need to send sha256 checksum useSHA256Checksum := remoteSHA256ChecksumSupport(parsedURL) req, err := http.NewRequestWithContext(ctx, http.MethodPut, presignedURL, callback.GetReader()) if err != nil { return nil, fmt.Errorf("error creating request: %w", err) } req.ContentLength = fileSize req.Header.Set("Content-Type", "application/octet-stream") if useSHA256Checksum { req.Header.Set("x-amz-content-sha256", metadata["sha256sum"]) } resp, err := c.HTTPClient.Do(req) callback.Finish() if err != nil { return nil, fmt.Errorf("error uploading image: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("%w: error uploading image: HTTP status %d", errHTTP, resp.StatusCode) } // send (PUT) image upload completion objJSON, err = c.apiUpdate(ctx, postURL+"/_complete", UploadImageCompleteRequest{}) if err != nil { return nil, fmt.Errorf("error sending upload complete request: %w", err) } if len(objJSON) == 0 { // success w/o detailed upload complete response return nil, nil } var uploadResp UploadImageCompleteResponse if err := json.Unmarshal(objJSON, &uploadResp); err != nil { return nil, fmt.Errorf("error decoding upload response: %w", err) } return &uploadResp.Data, nil } func getPartSHA256Sum(r io.Reader, size int64) (string, error) { // calculate sha256sum of part tmpChunk := io.LimitReader(r, size) chunkHash, _, err := sha256sum(tmpChunk) return chunkHash, err } func (c *Client) multipartUploadPart(ctx context.Context, partNumber int, m *uploadManager, callback UploadCallback, includeSHA256ChecksumHeader bool) (string, error) { var chunkHash string var err error if includeSHA256ChecksumHeader { // calculate sha256sum of part being uploaded chunkHash, err = getPartSHA256Sum(m.Source, int64(m.Size)) if err != nil { c.Logger.Logf("Error calculating SHA256 checksum: %v", err) return "", err } // rollback file pointer to beginning of part if _, err := m.Source.Seek(-(int64(m.Size)), io.SeekCurrent); err != nil { c.Logger.Logf("Error repositioning file pointer: %v", err) return "", err } } // send request to cloud-library for presigned PUT url uri := fmt.Sprintf("v2/imagefile/%s/_multipart", m.ImageID) c.Logger.Logf("multipartUploadPart calling %s", uri) objJSON, err := c.apiUpdate(ctx, uri, UploadImagePartRequest{ PartSize: m.Size, UploadID: m.UploadID, PartNumber: partNumber, SHA256Checksum: chunkHash, }) if err != nil { return "", err } var res UploadImagePartResponse if err := json.Unmarshal(objJSON, &res); err != nil { return "", err } // send request to S3 req, err := http.NewRequestWithContext(ctx, http.MethodPut, res.Data.PresignedURL, io.LimitReader(callback.GetReader(), m.Size)) if err != nil { return "", fmt.Errorf("error creating request: %w", err) } // add headers to be signed req.ContentLength = m.Size if includeSHA256ChecksumHeader { req.Header.Add("x-amz-content-sha256", chunkHash) } resp, err := c.HTTPClient.Do(req) if err != nil { c.Logger.Logf("Failure uploading to presigned URL: %w", err) return "", err } defer resp.Body.Close() // process response from S3 if resp.StatusCode != http.StatusOK { c.Logger.Logf("Object store returned an error: %d", resp.StatusCode) return "", fmt.Errorf("%w: object store returned an error: %d", errHTTP, resp.StatusCode) } etag := resp.Header.Get("ETag") c.Logger.Logf("Part %d accepted (ETag: %s)", partNumber, etag) return etag, nil } func (c *Client) completeMultipartUpload(ctx context.Context, completedParts *[]CompletedPart, m *uploadManager) (*UploadImageComplete, error) { c.Logger.Logf("Completing multipart upload: %s", m.UploadID) uri := fmt.Sprintf("v2/imagefile/%s/_multipart_complete", m.ImageID) c.Logger.Logf("completeMultipartUpload calling %s", uri) body := CompleteMultipartUploadRequest{ UploadID: m.UploadID, CompletedParts: *completedParts, } objJSON, err := c.apiUpdate(ctx, uri, body) if err != nil { c.Logger.Logf("Error completing multipart upload: %v", err) return nil, err } var res CompleteMultipartUploadResponse if err := json.Unmarshal(objJSON, &res); err != nil { c.Logger.Logf("Error decoding complete multipart upload request: %v", err) return nil, err } if res.Data.ContainerURL == "" { // success w/o detailed upload complete response return nil, nil } return &res.Data, nil } func (c *Client) abortMultipartUpload(ctx context.Context, m *uploadManager) error { c.Logger.Logf("Aborting multipart upload ID: %s", m.UploadID) uri := fmt.Sprintf("v2/imagefile/%s/_multipart_abort", m.ImageID) c.Logger.Logf("abortMultipartUpload calling %s", uri) body := AbortMultipartUploadRequest{ UploadID: m.UploadID, } if _, err := c.apiUpdate(ctx, uri, body); err != nil { c.Logger.Logf("error aborting multipart upload: %v", err) return err } return nil } container-library-client-1.4.10/client/push_test.go000066400000000000000000000704741471454260400223430ustar00rootroot00000000000000// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( "context" "encoding/json" "fmt" "net/http" "net/http/httptest" "net/url" "os" "runtime" "strings" "testing" jsonresp "github.com/sylabs/json-resp" ) const ( testQuotaUsageBytes int64 = 64 * 1024 * 1024 testQuotaTotalBytes int64 = 1024 * 1024 * 1024 testContainerURL = "/library/entity/collection/container" testPayload = "testtesttesttest" ) func Test_postFile(t *testing.T) { tests := []struct { description string imageRef string testFile string code int reqCallback func(*http.Request, *testing.T) expectError bool }{ { description: "Container not found response", code: 404, reqCallback: nil, imageRef: "5cb9c34d7d960d82f5f5bc55", testFile: "test_data/test_sha256", expectError: true, }, { description: "Unauthorized response", code: 401, reqCallback: nil, imageRef: "5cb9c34d7d960d82f5f5bc56", testFile: "test_data/test_sha256", expectError: true, }, { description: "Valid Response", code: 200, reqCallback: nil, imageRef: "5cb9c34d7d960d82f5f5bc57", testFile: "test_data/test_sha256", expectError: false, }, } // Loop over test cases for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { m := mockService{ t: t, code: tt.code, httpPath: "/v1/imagefile/" + tt.imageRef, } m.Run() defer m.Stop() c, err := NewClient(&Config{AuthToken: testToken, BaseURL: m.baseURI}) if err != nil { t.Errorf("Error initializing client: %v", err) } f, err := os.Open(tt.testFile) if err != nil { t.Errorf("Error opening file %s for reading: %v", tt.testFile, err) } defer f.Close() fi, err := f.Stat() if err != nil { t.Errorf("Error stats for file %s: %v", tt.testFile, err) } fileSize := fi.Size() callback := &defaultUploadCallback{r: f} _, err = c.postFile(context.Background(), fileSize, tt.imageRef, callback) if err != nil && !tt.expectError { t.Errorf("Unexpected error: %v", err) } if err == nil && tt.expectError { t.Errorf("Unexpected success. Expected error.") } }) } } type v2ImageUploadMockService struct { t *testing.T httpAddr string httpServer *httptest.Server baseURI string initCalled bool putCalled bool completeCalled bool } func (m *v2ImageUploadMockService) Run() { mux := http.NewServeMux() mux.HandleFunc("/v2/imagefile/5cb9c34d7d960d82f5f5bc55", m.MockImageFileEndpoint) mux.HandleFunc("/fake/s3/endpoint", m.MockS3PresignedURLPUTEndpoint) mux.HandleFunc("/v2/imagefile/5cb9c34d7d960d82f5f5bc55/_complete", m.MockImageFileCompleteEndpoint) m.httpServer = httptest.NewServer(mux) m.httpAddr = m.httpServer.Listener.Addr().String() m.baseURI = "http://" + m.httpAddr } func (m *v2ImageUploadMockService) Stop() { m.httpServer.Close() } func (m *v2ImageUploadMockService) MockImageFileEndpoint(w http.ResponseWriter, r *http.Request) { var uploadImageRequest UploadImageRequest if err := json.NewDecoder(r.Body).Decode(&uploadImageRequest); err != nil { if err := jsonresp.WriteError(w, "Provided image could not be decoded", http.StatusBadRequest); err != nil { m.t.Fatalf("error encoding error response: %v", err) } } // this is a bit of a nonsense assertion. All we're trying to do is confirm // that the sha256 checksum provided by the client is present in the // request. There is no actual validation of the sha256 checksum of the // payload in the PUT request. const expectedSha256 = "d7d356079af905c04e5ae10711ecf3f5b34385e9b143c5d9ddbf740665ce2fb7" if got, want := uploadImageRequest.SHA256Checksum, expectedSha256; got != want { m.t.Errorf("got checksum %v, want %v", got, want) } response := UploadImage{ UploadURL: m.baseURI + "/fake/s3/endpoint?key=value", } err := jsonresp.WriteResponse(w, &response, http.StatusOK) if err != nil { fmt.Printf("error: %v\n", err) } m.initCalled = true } func (m *v2ImageUploadMockService) MockS3PresignedURLPUTEndpoint(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) m.putCalled = true } func (m *v2ImageUploadMockService) MockImageFileCompleteEndpoint(w http.ResponseWriter, _ *http.Request) { response := UploadImageComplete{ Quota: QuotaResponse{ QuotaTotalBytes: testQuotaTotalBytes, QuotaUsageBytes: testQuotaUsageBytes, }, ContainerURL: testContainerURL, } if err := jsonresp.WriteResponse(w, &response, http.StatusOK); err != nil { fmt.Printf("error: %v\n", err) } m.completeCalled = true } func mockS3Server(t *testing.T, statusCode int) *httptest.Server { t.Helper() return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { if statusCode != http.StatusOK { w.WriteHeader(statusCode) return } w.Header().Set("ETag", "thisisasampleetag") })) } func initClient(t *testing.T, url string) *Client { t.Helper() c, err := NewClient(&Config{BaseURL: url, AuthToken: testToken, Logger: &stdLogger{}}) if err != nil { t.Fatalf("error initializing client: %v", err) } return c } func commonHandler(t *testing.T, code int, w http.ResponseWriter) { t.Helper() if code != http.StatusOK { if code != http.StatusInternalServerError { w.WriteHeader(code) return } _, err := w.Write([]byte("'?")) if err != nil { t.Fatalf("Unexpected error: %v", err) } return } } func uploadImageHelperHandler(t *testing.T, code int, w http.ResponseWriter) { t.Helper() if code != http.StatusOK { w.WriteHeader(code) return } } func Test_UploadImageBadPath(t *testing.T) { s3Server := mockS3Server(t, http.StatusOK) defer s3Server.Close() tests := []struct { name string path string expectError bool }{ {"badPath", "\n", true}, {"pathError", "library://entity/collection/container:te,st", true}, {"test", "entity/collection/container", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { h := http.NewServeMux() h.HandleFunc("/v1/imagefile/", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { commonHandler(t, http.StatusOK, w) })) h.HandleFunc("/v1/tags/", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { resp := TagMap{"test": "testValue"} if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } })) h.HandleFunc("/v1/entities/", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { uploadImageHelperHandler(t, http.StatusOK, w) resp := Entity{ID: "testID"} if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } })) h.HandleFunc("/v1/collections/", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { uploadImageHelperHandler(t, http.StatusOK, w) resp := Collection{ ID: "testID", } if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } })) h.HandleFunc("/v1/containers/", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { uploadImageHelperHandler(t, http.StatusOK, w) resp := Container{ID: "test"} if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } })) h.HandleFunc("/v1/images/", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { uploadImageHelperHandler(t, http.StatusOK, w) commonHandler(t, http.StatusOK, w) resp := Image{ID: "testID"} if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } })) libraryServer := httptest.NewServer(h) defer libraryServer.Close() c := initClient(t, libraryServer.URL) r := strings.NewReader(testPayload) etag, err := c.UploadImage( context.Background(), r, tt.path, runtime.GOARCH, []string{"tag"}, "test", &defaultUploadCallback{r: r}, ) if (err == nil) && tt.expectError { t.Fatal("unexpected success") } _ = etag }) } } type testCodesStruct struct { entity int collection int container int image int } func Test_UploadImage(t *testing.T) { tests := []struct { name string statusCode int s3StatusCode int expectError bool codes testCodesStruct }{ { "statusOK", http.StatusOK, http.StatusOK, false, testCodesStruct{http.StatusOK, http.StatusOK, http.StatusOK, http.StatusOK}, }, { "badRequest", http.StatusBadRequest, http.StatusBadRequest, true, testCodesStruct{http.StatusOK, http.StatusOK, http.StatusOK, http.StatusOK}, }, { "internalServerError", http.StatusInternalServerError, http.StatusInternalServerError, true, testCodesStruct{http.StatusOK, http.StatusOK, http.StatusOK, http.StatusOK}, }, { "entityNotFound", http.StatusOK, http.StatusOK, true, testCodesStruct{http.StatusNotFound, http.StatusOK, http.StatusOK, http.StatusOK}, }, { "entityBadRequest", http.StatusOK, http.StatusOK, true, testCodesStruct{http.StatusBadRequest, http.StatusOK, http.StatusOK, http.StatusOK}, }, { "collectionNotFound", http.StatusOK, http.StatusOK, true, testCodesStruct{http.StatusOK, http.StatusNotFound, http.StatusOK, http.StatusOK}, }, { "collectionBadRequest", http.StatusOK, http.StatusOK, true, testCodesStruct{http.StatusOK, http.StatusBadRequest, http.StatusOK, http.StatusOK}, }, { "containerNotFound", http.StatusOK, http.StatusOK, true, testCodesStruct{http.StatusOK, http.StatusOK, http.StatusNotFound, http.StatusOK}, }, { "containerBadRequest", http.StatusOK, http.StatusOK, true, testCodesStruct{http.StatusOK, http.StatusOK, http.StatusBadRequest, http.StatusOK}, }, { "imageNotFound", http.StatusOK, http.StatusOK, true, testCodesStruct{http.StatusOK, http.StatusOK, http.StatusOK, http.StatusNotFound}, }, { "imageBadRequest", http.StatusOK, http.StatusOK, true, testCodesStruct{http.StatusOK, http.StatusOK, http.StatusOK, http.StatusBadRequest}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s3Server := mockS3Server(t, tt.s3StatusCode) defer s3Server.Close() h := http.NewServeMux() h.HandleFunc("/v1/imagefile/", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { commonHandler(t, tt.statusCode, w) })) h.HandleFunc("/v1/tags/", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { resp := TagMap{"test": "testValue"} if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } })) h.HandleFunc("/v1/entities/", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { uploadImageHelperHandler(t, tt.codes.entity, w) resp := Entity{ID: "testID"} if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } })) h.HandleFunc("/v1/collections/", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { uploadImageHelperHandler(t, tt.codes.collection, w) resp := Collection{ ID: "testID", } if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } })) h.HandleFunc("/v1/containers/", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { uploadImageHelperHandler(t, tt.codes.container, w) resp := Container{ID: "test"} if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } })) h.HandleFunc("/v1/images/", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { uploadImageHelperHandler(t, tt.codes.image, w) commonHandler(t, tt.statusCode, w) resp := Image{ID: "testID"} if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } })) libraryServer := httptest.NewServer(h) defer libraryServer.Close() c := initClient(t, libraryServer.URL) r := strings.NewReader(testPayload) etag, err := c.UploadImage( context.Background(), r, "entity/collection/container", runtime.GOARCH, []string{"tag"}, "test", &defaultUploadCallback{r: r}, ) if (err != nil) != tt.expectError { t.Fatalf("unexpected error: %v", err) } _ = etag }) } } func Test_postFileWrapper(t *testing.T) { s3Server := mockS3Server(t, http.StatusOK) defer s3Server.Close() h := http.NewServeMux() h.HandleFunc("/v1/imagefile/", http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) libraryServer := httptest.NewServer(h) defer libraryServer.Close() c := initClient(t, libraryServer.URL) r := strings.NewReader(testPayload) etag, err := c.postFileWrapper( context.Background(), r, int64(len(testPayload)), "xxx", &defaultUploadCallback{r: r}, map[string]string{"sha256sum": "xxx"}, ) if err != nil { t.Fatalf("unexpected error: %v", err) } _ = etag } func Test_postFileV2(t *testing.T) { s3Server := mockS3Server(t, http.StatusOK) defer s3Server.Close() tests := []struct { name string size int64 expectError bool }{ {"basic", int64(len(testPayload)), false}, {"invalid", minimumPartSize + 1, true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { h := http.NewServeMux() h.HandleFunc("/v2/imagefile/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case strings.HasSuffix(r.URL.String(), "/_multipart"): switch r.Method { case http.MethodPost: resp := MultipartUploadStartResponse{ Data: MultipartUpload{ UploadID: "xxx", TotalParts: 1, PartSize: 123, Options: map[string]string{OptionS3Compliant: "true"}, }, } if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } case http.MethodPut: resp := UploadImagePartResponse{Data: UploadImagePart{PresignedURL: s3Server.URL}} if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } } case strings.HasSuffix(r.URL.String(), "/_multipart_complete"): resp := CompleteMultipartUploadResponse{Data: UploadImageComplete{}} if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } default: resp := UploadImageResponse{Data: UploadImage{UploadURL: s3Server.URL}} if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } } })) libraryServer := httptest.NewServer(h) defer libraryServer.Close() c := initClient(t, libraryServer.URL) r := strings.NewReader(testPayload) etag, err := c.postFileV2( context.Background(), r, tt.size, "xxx", &defaultUploadCallback{r: r}, map[string]string{"sha256sum": "xxx"}, ) if (err != nil) != tt.expectError { t.Fatalf("unexpected error: %v", err) } _ = etag }) } } func Test_postFileV2Multipart(t *testing.T) { tests := []struct { name string statusCode int s3StatusCode int expectError bool }{ {"statusOK", http.StatusOK, http.StatusOK, false}, {"internalServerError", http.StatusInternalServerError, http.StatusInternalServerError, true}, {"okBadRequest", http.StatusOK, http.StatusBadRequest, true}, {"badRequestOK", http.StatusBadRequest, http.StatusOK, true}, {"internalOK", http.StatusInternalServerError, http.StatusOK, true}, {"okInternal", http.StatusOK, http.StatusInternalServerError, true}, {"internalBadRequest", http.StatusInternalServerError, http.StatusBadRequest, true}, {"badRequestInternal", http.StatusBadRequest, http.StatusInternalServerError, true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s3Server := mockS3Server(t, tt.s3StatusCode) defer s3Server.Close() h := http.NewServeMux() h.HandleFunc("/v2/imagefile/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { commonHandler(t, tt.statusCode, w) if strings.HasSuffix(r.URL.String(), "/_multipart") { switch r.Method { case http.MethodPost: resp := MultipartUploadStartResponse{ Data: MultipartUpload{ UploadID: "xxx", TotalParts: 1, PartSize: 123456, Options: map[string]string{OptionS3Compliant: "true"}, }, } if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } case http.MethodPut: resp := UploadImagePartResponse{Data: UploadImagePart{PresignedURL: s3Server.URL}} if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } } } else if strings.HasSuffix(r.URL.String(), "/_multipart_complete") { resp := CompleteMultipartUploadResponse{Data: UploadImageComplete{}} if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } } })) libraryServer := httptest.NewServer(h) defer libraryServer.Close() c := initClient(t, libraryServer.URL) r := strings.NewReader(testPayload) etag, err := c.postFileV2Multipart( context.Background(), r, int64(len(testPayload)), "xxx", &defaultUploadCallback{r: r}, ) if (err != nil) != tt.expectError { t.Fatalf("unexpected error: %v", err) } _ = etag }) } } func Test_getPartSize(t *testing.T) { tests := []struct { name string bytesRemaining int64 partSize int64 want int64 }{ {"moreBytesThanParts", 2, 1, 1}, {"morePartsThanBytes", 1, 2, 1}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got, want := getPartSize(tt.bytesRemaining, tt.partSize), tt.want; got != want { t.Fatalf("got: %v, want: %v", got, want) } }) } } func Test_startMultipartUpload(t *testing.T) { s3Server := mockS3Server(t, http.StatusOK) defer s3Server.Close() tests := []struct { name string statusCode int expectError bool }{ {"success", http.StatusOK, false}, {"badRequest", http.StatusBadRequest, true}, {"internalServerError", http.StatusInternalServerError, true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { h := http.NewServeMux() h.HandleFunc("/v2/imagefile/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { commonHandler(t, tt.statusCode, w) if strings.HasSuffix(r.URL.String(), "/_multipart") { resp := MultipartUploadStartResponse{ Data: MultipartUpload{ UploadID: "testUploadID", TotalParts: 1, PartSize: 123456, Options: map[string]string{OptionS3Compliant: "true"}, }, } if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } } })) libraryServer := httptest.NewServer(h) defer libraryServer.Close() c := initClient(t, libraryServer.URL) multi, err := c.startMultipartUpload(context.Background(), 0, "testID") if (err != nil) != tt.expectError { t.Fatalf("error uploading part: %v", err) } _ = multi }) } } func Test_remoteSHA256ChecksumSupport(t *testing.T) { tests := []struct { name string value string want bool }{ {"basic", "x-amz-content-sha256", true}, {"NotMatch", "x-amz-content-sha256x", false}, {"NoValue", "", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { q := url.Values{} q.Set("X-Amz-SignedHeaders", tt.value) u := &url.URL{RawQuery: q.Encode()} if got, want := remoteSHA256ChecksumSupport(u), tt.want; got != want { t.Fatalf("got: %v, want: %v", got, want) } }) } } func Test_legacyPostFileV2(t *testing.T) { tests := []struct { name string imageRef string testFile string }{ { name: "Basic", imageRef: "5cb9c34d7d960d82f5f5bc55", testFile: "test_data/test_sha256", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { m := v2ImageUploadMockService{ t: t, } m.Run() defer m.Stop() c := initClient(t, m.baseURI) f, err := os.Open(tt.testFile) if err != nil { t.Errorf("Error opening file %s for reading: %v", tt.testFile, err) } defer f.Close() fi, err := f.Stat() if err != nil { t.Errorf("Error stats for file %s: %v", tt.testFile, err) } fileSize := fi.Size() // calculate sha256 checksum sha256checksum, _, err := sha256sum(f) if err != nil { t.Fatalf("error calculating sha256 checksum: %v", err) } _, err = f.Seek(0, 0) if err != nil { t.Fatalf("unexpected error seeking in sample data file: %v", err) } callback := &defaultUploadCallback{r: f} // include sha256 checksum in metadata resp, err := c.legacyPostFileV2(context.Background(), fileSize, tt.imageRef, callback, map[string]string{ "sha256sum": sha256checksum, }) if err != nil { t.Fatalf("unexpected error: %v", err) } if got, want := resp.Quota.QuotaUsageBytes, testQuotaUsageBytes; got != want { t.Errorf("got quota usage %v, want %v", got, want) } if got, want := resp.Quota.QuotaTotalBytes, testQuotaTotalBytes; got != want { t.Errorf("got quota total %v, want %v", got, want) } if got, want := resp.ContainerURL, testContainerURL; got != want { t.Errorf("got container URL %v, want %v", got, want) } if !m.initCalled { t.Errorf("init image upload request was not made") } if !m.putCalled { t.Errorf("file PUT request was not made") } if !m.completeCalled { t.Errorf("image upload complete request was not made") } }) } } func Test_legacyPostFileV2URL(t *testing.T) { s3Server := mockS3Server(t, http.StatusOK) defer s3Server.Close() tests := []struct { name string url string expectError bool }{ {"basic", s3Server.URL, false}, {"emptyURL", "", true}, {"parseURLError", "\n", true}, {"unsupported", "test", true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { h := http.NewServeMux() h.HandleFunc("/v2/imagefile/", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { commonHandler(t, http.StatusOK, w) resp := UploadImageResponse{Data: UploadImage{UploadURL: tt.url}} if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } })) libraryServer := httptest.NewServer(h) defer libraryServer.Close() c := initClient(t, libraryServer.URL) r := strings.NewReader(testPayload) etag, err := c.legacyPostFileV2( context.Background(), 0, "xxx", &defaultUploadCallback{r: r}, map[string]string{"sha256sum": "xxx"}, ) if (err != nil) != tt.expectError { t.Fatalf("unexpected error: %v", err) } _ = etag }) } } func Test_Test_multipartUploadPartBadSize(t *testing.T) { s3Server := mockS3Server(t, http.StatusOK) defer s3Server.Close() h := http.NewServeMux() h.HandleFunc("/v2/imagefile/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { commonHandler(t, http.StatusOK, w) if strings.HasSuffix(r.URL.String(), "/_multipart") { resp := UploadImagePartResponse{Data: UploadImagePart{PresignedURL: s3Server.URL}} if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } } })) libraryServer := httptest.NewServer(h) defer libraryServer.Close() c := initClient(t, libraryServer.URL) r := strings.NewReader(testPayload) etag, err := c.multipartUploadPart( context.Background(), 0, &uploadManager{ Source: r, Size: int64(len(testPayload)) + 1, ImageID: "testImageID", UploadID: "testUploadID", }, &defaultUploadCallback{r: r}, true, ) if err == nil { t.Fatal("unexpected success") } _ = etag } func Test_multipartUploadPart(t *testing.T) { tests := []struct { name string statusCode int s3StatusCode int expectError bool }{ {"statusOK", http.StatusOK, http.StatusOK, false}, {"badRequest", http.StatusBadRequest, http.StatusBadRequest, true}, {"internalServerError", http.StatusInternalServerError, http.StatusInternalServerError, true}, {"okBadRequest", http.StatusOK, http.StatusBadRequest, true}, {"badRequestOK", http.StatusBadRequest, http.StatusOK, true}, {"internalOK", http.StatusInternalServerError, http.StatusOK, true}, {"okInternal", http.StatusOK, http.StatusInternalServerError, true}, {"internalBadRequest", http.StatusInternalServerError, http.StatusBadRequest, true}, {"badRequestInternal", http.StatusBadRequest, http.StatusInternalServerError, true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s3Server := mockS3Server(t, tt.s3StatusCode) defer s3Server.Close() h := http.NewServeMux() h.HandleFunc("/v2/imagefile/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { commonHandler(t, tt.statusCode, w) if strings.HasSuffix(r.URL.String(), "/_multipart") { resp := UploadImagePartResponse{Data: UploadImagePart{PresignedURL: s3Server.URL}} if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } } })) libraryServer := httptest.NewServer(h) defer libraryServer.Close() c := initClient(t, libraryServer.URL) r := strings.NewReader(testPayload) etag, err := c.multipartUploadPart( context.Background(), 0, &uploadManager{ Source: r, Size: int64(len(testPayload)), ImageID: "testImageID", UploadID: "testUploadID", }, &defaultUploadCallback{r: r}, true, ) if (err != nil) != tt.expectError { t.Fatalf("error uploading part: %v", err) } _ = etag }) } } func Test_completeMultipartUpload(t *testing.T) { s3Server := mockS3Server(t, http.StatusOK) defer s3Server.Close() tests := []struct { name string statusCode int expectError bool }{ {"statusOK", http.StatusOK, false}, {"badRequest", http.StatusBadRequest, true}, {"internalServerError", http.StatusInternalServerError, true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { h := http.NewServeMux() h.HandleFunc("/v2/imagefile/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { commonHandler(t, tt.statusCode, w) if strings.HasSuffix(r.URL.String(), "/_multipart_complete") { resp := CompleteMultipartUploadResponse{Data: UploadImageComplete{ContainerURL: s3Server.URL}} if err := json.NewEncoder(w).Encode(&resp); err != nil { t.Fatalf("Error encoding JSON response: %v", err) } } })) libraryServer := httptest.NewServer(h) defer libraryServer.Close() c := initClient(t, libraryServer.URL) etag, err := c.completeMultipartUpload( context.Background(), &[]CompletedPart{{PartNumber: 0, Token: "xxx"}}, &uploadManager{ Source: strings.NewReader(testPayload), Size: int64(len(testPayload)), ImageID: "testImageID", UploadID: "testUploadID", }) if (err != nil) != tt.expectError { t.Fatalf("error uploading part: %v", err) } _ = etag }) } } func Test_abortMultipartUpload(t *testing.T) { s3Server := mockS3Server(t, http.StatusOK) defer s3Server.Close() tests := []struct { name string statusCode int expectError bool }{ {"statusOK", http.StatusOK, false}, {"badRequest", http.StatusBadRequest, true}, {"internalServerError", http.StatusInternalServerError, true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { h := http.NewServeMux() h.HandleFunc("/v2/imagefile/", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { uploadImageHelperHandler(t, tt.statusCode, w) })) libraryServer := httptest.NewServer(h) defer libraryServer.Close() c := initClient(t, libraryServer.URL) err := c.abortMultipartUpload( context.Background(), &uploadManager{ Source: strings.NewReader(testPayload), Size: int64(len(testPayload)), ImageID: "testImageID", UploadID: "testUploadID", }) if (err != nil) != tt.expectError { t.Fatalf("unexpected error: %v", err) } }) } } container-library-client-1.4.10/client/ref.go000066400000000000000000000123721471454260400210720ustar00rootroot00000000000000// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( "net/url" "strings" ) // Scheme is the required scheme for Library URIs. const Scheme = "library" // A Ref represents a parsed Library URI. // // The general form represented is: // // scheme:[//host][/]path[:tags] // // The host contains both the hostname and port, if present. These values can be accessed using // the Hostname and Port methods. // // Examples of valid URIs: // // library:path:tags // library:/path:tags // library:///path:tags // library://host/path:tags // library://host:port/path:tags // // The tags component is a comma-separated list of one or more tags. type Ref struct { Host string // host or host:port Path string // project or entity/project Tags []string // list of tags } // parseTags takes raw tags and returns a slice of tags. func parseTags(rawTags string) (tags []string, err error) { if len(rawTags) == 0 { return nil, ErrRefTagsNotValid } return strings.Split(rawTags, ","), nil } // parsePath takes the URI path and parses the path and tags. func parsePath(rawPath string) (path string, tags []string, err error) { if len(rawPath) == 0 { return "", nil, ErrRefPathNotValid } // The path is separated from the tags (if present) by a single colon. parts := strings.Split(rawPath, ":") if len(parts) > 2 { return "", nil, ErrRefPathNotValid } // TODO: not sure we should modify the path here... // Name can optionally start with a leading "/". path = strings.TrimPrefix(parts[0], "/") if len(path) == 0 { return "", nil, ErrRefPathNotValid } if len(parts) > 1 { tags, err = parseTags(parts[1]) if err != nil { return "", nil, err } } else { tags = nil } return path, tags, nil } // parse parses a raw Library reference, optionally taking into account ambiguity that exists // within Library references. func parse(rawRef string, ambiguous bool) (*Ref, error) { var u *url.URL if ambiguous && strings.HasPrefix(rawRef, "library://") && !strings.HasPrefix(rawRef, "library:///") { // Parse as if there's no host component. uri, err := url.Parse(strings.Replace(rawRef, "library://", "library:///", 1)) if err != nil { return nil, err } // If the path contains one or three parts, there was no host component. Otherwise, fall // through to the normal logic. if n := len(strings.Split(uri.Path[1:], "/")); n == 1 || n == 3 { u = uri } } if u == nil { var err error if u, err = url.Parse(rawRef); err != nil { return nil, err } } if u.Scheme != Scheme { return nil, ErrRefSchemeNotValid } if u.User != nil { return nil, ErrRefUserNotPermitted } if u.RawQuery != "" { return nil, ErrRefQueryNotPermitted } if u.Fragment != "" { return nil, ErrRefFragmentNotPermitted } rawPath := u.Path if u.Opaque != "" { rawPath = u.Opaque } path, tags, err := parsePath(rawPath) if err != nil { return nil, err } r := &Ref{ Host: u.Host, Path: path, Tags: tags, } return r, nil } // Parse parses a raw Library reference. func Parse(rawRef string) (*Ref, error) { return parse(rawRef, false) } // ParseAmbiguous behaves like Parse, but takes into account ambiguity that exists within // Library references that begin with the prefix "library://". // // In particular, Apptainer supports hostless Library references in the form of "library://path". // This creates ambiguity in whether or not a host is present in the path or not. To account for // this, ParseAmbiguous treats library references beginning with "library://" followed by one or // three path components (ex. "library://a", "library://a/b/c") as hostless. All other references // are treated the same as Parse. func ParseAmbiguous(rawRef string) (*Ref, error) { return parse(rawRef, true) } // String reassembles the ref into a valid URI string. The general form of the result is one of: // // scheme:path[:tags] // scheme://host/path[:tags] // // If u.Host is empty, String uses the first form; otherwise it uses the second form. func (r *Ref) String() string { u := url.URL{ Scheme: Scheme, Host: r.Host, } rawPath := r.Path if len(r.Tags) > 0 { rawPath += ":" + strings.Join(r.Tags, ",") } if u.Host != "" { u.Path = rawPath } else { u.Opaque = rawPath } return u.String() } // Hostname returns r.Host, without any port number. // // If Host is an IPv6 literal with a port number, Hostname returns the IPv6 literal without the // square brackets. IPv6 literals may include a zone identifier. func (r *Ref) Hostname() string { colon := strings.IndexByte(r.Host, ':') if colon == -1 { return r.Host } if i := strings.IndexByte(r.Host, ']'); i != -1 { return strings.TrimPrefix(r.Host[:i], "[") } return r.Host[:colon] } // Port returns the port part of u.Host, without the leading colon. If u.Host doesn't contain a // port, Port returns an empty string. func (r *Ref) Port() string { colon := strings.IndexByte(r.Host, ':') if colon == -1 { return "" } if i := strings.Index(r.Host, "]:"); i != -1 { return r.Host[i+len("]:"):] } if strings.Contains(r.Host, "]") { return "" } return r.Host[colon+len(":"):] } container-library-client-1.4.10/client/ref_test.go000066400000000000000000000655551471454260400221440ustar00rootroot00000000000000// Copyright (c) 2018-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( "errors" "reflect" "testing" ) func TestParse(t *testing.T) { tests := []struct { name string rawRef string wantErr bool wantErrSpecific error wantHost string wantPath string wantTags []string }{ // The URI must be valid. {"InvalidURL", ":", true, nil, "", "", nil}, // A path is required. {"NoName", "library:", true, ErrRefPathNotValid, "", "", nil}, {"NoNameAndTag", "library::tag", true, ErrRefPathNotValid, "", "", nil}, {"NoNameAndSlash", "library:/", true, ErrRefPathNotValid, "", "", nil}, {"NoNameAndSlashAndTag", "library:/:tag", true, ErrRefPathNotValid, "", "", nil}, {"NoNameAndSlashes", "library:///", true, ErrRefPathNotValid, "", "", nil}, {"NoNameAndSlashesAndTag", "library:///:tag", true, ErrRefPathNotValid, "", "", nil}, {"NoNameAndHost", "library://host", true, ErrRefPathNotValid, "", "", nil}, {"NoNameAndHostSlash", "library://host/", true, ErrRefPathNotValid, "", "", nil}, {"NoNameAndHostSlashAndTag", "library://host/:tag", true, ErrRefPathNotValid, "", "", nil}, {"NoNameAndHostPort", "library://host:443", true, ErrRefPathNotValid, "", "", nil}, {"NoNameAndHostPortSlash", "library://host:443/", true, ErrRefPathNotValid, "", "", nil}, {"NoNameAndHostPortSlashAndTag", "library://host:443/:tag", true, ErrRefPathNotValid, "", "", nil}, // The scheme must be present. {"NoScheme", "", true, ErrRefSchemeNotValid, "", "", nil}, {"NoSchemeSlash", "/", true, ErrRefSchemeNotValid, "", "", nil}, {"NoSchemeSlashAndTag", "/:tag", true, ErrRefSchemeNotValid, "", "", nil}, {"NoSchemeSlashes", "///", true, ErrRefSchemeNotValid, "", "", nil}, {"NoSchemeSlashesAndTag", "///:tag", true, ErrRefSchemeNotValid, "", "", nil}, {"NoSchemeAndHost", "//host/", true, ErrRefSchemeNotValid, "", "", nil}, {"NoSchemeAndHostAndTag", "//host/:tag", true, ErrRefSchemeNotValid, "", "", nil}, {"NoSchemeAndHostPort", "//host:443/", true, ErrRefSchemeNotValid, "", "", nil}, {"NoSchemeAndHostPortAndTag", "//host:443/:tag", true, ErrRefSchemeNotValid, "", "", nil}, // The scheme must be valid. {"BadScheme", "bad:project", true, ErrRefSchemeNotValid, "", "", nil}, {"BadSchemeAndTag", "bad:project:tag", true, ErrRefSchemeNotValid, "", "", nil}, {"BadSchemeAndSlash", "bad:/project", true, ErrRefSchemeNotValid, "", "", nil}, {"BadSchemeAndSlashAndTag", "bad:/project:tag", true, ErrRefSchemeNotValid, "", "", nil}, {"BadSchemeAndSlashes", "bad:///project", true, ErrRefSchemeNotValid, "", "", nil}, {"BadSchemeAndSlashesAndTag", "bad:///project:tag", true, ErrRefSchemeNotValid, "", "", nil}, {"BadSchemeAndHost", "bad://host/project", true, ErrRefSchemeNotValid, "", "", nil}, {"BadSchemeAndHostAndTag", "bad://host/project:tag", true, ErrRefSchemeNotValid, "", "", nil}, {"BadSchemeAndHostPort", "bad://host:443/project", true, ErrRefSchemeNotValid, "", "", nil}, {"BadSchemeAndHostPortAndTag", "bad://host:443/project:tag", true, ErrRefSchemeNotValid, "", "", nil}, // User not permitted. {"UserAndHost", "library://user@host/project", true, ErrRefUserNotPermitted, "", "", nil}, {"UserAndHostAndTag", "library://user@host/project:tag", true, ErrRefUserNotPermitted, "", "", nil}, {"UserAndHostPort", "library://user@host:443/project", true, ErrRefUserNotPermitted, "", "", nil}, {"UserAndHostPortAndTag", "library://user@host:443/project:tag", true, ErrRefUserNotPermitted, "", "", nil}, // Query not permitted. {"Query", "library:project?query", true, ErrRefQueryNotPermitted, "", "", nil}, {"QueryAndTag", "library:project:tag?query", true, ErrRefQueryNotPermitted, "", "", nil}, {"QueryAndSlash", "library:/project?query", true, ErrRefQueryNotPermitted, "", "", nil}, {"QueryAndSlashAndTag", "library:/project:tag?query", true, ErrRefQueryNotPermitted, "", "", nil}, {"QueryAndSlashes", "library:///project?query", true, ErrRefQueryNotPermitted, "", "", nil}, {"QueryAndSlashesAndTag", "library:///project:tag?query", true, ErrRefQueryNotPermitted, "", "", nil}, {"QueryAndHost", "library://host/project?query", true, ErrRefQueryNotPermitted, "", "", nil}, {"QueryAndHostAndTag", "library://host/project:tag?query", true, ErrRefQueryNotPermitted, "", "", nil}, {"QueryAndHostPort", "library://host:443/project?query", true, ErrRefQueryNotPermitted, "", "", nil}, {"QueryAndHostPortAndTag", "library://host:443/project:tag?query", true, ErrRefQueryNotPermitted, "", "", nil}, // Fragment not permitted. {"Fragment", "library:project#fragment", true, ErrRefFragmentNotPermitted, "", "", nil}, {"FragmentAndTag", "library:project:tag#fragment", true, ErrRefFragmentNotPermitted, "", "", nil}, {"FragmentAndSlash", "library:/project#fragment", true, ErrRefFragmentNotPermitted, "", "", nil}, {"FragmentAndSlashAndTag", "library:/project:tag#fragment", true, ErrRefFragmentNotPermitted, "", "", nil}, {"FragmentAndSlashes", "library:///project#fragment", true, ErrRefFragmentNotPermitted, "", "", nil}, {"FragmentAndSlashesAndTag", "library:///project:tag#fragment", true, ErrRefFragmentNotPermitted, "", "", nil}, {"FragmentAndHost", "library://host/project#fragment", true, ErrRefFragmentNotPermitted, "", "", nil}, {"FragmentAndHostAndTag", "library://host/project:tag#fragment", true, ErrRefFragmentNotPermitted, "", "", nil}, {"FragmentAndHostPort", "library://host:443/project#fragment", true, ErrRefFragmentNotPermitted, "", "", nil}, {"FragmentAndHostPortAndTag", "library://host:443/project:tag#fragment", true, ErrRefFragmentNotPermitted, "", "", nil}, // The URI cannot have a trailing colon without tags. {"TrailingSemicolon", "library:project:", true, ErrRefTagsNotValid, "", "", nil}, {"TrailingSemicolonAndSlash", "library:/project:", true, ErrRefTagsNotValid, "", "", nil}, {"TrailingSemicolonAndSlashes", "library:///project:", true, ErrRefTagsNotValid, "", "", nil}, {"TrailingSemicolonAndHost", "library://host/project:", true, ErrRefTagsNotValid, "", "", nil}, {"TrailingSemicolonAndHostPort", "library://host:443/project:", true, ErrRefTagsNotValid, "", "", nil}, // The URI path can only have one colon to separate path/tags. {"ExtraSemicolon", "library:project:tag:extra", true, ErrRefPathNotValid, "", "", nil}, {"ExtraSemicolonAndSlash", "library:/project:tag:extra", true, ErrRefPathNotValid, "", "", nil}, {"ExtraSemicolonAndSlashes", "library:///project:tag:extra", true, ErrRefPathNotValid, "", "", nil}, {"ExtraSemicolonAndHost", "library://host/project:tag:extra", true, ErrRefPathNotValid, "", "", nil}, {"ExtraSemicolonAhdHostPort", "library://host:443/project:tag:extra", true, ErrRefPathNotValid, "", "", nil}, // Test valid abbreviated paths (project). {"AbbreviatedPath", "library:project", false, nil, "", "project", nil}, {"AbbreviatedPathAndTag", "library:project:tag", false, nil, "", "project", []string{"tag"}}, {"AbbreviatedPathAndSlash", "library:/project", false, nil, "", "project", nil}, {"AbbreviatedPathAndSlashAndTag", "library:/project:tag", false, nil, "", "project", []string{"tag"}}, {"AbbreviatedPathAndSlashes", "library:///project", false, nil, "", "project", nil}, {"AbbreviatedPathAndSlashesAndTag", "library:///project:tag", false, nil, "", "project", []string{"tag"}}, {"AbbreviatedPathAndHost", "library://host/project", false, nil, "host", "project", nil}, {"AbbreviatedPathAndHostAndTag", "library://host/project:tag", false, nil, "host", "project", []string{"tag"}}, {"AbbreviatedPathAndHostPort", "library://host:443/project", false, nil, "host:443", "project", nil}, {"AbbreviatedPathAndHostPortAndTag", "library://host:443/project:tag", false, nil, "host:443", "project", []string{"tag"}}, // Test valid paths (entity/collection/container). {"LegacyPath", "library:entity/collection/container", false, nil, "", "entity/collection/container", nil}, {"LegacyPathAndTag", "library:entity/collection/container:tag", false, nil, "", "entity/collection/container", []string{"tag"}}, {"LegacyPathAndSlash", "library:/entity/collection/container", false, nil, "", "entity/collection/container", nil}, {"LegacyPathAndSlashAndTag", "library:/entity/collection/container:tag", false, nil, "", "entity/collection/container", []string{"tag"}}, {"LegacyPathAndSlashes", "library:///entity/collection/container", false, nil, "", "entity/collection/container", nil}, {"LegacyPathAndSlashesAndTag", "library:///entity/collection/container:tag", false, nil, "", "entity/collection/container", []string{"tag"}}, {"LegacyPathAndHost", "library://host/entity/collection/container", false, nil, "host", "entity/collection/container", nil}, {"LegacyPathAndHostAndTag", "library://host/entity/collection/container:tag", false, nil, "host", "entity/collection/container", []string{"tag"}}, {"LegacyPathAndHostPort", "library://host:443/entity/collection/container", false, nil, "host:443", "entity/collection/container", nil}, {"LegacyPathAndHostPortAndTag", "library://host:443/entity/collection/container:tag", false, nil, "host:443", "entity/collection/container", []string{"tag"}}, // Test with a different number of tags. {"TagsNone", "library:project", false, nil, "", "project", nil}, {"TagsOne", "library:project:tag1", false, nil, "", "project", []string{"tag1"}}, {"TagsTwo", "library:project:tag1,tag2", false, nil, "", "project", []string{"tag1", "tag2"}}, {"TagsThree", "library:project:tag1,tag2,tag3", false, nil, "", "project", []string{"tag1", "tag2", "tag3"}}, // Test with IP addresses. {"IPv4Host", "library://127.0.0.1/project", false, nil, "127.0.0.1", "project", nil}, {"IPv4HostPort", "library://127.0.0.1:443/project", false, nil, "127.0.0.1:443", "project", nil}, {"IPv6Host", "library://[fe80::1ff:fe23:4567:890a]/project", false, nil, "[fe80::1ff:fe23:4567:890a]", "project", nil}, {"IPv6HostPort", "library://[fe80::1ff:fe23:4567:890a]:443/project", false, nil, "[fe80::1ff:fe23:4567:890a]:443", "project", nil}, {"IPv6HostZone", "library://[fe80::1ff:fe23:4567:890a%25eth1]/project", false, nil, "[fe80::1ff:fe23:4567:890a%eth1]", "project", nil}, {"IPv6HostZonePort", "library://[fe80::1ff:fe23:4567:890a%25eth1]:443/project", false, nil, "[fe80::1ff:fe23:4567:890a%eth1]:443", "project", nil}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { r, err := Parse(tt.rawRef) if (err != nil) != tt.wantErr { t.Fatalf("got err %v, want %v", err, tt.wantErr) } if tt.wantErrSpecific != nil { if got, want := err, tt.wantErrSpecific; !errors.Is(got, want) { t.Fatalf("got err %v, want %v", got, want) } } if err == nil { if got, want := r.Host, tt.wantHost; got != want { t.Errorf("got host %v, want %v", got, want) } if got, want := r.Path, tt.wantPath; got != want { t.Errorf("got path %v, want %v", got, want) } if got, want := r.Tags, tt.wantTags; !reflect.DeepEqual(got, want) { t.Errorf("got tags %v, want %v", got, want) } } }) } } func TestParseAmbiguous(t *testing.T) { tests := []struct { name string rawRef string wantErr bool wantErrSpecific error wantHost string wantPath string wantTags []string }{ // The URI must be valid. {"InvalidURL", ":", true, nil, "", "", nil}, // A path is required. {"NoName", "library:", true, ErrRefPathNotValid, "", "", nil}, {"NoNameAndTag", "library::tag", true, ErrRefPathNotValid, "", "", nil}, {"NoNameAndSlash", "library:/", true, ErrRefPathNotValid, "", "", nil}, {"NoNameAndSlashAndTag", "library:/:tag", true, ErrRefPathNotValid, "", "", nil}, {"NoNameAndSlashes", "library:///", true, ErrRefPathNotValid, "", "", nil}, {"NoNameAndSlashesAndTag", "library:///:tag", true, ErrRefPathNotValid, "", "", nil}, {"NoNameAndHostSlash", "library://host/", true, ErrRefPathNotValid, "", "", nil}, {"NoNameAndHostSlashAndTag", "library://host/:tag", true, ErrRefPathNotValid, "", "", nil}, {"NoNameAndHostPortSlash", "library://host:443/", true, ErrRefPathNotValid, "", "", nil}, {"NoNameAndHostPortSlashAndTag", "library://host:443/:tag", true, ErrRefPathNotValid, "", "", nil}, // The scheme must be present. {"NoScheme", "", true, ErrRefSchemeNotValid, "", "", nil}, {"NoSchemeSlash", "/", true, ErrRefSchemeNotValid, "", "", nil}, {"NoSchemeSlashAndTag", "/:tag", true, ErrRefSchemeNotValid, "", "", nil}, {"NoSchemeSlashes", "///", true, ErrRefSchemeNotValid, "", "", nil}, {"NoSchemeSlashesAndTag", "///:tag", true, ErrRefSchemeNotValid, "", "", nil}, {"NoSchemeAndHost", "//host/", true, ErrRefSchemeNotValid, "", "", nil}, {"NoSchemeAndHostAndTag", "//host/:tag", true, ErrRefSchemeNotValid, "", "", nil}, {"NoSchemeAndHostPort", "//host:443/", true, ErrRefSchemeNotValid, "", "", nil}, {"NoSchemeAndHostPortAndTag", "//host:443/:tag", true, ErrRefSchemeNotValid, "", "", nil}, // The scheme must be valid. {"BadScheme", "bad:project", true, ErrRefSchemeNotValid, "", "", nil}, {"BadSchemeAndTag", "bad:project:tag", true, ErrRefSchemeNotValid, "", "", nil}, {"BadSchemeAndSlash", "bad:/project", true, ErrRefSchemeNotValid, "", "", nil}, {"BadSchemeAndSlashAndTag", "bad:/project:tag", true, ErrRefSchemeNotValid, "", "", nil}, {"BadSchemeAndSlashes", "bad:///project", true, ErrRefSchemeNotValid, "", "", nil}, {"BadSchemeAndSlashesAndTag", "bad:///project:tag", true, ErrRefSchemeNotValid, "", "", nil}, {"BadSchemeAndHost", "bad://host/project", true, ErrRefSchemeNotValid, "", "", nil}, {"BadSchemeAndHostAndTag", "bad://host/project:tag", true, ErrRefSchemeNotValid, "", "", nil}, {"BadSchemeAndHostPort", "bad://host:443/project", true, ErrRefSchemeNotValid, "", "", nil}, {"BadSchemeAndHostPortAndTag", "bad://host:443/project:tag", true, ErrRefSchemeNotValid, "", "", nil}, // User not permitted. {"UserAndHost", "library://user@host/project", true, ErrRefUserNotPermitted, "", "", nil}, {"UserAndHostAndTag", "library://user@host/project:tag", true, ErrRefUserNotPermitted, "", "", nil}, {"UserAndHostPort", "library://user@host:443/project", true, ErrRefUserNotPermitted, "", "", nil}, {"UserAndHostPortAndTag", "library://user@host:443/project:tag", true, ErrRefUserNotPermitted, "", "", nil}, // Query not permitted. {"Query", "library:project?query", true, ErrRefQueryNotPermitted, "", "", nil}, {"QueryAndTag", "library:project:tag?query", true, ErrRefQueryNotPermitted, "", "", nil}, {"QueryAndSlash", "library:/project?query", true, ErrRefQueryNotPermitted, "", "", nil}, {"QueryAndSlashAndTag", "library:/project:tag?query", true, ErrRefQueryNotPermitted, "", "", nil}, {"QueryAndSlashes", "library:///project?query", true, ErrRefQueryNotPermitted, "", "", nil}, {"QueryAndSlashesAndTag", "library:///project:tag?query", true, ErrRefQueryNotPermitted, "", "", nil}, {"QueryAndHost", "library://host/project?query", true, ErrRefQueryNotPermitted, "", "", nil}, {"QueryAndHostAndTag", "library://host/project:tag?query", true, ErrRefQueryNotPermitted, "", "", nil}, {"QueryAndHostPort", "library://host:443/project?query", true, ErrRefQueryNotPermitted, "", "", nil}, {"QueryAndHostPortAndTag", "library://host:443/project:tag?query", true, ErrRefQueryNotPermitted, "", "", nil}, // Fragment not permitted. {"Fragment", "library:project#fragment", true, ErrRefFragmentNotPermitted, "", "", nil}, {"FragmentAndTag", "library:project:tag#fragment", true, ErrRefFragmentNotPermitted, "", "", nil}, {"FragmentAndSlash", "library:/project#fragment", true, ErrRefFragmentNotPermitted, "", "", nil}, {"FragmentAndSlashAndTag", "library:/project:tag#fragment", true, ErrRefFragmentNotPermitted, "", "", nil}, {"FragmentAndSlashes", "library:///project#fragment", true, ErrRefFragmentNotPermitted, "", "", nil}, {"FragmentAndSlashesAndTag", "library:///project:tag#fragment", true, ErrRefFragmentNotPermitted, "", "", nil}, {"FragmentAndHost", "library://host/project#fragment", true, ErrRefFragmentNotPermitted, "", "", nil}, {"FragmentAndHostAndTag", "library://host/project:tag#fragment", true, ErrRefFragmentNotPermitted, "", "", nil}, {"FragmentAndHostPort", "library://host:443/project#fragment", true, ErrRefFragmentNotPermitted, "", "", nil}, {"FragmentAndHostPortAndTag", "library://host:443/project:tag#fragment", true, ErrRefFragmentNotPermitted, "", "", nil}, // The URI cannot have a trailing colon without tags. {"TrailingSemicolon", "library:project:", true, ErrRefTagsNotValid, "", "", nil}, {"TrailingSemicolonAndSlash", "library:/project:", true, ErrRefTagsNotValid, "", "", nil}, {"TrailingSemicolonAndSlashes", "library:///project:", true, ErrRefTagsNotValid, "", "", nil}, {"TrailingSemicolonAndHost", "library://host/project:", true, ErrRefTagsNotValid, "", "", nil}, {"TrailingSemicolonAndHostPort", "library://host:443/project:", true, ErrRefTagsNotValid, "", "", nil}, // The URI path can only have one colon to separate path/tags. {"ExtraSemicolon", "library:project:tag:extra", true, ErrRefPathNotValid, "", "", nil}, {"ExtraSemicolonAndSlash", "library:/project:tag:extra", true, ErrRefPathNotValid, "", "", nil}, {"ExtraSemicolonAndSlashes", "library:///project:tag:extra", true, ErrRefPathNotValid, "", "", nil}, {"ExtraSemicolonAndHost", "library://host/project:tag:extra", true, ErrRefPathNotValid, "", "", nil}, {"ExtraSemicolonAhdHostPort", "library://host:443/project:tag:extra", true, ErrRefPathNotValid, "", "", nil}, // Test valid abbreviated paths (project). {"AbbreviatedPath", "library:project", false, nil, "", "project", nil}, {"AbbreviatedPathAndTag", "library:project:tag", false, nil, "", "project", []string{"tag"}}, {"AbbreviatedPathAndSlash", "library:/project", false, nil, "", "project", nil}, {"AbbreviatedPathAndSlashAndTag", "library:/project:tag", false, nil, "", "project", []string{"tag"}}, {"AbbreviatedPathAndSlashes", "library:///project", false, nil, "", "project", nil}, {"AbbreviatedPathAndSlashesAndTag", "library:///project:tag", false, nil, "", "project", []string{"tag"}}, {"AbbreviatedPathHostless", "library://project", false, nil, "", "project", nil}, {"AbbreviatedPathHostlessAndTag", "library://project:tag", false, nil, "", "project", []string{"tag"}}, {"AbbreviatedPathAndHost", "library://host/project", false, nil, "host", "project", nil}, {"AbbreviatedPathAndHostAndTag", "library://host/project:tag", false, nil, "host", "project", []string{"tag"}}, {"AbbreviatedPathAndHostPort", "library://host:443/project", false, nil, "host:443", "project", nil}, {"AbbreviatedPathAndHostPortAndTag", "library://host:443/project:tag", false, nil, "host:443", "project", []string{"tag"}}, // Test valid paths (entity/collection/container). {"LegacyPath", "library:entity/collection/container", false, nil, "", "entity/collection/container", nil}, {"LegacyPathAndTag", "library:entity/collection/container:tag", false, nil, "", "entity/collection/container", []string{"tag"}}, {"LegacyPathAndSlash", "library:/entity/collection/container", false, nil, "", "entity/collection/container", nil}, {"LegacyPathAndSlashAndTag", "library:/entity/collection/container:tag", false, nil, "", "entity/collection/container", []string{"tag"}}, {"LegacyPathAndSlashes", "library:///entity/collection/container", false, nil, "", "entity/collection/container", nil}, {"LegacyPathAndSlashesAndTag", "library:///entity/collection/container:tag", false, nil, "", "entity/collection/container", []string{"tag"}}, {"LegacyPathHostless", "library://entity/collection/container", false, nil, "", "entity/collection/container", nil}, {"LegacyPathHostlessAndTag", "library://entity/collection/container:tag", false, nil, "", "entity/collection/container", []string{"tag"}}, {"LegacyPathAndHost", "library://host/entity/collection/container", false, nil, "host", "entity/collection/container", nil}, {"LegacyPathAndHostAndTag", "library://host/entity/collection/container:tag", false, nil, "host", "entity/collection/container", []string{"tag"}}, {"LegacyPathAndHostPort", "library://host:443/entity/collection/container", false, nil, "host:443", "entity/collection/container", nil}, {"LegacyPathAndHostPortAndTag", "library://host:443/entity/collection/container:tag", false, nil, "host:443", "entity/collection/container", []string{"tag"}}, // Test with a different number of tags. {"TagsNone", "library:project", false, nil, "", "project", nil}, {"TagsOne", "library:project:tag1", false, nil, "", "project", []string{"tag1"}}, {"TagsTwo", "library:project:tag1,tag2", false, nil, "", "project", []string{"tag1", "tag2"}}, {"TagsThree", "library:project:tag1,tag2,tag3", false, nil, "", "project", []string{"tag1", "tag2", "tag3"}}, // Test with IP addresses. {"IPv4Host", "library://127.0.0.1/project", false, nil, "127.0.0.1", "project", nil}, {"IPv4HostPort", "library://127.0.0.1:443/project", false, nil, "127.0.0.1:443", "project", nil}, {"IPv6Host", "library://[fe80::1ff:fe23:4567:890a]/project", false, nil, "[fe80::1ff:fe23:4567:890a]", "project", nil}, {"IPv6HostPort", "library://[fe80::1ff:fe23:4567:890a]:443/project", false, nil, "[fe80::1ff:fe23:4567:890a]:443", "project", nil}, {"IPv6HostZone", "library://[fe80::1ff:fe23:4567:890a%25eth1]/project", false, nil, "[fe80::1ff:fe23:4567:890a%eth1]", "project", nil}, {"IPv6HostZonePort", "library://[fe80::1ff:fe23:4567:890a%25eth1]:443/project", false, nil, "[fe80::1ff:fe23:4567:890a%eth1]:443", "project", nil}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { r, err := ParseAmbiguous(tt.rawRef) if (err != nil) != tt.wantErr { t.Fatalf("got err %v, want %v", err, tt.wantErr) } if tt.wantErrSpecific != nil { if got, want := err, tt.wantErrSpecific; !errors.Is(got, want) { t.Fatalf("got err %v, want %v", got, want) } } if err == nil { if got, want := r.Host, tt.wantHost; got != want { t.Errorf("got host %v, want %v", got, want) } if got, want := r.Path, tt.wantPath; got != want { t.Errorf("got path %v, want %v", got, want) } if got, want := r.Tags, tt.wantTags; !reflect.DeepEqual(got, want) { t.Errorf("got tags %v, want %v", got, want) } } }) } } func TestString(t *testing.T) { tests := []struct { name string host string path string tags []string wantString string }{ // Test valid abbreviated paths (project). {"AbbreviatedPath", "", "project", nil, "library:project"}, {"AbbreviatedPathAndTag", "", "project", []string{"tag"}, "library:project:tag"}, {"AbbreviatedPathAndSlash", "", "project", nil, "library:project"}, {"AbbreviatedPathAndSlashAndTag", "", "project", []string{"tag"}, "library:project:tag"}, {"AbbreviatedPathAndHost", "host", "project", nil, "library://host/project"}, {"AbbreviatedPathAndHostAndTag", "host", "project", []string{"tag"}, "library://host/project:tag"}, {"AbbreviatedPathAndHostPort", "host:443", "project", nil, "library://host:443/project"}, {"AbbreviatedPathAndHostPortAndTag", "host:443", "project", []string{"tag"}, "library://host:443/project:tag"}, // Test valid paths (entity/collection/container). {"LegacyPath", "", "entity/collection/container", nil, "library:entity/collection/container"}, {"LegacyPathAndTag", "", "entity/collection/container", []string{"tag"}, "library:entity/collection/container:tag"}, {"LegacyPathAndSlash", "", "entity/collection/container", nil, "library:entity/collection/container"}, {"LegacyPathAndSlashAndTag", "", "entity/collection/container", []string{"tag"}, "library:entity/collection/container:tag"}, {"LegacyPathAndHost", "host", "entity/collection/container", nil, "library://host/entity/collection/container"}, {"LegacyPathAndHostAndTag", "host", "entity/collection/container", []string{"tag"}, "library://host/entity/collection/container:tag"}, {"LegacyPathAndHostPort", "host:443", "entity/collection/container", nil, "library://host:443/entity/collection/container"}, {"LegacyPathAndHostPortAndTag", "host:443", "entity/collection/container", []string{"tag"}, "library://host:443/entity/collection/container:tag"}, // Test with a different number of tags. {"TagsNone", "", "project", nil, "library:project"}, {"TagsOne", "", "project", []string{"tag1"}, "library:project:tag1"}, {"TagsTwo", "", "project", []string{"tag1", "tag2"}, "library:project:tag1,tag2"}, {"TagsThree", "", "project", []string{"tag1", "tag2", "tag3"}, "library:project:tag1,tag2,tag3"}, // Test with IP addresses. {"IPv4Host", "127.0.0.1", "project", nil, "library://127.0.0.1/project"}, {"IPv4HostPort", "127.0.0.1:443", "project", nil, "library://127.0.0.1:443/project"}, {"IPv6Host", "[fe80::1ff:fe23:4567:890a]", "project", nil, "library://[fe80::1ff:fe23:4567:890a]/project"}, {"IPv6HostPort", "[fe80::1ff:fe23:4567:890a]:443", "project", nil, "library://[fe80::1ff:fe23:4567:890a]:443/project"}, {"IPv6HostZone", "[fe80::1ff:fe23:4567:890a%eth1]", "project", nil, "library://[fe80::1ff:fe23:4567:890a%25eth1]/project"}, {"IPv6HostZonePort", "[fe80::1ff:fe23:4567:890a%eth1]:443", "project", nil, "library://[fe80::1ff:fe23:4567:890a%25eth1]:443/project"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { r := Ref{ Host: tt.host, Path: tt.path, Tags: tt.tags, } if got, want := r.String(), tt.wantString; got != want { t.Errorf("got string %v, want %v", got, want) } }) } } func TestHostnamePort(t *testing.T) { tests := []struct { name string rawRef string wantHostname string wantPort string }{ // Test without hostname/port. {"Path", "library:project", "", ""}, {"PathAndSlash", "library:/project", "", ""}, {"PathAndSlashes", "library:///project", "", ""}, // Test with hostnames, with/without port. {"IPv4Host", "library://host/project", "host", ""}, {"IPv4HostPort", "library://host:443/project", "host", "443"}, {"IPv4Host", "library://host.name/project", "host.name", ""}, {"IPv4HostPort", "library://host.name:443/project", "host.name", "443"}, // Test with IP addresses, with/without port. {"IPv4Host", "library://127.0.0.1/project", "127.0.0.1", ""}, {"IPv4HostPort", "library://127.0.0.1:443/project", "127.0.0.1", "443"}, {"IPv6Host", "library://[fe80::1ff:fe23:4567:890a]/project", "fe80::1ff:fe23:4567:890a", ""}, {"IPv6HostPort", "library://[fe80::1ff:fe23:4567:890a]:443/project", "fe80::1ff:fe23:4567:890a", "443"}, {"IPv6HostZone", "library://[fe80::1ff:fe23:4567:890a%25eth1]/project", "fe80::1ff:fe23:4567:890a%eth1", ""}, {"IPv6HostZonePort", "library://[fe80::1ff:fe23:4567:890a%25eth1]:443/project", "fe80::1ff:fe23:4567:890a%eth1", "443"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { r, err := Parse(tt.rawRef) if err != nil { t.Fatalf("failed to parse: %v", err) } if got, want := r.Hostname(), tt.wantHostname; got != want { t.Errorf("got hostname %v, want %v", got, want) } if got, want := r.Port(), tt.wantPort; got != want { t.Errorf("got port %v, want %v", got, want) } }) } } container-library-client-1.4.10/client/request.go000066400000000000000000000031601471454260400220010ustar00rootroot00000000000000// Copyright (c) 2019-2020, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client // UploadImageRequest is sent to initiate V2 image upload type UploadImageRequest struct { Size int64 `json:"filesize"` MD5Checksum string `json:"md5sum,omitempty"` SHA256Checksum string `json:"sha256sum,omitempty"` } // UploadImageCompleteRequest is sent to complete V2 image upload; it is // currently unused. type UploadImageCompleteRequest struct{} // MultipartUploadStartRequest is sent to initiate V2 multipart image upload type MultipartUploadStartRequest struct { Size int64 `json:"filesize"` } // UploadImagePartRequest is sent prior to each part in a multipart upload type UploadImagePartRequest struct { PartSize int64 `json:"partSize"` UploadID string `json:"uploadID"` PartNumber int `json:"partNumber"` SHA256Checksum string `json:"sha256sum"` } // CompletedPart represents a single part of a multipart image upload type CompletedPart struct { PartNumber int `json:"partNumber"` Token string `json:"token"` } // CompleteMultipartUploadRequest is sent to complete V2 multipart image upload type CompleteMultipartUploadRequest struct { UploadID string `json:"uploadID"` CompletedParts []CompletedPart `json:"completedParts"` } // AbortMultipartUploadRequest is sent to abort V2 multipart image upload type AbortMultipartUploadRequest struct { UploadID string `json:"uploadID"` } container-library-client-1.4.10/client/response.go000066400000000000000000000076031471454260400221550ustar00rootroot00000000000000// Copyright (c) 2018-2020, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( jsonresp "github.com/sylabs/json-resp" ) // EntityResponse - Response from the API for an Entity request type EntityResponse struct { Data Entity `json:"data"` Error *jsonresp.Error `json:"error,omitempty"` } // CollectionResponse - Response from the API for an Collection request type CollectionResponse struct { Data Collection `json:"data"` Error *jsonresp.Error `json:"error,omitempty"` } // ContainerResponse - Response from the API for an Container request type ContainerResponse struct { Data Container `json:"data"` Error *jsonresp.Error `json:"error,omitempty"` } // ImageResponse - Response from the API for an Image request type ImageResponse struct { Data Image `json:"data"` Error *jsonresp.Error `json:"error,omitempty"` } // TagsResponse - Response from the API for a tags request type TagsResponse struct { Data TagMap `json:"data"` Error *jsonresp.Error `json:"error,omitempty"` } // ArchTagsResponse - Response from the API for a v2 tags request (with arch) type ArchTagsResponse struct { Data ArchTagMap `json:"data"` Error *jsonresp.Error `json:"error,omitempty"` } // SearchResults - Results structure for searches type SearchResults struct { Entities []Entity `json:"entity"` Collections []Collection `json:"collection"` Containers []Container `json:"container"` Images []Image `json:"image"` } // SearchResponse - Response from the API for a search request type SearchResponse struct { Data SearchResults `json:"data"` Error *jsonresp.Error `json:"error,omitempty"` } // UploadImage - Contains requisite data for direct S3 image upload support type UploadImage struct { UploadURL string `json:"uploadURL"` } // UploadImageResponse - Response from the API for an image upload request type UploadImageResponse struct { Data UploadImage `json:"data"` Error *jsonresp.Error `json:"error,omitempty"` } // QuotaResponse contains quota usage and total available storage type QuotaResponse struct { QuotaTotalBytes int64 `json:"quotaTotal"` QuotaUsageBytes int64 `json:"quotaUsage"` } // UploadImageComplete contains data from upload image completion type UploadImageComplete struct { Quota QuotaResponse `json:"quota"` ContainerURL string `json:"containerUrl"` } // UploadImageCompleteResponse is the response to the upload image completion request type UploadImageCompleteResponse struct { Data UploadImageComplete `json:"data"` Error *jsonresp.Error `json:"error,omitempty"` } // MultipartUpload - Contains data for multipart image upload start request type MultipartUpload struct { UploadID string `json:"uploadID"` TotalParts int `json:"totalParts"` PartSize int64 `json:"partSize"` Options map[string]string `json:"options"` } // MultipartUploadStartResponse - Response from the API for a multipart image upload start request type MultipartUploadStartResponse struct { Data MultipartUpload `json:"data"` Error *jsonresp.Error `json:"error,omitempty"` } // UploadImagePart - Contains data for multipart image upload part request type UploadImagePart struct { PresignedURL string `json:"presignedURL"` } // UploadImagePartResponse - Response from the API for a multipart image upload part request type UploadImagePartResponse struct { Data UploadImagePart `json:"data"` Error *jsonresp.Error `json:"error,omitempty"` } // CompleteMultipartUploadResponse - Response from the API for a multipart image upload complete request type CompleteMultipartUploadResponse struct { Data UploadImageComplete `json:"data"` Error *jsonresp.Error `json:"error,omitempty"` } container-library-client-1.4.10/client/restclient.go000066400000000000000000000065021471454260400224700ustar00rootroot00000000000000// Copyright (c) 2018-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( "bytes" "context" "encoding/json" "fmt" "io" "net/http" "net/url" jsonresp "github.com/sylabs/json-resp" ) func (c *Client) apiGet(ctx context.Context, path string) (objJSON []byte, err error) { c.Logger.Logf("apiGet calling %s", path) return c.doGETRequest(ctx, path) } func (c *Client) apiCreate(ctx context.Context, url string, o interface{}) (objJSON []byte, err error) { c.Logger.Logf("apiCreate calling %s", url) return c.doPOSTRequest(ctx, url, o) } func (c *Client) apiUpdate(ctx context.Context, url string, o interface{}) (objJSON []byte, err error) { c.Logger.Logf("apiUpdate calling %s", url) return c.doPUTRequest(ctx, url, o) } func (c *Client) doGETRequest(ctx context.Context, path string) (objJSON []byte, err error) { return c.commonRequestHandler(ctx, "GET", path, nil, []int{http.StatusOK}) } func (c *Client) doPUTRequest(ctx context.Context, path string, o interface{}) (objJSON []byte, err error) { return c.commonRequestHandler(ctx, "PUT", path, o, []int{http.StatusOK, http.StatusNoContent}) } func (c *Client) doPOSTRequest(ctx context.Context, path string, o interface{}) (objJSON []byte, err error) { return c.commonRequestHandler(ctx, "POST", path, o, []int{http.StatusOK, http.StatusCreated}) } func (c *Client) doDeleteRequest(ctx context.Context, path string) (objJSON []byte, err error) { return c.commonRequestHandler(ctx, "DELETE", path, nil, []int{http.StatusOK}) } func (c *Client) commonRequestHandler(ctx context.Context, method string, path string, o interface{}, acceptedStatusCodes []int) (objJSON []byte, err error) { var payload io.Reader // only PUT and POST methods if method != "GET" && method != "DELETE" { s, err := json.Marshal(o) if err != nil { return []byte{}, fmt.Errorf("error encoding object to JSON:\n\t%w", err) } payload = bytes.NewBuffer(s) } // split url containing query into component pieces (path and raw query) u, err := url.Parse(path) if err != nil { return []byte{}, fmt.Errorf("error parsing url:\n\t%w", err) } req, err := c.newRequest(ctx, method, u.Path, u.RawQuery, payload) if err != nil { return []byte{}, fmt.Errorf("error creating %s request:\n\t%w", method, err) } res, err := c.HTTPClient.Do(req) if err != nil { return []byte{}, fmt.Errorf("error making request to server:\n\t%w", err) } defer res.Body.Close() // check http status code if res.StatusCode == http.StatusNotFound { return []byte{}, ErrNotFound } if !isValidStatusCode(res.StatusCode, acceptedStatusCodes) { err := jsonresp.ReadError(res.Body) if err != nil { return []byte{}, fmt.Errorf("request did not succeed: %w", err) } return []byte{}, fmt.Errorf("%w: request did not succeed: http status code: %d", errHTTP, res.StatusCode) } objJSON, err = io.ReadAll(res.Body) if err != nil { return []byte{}, fmt.Errorf("error reading response from server:\n\t%w", err) } return objJSON, nil } func isValidStatusCode(statusCode int, acceptedStatusCodes []int) bool { for _, value := range acceptedStatusCodes { if value == statusCode { return true } } return false } container-library-client-1.4.10/client/restclient_test.go000066400000000000000000000035011471454260400235230ustar00rootroot00000000000000// Copyright (c) 2018-2019, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( "context" "encoding/json" "reflect" "testing" ) func Test_apiUpdate(t *testing.T) { ctx := context.Background() type endpointResponse struct { Value string `json:"avalue"` } tests := []struct { description string code int url string body interface{} expectError bool }{ {"simple", 200, "v2/imagefile/5cb9c34d7d960d82f5f5bc54/_complete", nil, false}, {"notfound", 404, "v2/nonexistent", nil, true}, {"with_response", 200, "v2/withresponse", endpointResponse{Value: "hello"}, false}, } for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { m := mockService{ t: t, code: tt.code, httpPath: "/" + tt.url, body: tt.body, } m.Run() defer m.Stop() c, err := NewClient(&Config{AuthToken: testToken, BaseURL: m.baseURI}) if err != nil { t.Errorf("Error initializing client: %v", err) } // use payload matching /v2/imagefile/{ref}/_complete endpoint. This // is an arbitrary test of apiUpdate not specific to this endpoint. res, err := c.apiUpdate(ctx, tt.url, UploadImageCompleteRequest{}) if err != nil && !tt.expectError { t.Errorf("Unexpected error: %v", err) } if err == nil && tt.expectError { t.Errorf("Unexpected success. Expected error.") } if tt.body != nil { var r endpointResponse err = json.Unmarshal(res, &r) if err != nil { t.Errorf("error decoding expected response: %v", err) } if !reflect.DeepEqual(r, tt.body) { t.Errorf("unexpected response") } } }) } } container-library-client-1.4.10/client/search.go000066400000000000000000000036361471454260400215660ustar00rootroot00000000000000// Copyright (c) 2018-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( "context" "encoding/json" "fmt" "net/url" ) // Search performs a library search, returning any matching collections, // containers, entities, or images. // // args specifies key-value pairs to be used as a search spec, such as "arch" // (ie. "amd64") or "signed" (valid values "true" or "false"). // // "value" is a required keyword for all searches. It will be matched against // all collections (Entity, Collection, Container, and Image) // // Multiple architectures may be searched by specifying a comma-separated list // (ie. "amd64,arm64") for the value of "arch". // // Match all collections with name "thename": // // c.Search(ctx, map[string]string{"value": "thename"}) // // Match all images with name "imagename" and arch "amd64" // // c.Search(ctx, map[string]string{ // "value": "imagename", // "arch": "amd64" // }) // // Note: if 'arch' and/or 'signed' are specified, the search is limited in // scope only to the "Image" collection. func (c *Client) Search(ctx context.Context, args map[string]string) (*SearchResults, error) { // "value" is minimally required in "args" value, ok := args["value"] if !ok { return nil, errQueryValueMustBeSpecified } if len(value) < 3 { return nil, fmt.Errorf("%w: bad query '%s'. You must search for at least 3 characters", errBadRequest, value) } v := url.Values{} for key, value := range args { v.Set(key, value) } resJSON, err := c.apiGet(ctx, "v1/search?"+v.Encode()) if err != nil { return nil, err } var res SearchResponse if err := json.Unmarshal(resJSON, &res); err != nil { return nil, fmt.Errorf("error decoding results: %w", err) } return &res.Data, nil } container-library-client-1.4.10/client/search_test.go000066400000000000000000000046651471454260400226300ustar00rootroot00000000000000// Copyright (c) 2018, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( "context" "net/http" "reflect" "testing" jsonresp "github.com/sylabs/json-resp" ) func Test_Search(t *testing.T) { tests := []struct { description string code int body interface{} reqCallback func(*http.Request, *testing.T) searchArgs map[string]string expectResults *SearchResults expectError bool }{ { description: "ValidRequest", searchArgs: map[string]string{ "value": "test", }, code: http.StatusOK, body: jsonresp.Response{Data: testSearch}, expectResults: &testSearch, expectError: false, }, { description: "ValidRequestMultiArg", searchArgs: map[string]string{ "value": "test", "arch": "x86_64", "signed": "true", }, code: http.StatusOK, body: jsonresp.Response{Data: testSearch}, expectResults: &testSearch, expectError: false, }, { description: "InternalServerError", searchArgs: map[string]string{"value": "test"}, code: http.StatusInternalServerError, expectError: true, }, { description: "BadRequest", searchArgs: map[string]string{}, code: http.StatusBadRequest, expectError: true, }, { description: "InvalidValue", searchArgs: map[string]string{"value": "aa"}, code: http.StatusBadRequest, expectError: true, }, } // Loop over test cases for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { m := mockService{ t: t, code: tt.code, body: tt.body, reqCallback: tt.reqCallback, httpPath: "/v1/search", } m.Run() defer m.Stop() c, err := NewClient(&Config{AuthToken: testToken, BaseURL: m.baseURI}) if err != nil { t.Errorf("Error initializing client: %v", err) } results, err := c.Search(context.Background(), tt.searchArgs) if err != nil && !tt.expectError { t.Errorf("Unexpected error: %v", err) } if err == nil && tt.expectError { t.Errorf("Unexpected success. Expected error.") } if !reflect.DeepEqual(results, tt.expectResults) { t.Errorf("Got created collection %v - expected %v", results, tt.expectResults) } }) } } container-library-client-1.4.10/client/test_data/000077500000000000000000000000001471454260400217325ustar00rootroot00000000000000container-library-client-1.4.10/client/test_data/test_sha256000066400000000000000000000000411471454260400237170ustar00rootroot00000000000000THIS IS A TEST FOR SHA256 SUMMINGcontainer-library-client-1.4.10/client/util.go000066400000000000000000000113271471454260400212720ustar00rootroot00000000000000// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( "crypto/md5" "crypto/sha256" "encoding/hex" "encoding/json" "io" "os" "regexp" "strings" ) // IsLibraryPullRef returns true if the provided string is a valid library // reference for a pull operation. func IsLibraryPullRef(libraryRef string) bool { match, _ := regexp.MatchString("^(library://)?([a-z0-9]+(?:[._-][a-z0-9]+)*/){0,2}([a-z0-9]+(?:[._-][a-z0-9]+)*)(:[a-z0-9]+(?:[._-][a-z0-9]+)*)?$", libraryRef) return match } // IsLibraryPushRef returns true if the provided string is a valid library // reference for a push operation. func IsLibraryPushRef(libraryRef string) bool { // For push we allow specifying multiple tags, delimited with , match, _ := regexp.MatchString("^(library://)?([a-z0-9]+(?:[._-][a-z0-9]+)*/){2}([a-z0-9]+(?:[._-][a-z0-9]+)*)(:[a-z0-9]+(?:[,._-][a-z0-9]+)*)?$", libraryRef) return match } // IsRefPart returns true if the provided string is valid as a component of a // library URI (i.e. a valid entity, collection etc. name) func IsRefPart(refPart string) bool { match, err := regexp.MatchString("^[a-z0-9]+(?:[._-][a-z0-9]+)*$", refPart) if err != nil { return false } return match } // IsImageHash returns true if the provided string is valid as a unique hash // for an image func IsImageHash(refPart string) bool { // Legacy images will be sent with hash sha256.[a-f0-9]{64} // SIF images will be sent with hash sif.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12} // which is the unique SIF UUID match, err := regexp.MatchString("^((sha256\\.[a-f0-9]{64})|(sif\\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}))$", refPart) if err != nil { return false } return match } func ParseLibraryPath(libraryRef string) (entity string, collection string, container string, tags []string) { libraryRef = strings.TrimPrefix(libraryRef, "library://") refParts := strings.Split(libraryRef, "/") switch len(refParts) { case 3: entity = refParts[0] collection = refParts[1] container = refParts[2] case 2: entity = "" collection = refParts[0] container = refParts[1] case 1: entity = "" collection = "" container = refParts[0] default: // malformed libraryRef; must conform to "library://entity/collection/container[:tag[,tag]...]" return "", "", "", []string{} } if strings.Contains(container, ":") { imageParts := strings.Split(container, ":") container = imageParts[0] tags = []string{imageParts[1]} if strings.Contains(tags[0], ",") { tags = strings.Split(tags[0], ",") } } return entity, collection, container, tags } // IDInSlice returns true if ID is present in the slice func IDInSlice(a string, list []string) bool { for _, b := range list { if b == a { return true } } return false } // SliceWithoutID returns slice with specified ID removed func SliceWithoutID(list []string, a string) []string { var newList []string for _, b := range list { if b != a { newList = append(newList, b) } } return newList } // StringInSlice returns true if string is present in the slice func StringInSlice(a string, list []string) bool { for _, b := range list { if b == a { return true } } return false } // PrettyPrint - Debug helper, print nice json for any interface func PrettyPrint(v interface{}) { if b, err := json.MarshalIndent(v, "", " "); err == nil { println(string(b)) } } // ImageHash returns the appropriate hash for a provided image file // // e.g. sif. or sha256. func ImageHash(filePath string) (result string, err error) { // Currently using sha256 always // TODO - use sif uuid for sif files! file, err := os.Open(filePath) if err != nil { return "", err } defer file.Close() result, _, err = sha256sum(file) return "sha256." + result, err } // sha256sum computes the sha256sum of the specified reader; caller is // responsible for resetting file pointer. 'nBytes' indicates number of // bytes read from reader func sha256sum(r io.Reader) (result string, nBytes int64, err error) { hash := sha256.New() nBytes, err = io.Copy(hash, r) if err != nil { return "", 0, err } return hex.EncodeToString(hash.Sum(nil)), nBytes, nil } // md5sum computes the MD5 checksum of the specified reader; caller is // responsible for resetting file pointer. nBytes' indicates number of // bytes read from reader func md5sum(r io.Reader) (result string, nBytes int64, err error) { hash := md5.New() nBytes, err = io.Copy(hash, r) if err != nil { return "", 0, err } return hex.EncodeToString(hash.Sum(nil)), nBytes, nil } container-library-client-1.4.10/client/util_test.go000066400000000000000000000237151471454260400223350ustar00rootroot00000000000000// Copyright (c) 2018, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( "os" "reflect" "testing" ) func Test_isLibraryPullRef(t *testing.T) { tests := []struct { name string libraryRef string want bool }{ {"Good long ref 1", "library://entity/collection/image:tag", true}, {"Good long ref 2", "entity/collection/image:tag", true}, {"Good long ref 3", "entity/collection/image", true}, {"Good short ref 1", "library://image:tag", true}, {"Good short ref 2", "library://image", true}, {"Good short ref 3", "library://collection/image:tag", true}, {"Good short ref 4", "library://image", true}, {"Good long sha ref 1", "library://entity/collection/image:sha256.e50a30881ace3d5944f5661d222db7bee5296be9e4dc7c1fcb7604bcae926e88", true}, {"Good long sha ref 2", "entity/collection/image:sha256.e50a30881ace3d5944f5661d222db7bee5296be9e4dc7c1fcb7604bcae926e88", true}, {"Good short sha ref 1", "library://image:sha256.e50a30881ace3d5944f5661d222db7bee5296be9e4dc7c1fcb7604bcae926e88", true}, {"Good short sha ref 2", "image:sha256.e50a30881ace3d5944f5661d222db7bee5296be9e4dc7c1fcb7604bcae926e88", true}, {"Good short sha ref 3", "library://collection/image:sha256.e50a30881ace3d5944f5661d222db7bee5296be9e4dc7c1fcb7604bcae926e88", true}, {"Good short sha ref 4", "collection/image:sha256.e50a30881ace3d5944f5661d222db7bee5296be9e4dc7c1fcb7604bcae926e88", true}, {"Too many components", "library://entity/collection/extra/image:tag", false}, {"Bad character", "library://entity/collection/im,age:tag", false}, {"Bad initial character", "library://entity/collection/-image:tag", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := IsLibraryPullRef(tt.libraryRef); got != tt.want { t.Errorf("isLibraryPullRef() = %v, want %v", got, tt.want) } }) } } func Test_isLibraryPushRef(t *testing.T) { tests := []struct { name string libraryRef string want bool }{ {"Good long ref 1", "library://entity/collection/image:tag", true}, {"Good long ref 2", "entity/collection/image:tag", true}, {"Good long ref 3", "entity/collection/image", true}, {"Short ref not allowed 1", "library://image:tag", false}, {"Short ref not allowed 2", "library://image", false}, {"Short ref not allowed 3", "library://collection/image:tag", false}, {"Short ref not allowed 4", "library://image", false}, {"Good long sha ref 1", "library://entity/collection/image:sha256.e50a30881ace3d5944f5661d222db7bee5296be9e4dc7c1fcb7604bcae926e88", true}, {"Good long sha ref 2", "entity/collection/image:sha256.e50a30881ace3d5944f5661d222db7bee5296be9e4dc7c1fcb7604bcae926e88", true}, {"Too many components", "library://entity/collection/extra/image:tag", false}, {"Bad character", "library://entity/collection/im,age:tag", false}, {"Bad initial character", "library://entity/collection/-image:tag", false}, {"No capitals", "library://Entity/collection/image:tag", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := IsLibraryPushRef(tt.libraryRef); got != tt.want { t.Errorf("isLibraryPushRef() = %v, want %v", got, tt.want) } }) } } func Test_IsRefPart(t *testing.T) { tests := []struct { name string libraryRef string want bool }{ {"Good ref 1", "abc123", true}, {"Good ref 2", "abc-123", true}, {"Good ref 3", "abc_123", true}, {"Good ref 4", "abc.123", true}, {"Bad character", "abc,123", false}, {"Bad initial character", "-abc123", false}, {"No capitals", "Abc123", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := IsRefPart(tt.libraryRef); got != tt.want { t.Errorf("IsRefPart() = %v, want %v", got, tt.want) } }) } } func Test_IsImageHash(t *testing.T) { tests := []struct { name string libraryRef string want bool }{ {"Good sha256", "sha256.e50a30881ace3d5944f5661d222db7bee5296be9e4dc7c1fcb7604bcae926e88", true}, {"Good sif", "sif.5574b72c-7705-49cc-874e-424fc3b78116", true}, {"sha256 too long", "sha256.e50a30881ace3d5944f5661d222db7bee5296be9e4dc7c1fcb7604bcae926e88a", false}, {"sha256 too short", "sha256.e50a30881ace3d5944f5661d222db7bee5296be9e4dc7c1fcb7604bcae926e8", false}, {"sha256 bad character", "sha256.g50a30881ace3d5944f5661d222db7bee5296be9e4dc7c1fcb7604bcae926e88", false}, {"sif too long", "sif.5574b72c-7705-49cc-874e-424fc3b78116a", false}, {"sif too short", "sif.5574b72c-7705-49cc-874e-424fc3b7811", false}, {"sif bad character", "sif.g574b72c-7705-49cc-874e-424fc3b78116", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if got := IsImageHash(tt.libraryRef); got != tt.want { t.Errorf("IsImageHash() = %v, want %v", got, tt.want) } }) } } func Test_ParseLibraryPath(t *testing.T) { tests := []struct { name string libraryRef string wantEnt string wantCol string wantCon string wantTags []string }{ {"Good long ref 1", "library://entity/collection/image:tag", "entity", "collection", "image", []string{"tag"}}, {"Good long ref 2", "entity/collection/image:tag", "entity", "collection", "image", []string{"tag"}}, {"Good long ref multi tag", "library://entity/collection/image:tag1,tag2,tag3", "entity", "collection", "image", []string{"tag1", "tag2", "tag3"}}, {"Good short ref 1", "library://image:tag", "", "", "image", []string{"tag"}}, {"Good short ref 2", "image:tag", "", "", "image", []string{"tag"}}, {"Good short ref 3", "library://collection/image:tag", "", "collection", "image", []string{"tag"}}, {"Good short ref 4", "collection/image:tag", "", "collection", "image", []string{"tag"}}, {"Good short ref multi tag", "library://image:tag1,tag2,tag3", "", "", "image", []string{"tag1", "tag2", "tag3"}}, {"Bad ref", "library://entity/collection/extra/image", "", "", "", []string{}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { ent, col, con, tags := ParseLibraryPath(tt.libraryRef) if ent != tt.wantEnt { t.Errorf("ParseLibraryPath() = entity %v, want %v", ent, tt.wantEnt) } if col != tt.wantCol { t.Errorf("ParseLibraryPath() = collection %v, want %v", col, tt.wantCol) } if con != tt.wantCon { t.Errorf("ParseLibraryPath() = container %v, want %v", con, tt.wantCon) } if !reflect.DeepEqual(tags, tt.wantTags) { t.Errorf("ParseLibraryPath() = tags %v, want %v", tags, tt.wantTags) } }) } } func TestIdInSlice(t *testing.T) { trueID := "5cb9c34d7d960d82f5f5bc58" slice := []string{trueID, "5cb9c34d7d960d82f5f5bc59", "5cb9c34d7d960d82f5f5bc5a", "5cb9c34d7d960d82f5f5bc5b"} if !IDInSlice(trueID, slice) { t.Errorf("should find %v in %v", trueID, slice) } slice = []string{"5cb9c34d7d960d82f5f5bc5c", "5cb9c34d7d960d82f5f5bc5d", trueID, "5cb9c34d7d960d82f5f5bc5e"} if !IDInSlice(trueID, slice) { t.Errorf("should find %v in %v", trueID, slice) } slice = []string{"5cb9c34d7d960d82f5f5bc5f", "5cb9c34d7d960d82f5f5bc60", "5cb9c34d7d960d82f5f5bc61", trueID} if !IDInSlice(trueID, slice) { t.Errorf("should find %v in %v", trueID, slice) } falseID := "5cb9c34d7d960d82f5f5bc62" if IDInSlice(falseID, slice) { t.Errorf("should not find %v in %v", trueID, slice) } } func TestSliceWithoutID(t *testing.T) { a := "5cb9c34d7d960d82f5f5bc63" b := "5cb9c34d7d960d82f5f5bc64" c := "5cb9c34d7d960d82f5f5bc65" d := "5cb9c34d7d960d82f5f5bc66" z := "5cb9c34d7d960d82f5f5bc67" slice := []string{a, b, c, d} result := SliceWithoutID(slice, a) if !reflect.DeepEqual([]string{b, c, d}, result) { t.Errorf("error removing a from {a, b, c, d}, got: %v", result) } result = SliceWithoutID(slice, b) if !reflect.DeepEqual([]string{a, c, d}, result) { t.Errorf("error removing b from {a, b, c, d}, got: %v", result) } result = SliceWithoutID(slice, d) if !reflect.DeepEqual([]string{a, b, c}, result) { t.Errorf("error removing c from {a, b, c, d}, got: %v", result) } result = SliceWithoutID(slice, z) if !reflect.DeepEqual([]string{a, b, c, d}, result) { t.Errorf("error removing non-existent z from {a, b, c, d}, got: %v", result) } } func TestStringInSlice(t *testing.T) { trueID := "5cb9c34d7d960d82f5f5bc68" slice := []string{trueID, "5cb9c34d7d960d82f5f5bc69", "5cb9c34d7d960d82f5f5bc6a", "5cb9c34d7d960d82f5f5bc6b"} if !StringInSlice(trueID, slice) { t.Errorf("should find %v in %v", trueID, slice) } slice = []string{"5cb9c34d7d960d82f5f5bc6c", "5cb9c34d7d960d82f5f5bc6d", trueID, "5cb9c34d7d960d82f5f5bc6e"} if !StringInSlice(trueID, slice) { t.Errorf("should find %v in %v", trueID, slice) } slice = []string{"5cb9c34d7d960d82f5f5bc6f", "5cb9c34d7d960d82f5f5bc70", "5cb9c34d7d960d82f5f5bc71", trueID} if !StringInSlice(trueID, slice) { t.Errorf("should find %v in %v", trueID, slice) } falseID := "5cb9c34d7d960d82f5f5bc72" if StringInSlice(falseID, slice) { t.Errorf("should not find %v in %v", trueID, slice) } } func Test_imageHash(t *testing.T) { expectedSha256 := "sha256.d7d356079af905c04e5ae10711ecf3f5b34385e9b143c5d9ddbf740665ce2fb7" _, err := ImageHash("no_such_file.txt") if err == nil { t.Error("Invalid file must return an error") } shasum, err := ImageHash("test_data/test_sha256") if err != nil { t.Errorf("ImageHash on valid file should not raise error: %v", err) } if shasum != expectedSha256 { t.Errorf("ImageHash returned %v - expected %v", shasum, expectedSha256) } } func Test_sha256sum(t *testing.T) { expectedSha256 := "d7d356079af905c04e5ae10711ecf3f5b34385e9b143c5d9ddbf740665ce2fb7" const filename = "test_data/test_sha256" f, err := os.Open(filename) if err != nil { t.Errorf("Unable to open file %s: %v", filename, err) } defer f.Close() shasum, _, err := sha256sum(f) if err != nil { t.Errorf("sha256sum on valid file should not raise error: %v", err) } if shasum != expectedSha256 { t.Errorf("sha256sum returned %v - expected %v", shasum, expectedSha256) } } container-library-client-1.4.10/client/version.go000066400000000000000000000040611471454260400217770ustar00rootroot00000000000000// Copyright (c) 2019, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the LICENSE.md file // distributed with the sources of this project regarding your rights to use or distribute this // software. package client import ( "context" "net/http" "github.com/blang/semver/v4" jsonresp "github.com/sylabs/json-resp" ) const ( // APIVersionV2Upload supports extended image upload functionality. APIVersionV2Upload = "2.0.0-alpha.1" // APIVersionV2ArchTags supports extended arch tags functionality. APIVersionV2ArchTags = "2.0.0-alpha.2" ) // VersionInfo contains version information. type VersionInfo struct { Version string `json:"version"` APIVersion string `json:"apiVersion"` } // GetVersion gets version information from the Cloud-Library Service. The context controls the lifetime of // the request. func (c *Client) GetVersion(ctx context.Context) (vi VersionInfo, err error) { req, err := c.newRequest(ctx, http.MethodGet, "version", "", nil) if err != nil { return VersionInfo{}, err } res, err := c.HTTPClient.Do(req) if err != nil { return VersionInfo{}, err } defer res.Body.Close() if err := jsonresp.ReadResponse(res.Body, &vi); err != nil { return VersionInfo{}, err } return vi, nil } // apiAtLeast returns true if cloud-library server supports requested (or greater) API version func (c *Client) apiAtLeast(ctx context.Context, reqVersion string) bool { // query cloud-library server for supported api version vi, err := c.GetVersion(ctx) if err != nil || vi.APIVersion == "" { // unable to get cloud-library server API version, fallback to lowest // common denominator c.Logger.Logf("Unable to determine remote API version: %v", err) return false } v, err := semver.Make(vi.APIVersion) if err != nil { c.Logger.Logf("Unable to decode remote API version: %v", err) return false } minRequiredVers, err := semver.Make(reqVersion) if err != nil { c.Logger.Logf("Unable to decode minimum required version: %v", err) return false } return v.GTE(minRequiredVers) } container-library-client-1.4.10/client/version_test.go000066400000000000000000000041731471454260400230420ustar00rootroot00000000000000// Copyright (c) 2018, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. package client import ( "context" "testing" jsonresp "github.com/sylabs/json-resp" ) func Test_apiAtLeast(t *testing.T) { type legacyVersionInfo struct { Version string `json:"version"` } ctx := context.Background() tests := []struct { description string body interface{} code int isV2APIUpload bool isV2APIArchTags bool }{ {"legacy", legacyVersionInfo{Version: "1.0.0-alpha.1"}, 200, false, false}, {"malformed", legacyVersionInfo{}, 200, false, false}, {"error", nil, 404, false, false}, {"current", VersionInfo{Version: "1.0.0-alpha.1", APIVersion: "2.0.0-alpha.2"}, 200, true, true}, {"slightly older", VersionInfo{Version: "1.0.0-alpha.1", APIVersion: "2.0.0-alpha.1"}, 200, true, false}, {"newer than that", VersionInfo{Version: "1.0.0-alpha.1", APIVersion: "2.0.5-alpha.1"}, 200, true, true}, {"distant future", VersionInfo{Version: "1.0.0-alpha.1", APIVersion: "3.0.0"}, 200, true, true}, } for _, tt := range tests { t.Run(tt.description, func(t *testing.T) { m := mockService{ t: t, code: 200, body: jsonresp.Response{Data: tt.body}, httpPath: "/version", } m.Run() defer m.Stop() c, err := NewClient(&Config{BaseURL: m.baseURI}) if err != nil { t.Errorf("Error initializing client: %v", err) } result := c.apiAtLeast(ctx, APIVersionV2Upload) if result && !tt.isV2APIUpload { t.Errorf("Unexpected true for API version not supporting V2 Upload.") } if !result && tt.isV2APIUpload { t.Errorf("Unexpected false for API version supporting V2 Upload.") } result = c.apiAtLeast(ctx, APIVersionV2ArchTags) if result && !tt.isV2APIArchTags { t.Errorf("Unexpected true for API version not supporting V2 ArchTags.") } if !result && tt.isV2APIArchTags { t.Errorf("Unexpected false for API version supporting V2 ArchTags.") } }) } } container-library-client-1.4.10/go.mod000066400000000000000000000007221471454260400176130ustar00rootroot00000000000000module github.com/apptainer/container-library-client go 1.22.5 toolchain go1.22.7 require ( github.com/apptainer/sif/v2 v2.19.3 github.com/blang/semver/v4 v4.0.0 github.com/go-log/log v0.2.0 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.0 github.com/sylabs/json-resp v0.9.4 golang.org/x/sync v0.8.0 ) require ( github.com/google/go-containerregistry v0.20.2 // indirect github.com/google/uuid v1.6.0 // indirect ) container-library-client-1.4.10/go.sum000066400000000000000000000043211471454260400176370ustar00rootroot00000000000000github.com/apptainer/sif/v2 v2.19.3 h1:Zu78kyd/Ck1Ax8CTr+QDagg1kbBxDPpzHP87IJSOzBo= github.com/apptainer/sif/v2 v2.19.3/go.mod h1:BZ1cI3nPlPCrBQ0tTViCngeHj35+v30aU9ZBr8oDujI= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/go-log/log v0.2.0 h1:z8i91GBudxD5L3RmF0KVpetCbcGWAV7q1Tw1eRwQM9Q= github.com/go-log/log v0.2.0/go.mod h1:xzCnwajcues/6w7lne3yK2QU7DBPW7kqbgPGG5AF65U= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-containerregistry v0.20.2 h1:B1wPJ1SN/S7pB+ZAimcciVD+r+yV/l/DSArMxlbwseo= github.com/google/go-containerregistry v0.20.2/go.mod h1:z38EKdKh4h7IP2gSfUUqEvalZBqs6AoLeWfUy34nQC8= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sebdah/goldie/v2 v2.5.5 h1:rx1mwF95RxZ3/83sdS4Yp7t2C5TCokvWP4TBRbAyEWY= github.com/sebdah/goldie/v2 v2.5.5/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sylabs/json-resp v0.9.4 h1:gFvnPdfrBUQgTAFKcxW8VOTfFdj/eOwBrwSG76BwiCw= github.com/sylabs/json-resp v0.9.4/go.mod h1:Q9X4wRlZNPv3x76KaL8vTCBO4aC/DP2gh13xdtEqd1g= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=