pax_global_header00006660000000000000000000000064136555040740014523gustar00rootroot0000000000000052 comment=e4d0540ae5ee8c698a2c2c4eaf0f96adb6190a5b swift-1.0.52/000077500000000000000000000000001365550407400127445ustar00rootroot00000000000000swift-1.0.52/.gitignore000066400000000000000000000000301365550407400147250ustar00rootroot00000000000000*~ *.pyc test-env* junk/swift-1.0.52/.travis.yml000066400000000000000000000010031365550407400150470ustar00rootroot00000000000000language: go sudo: false go: - 1.2.x - 1.3.x - 1.4.x - 1.5.x - 1.6.x - 1.7.x - 1.8.x - 1.9.x - 1.10.x - 1.11.x - 1.12.x - 1.13.x - 1.14.x - master matrix: include: - go: 1.14.x env: TEST_REAL_SERVER=rackspace - go: 1.14.x env: TEST_REAL_SERVER=memset allow_failures: - go: 1.14.x env: TEST_REAL_SERVER=rackspace - go: 1.14.x env: TEST_REAL_SERVER=memset install: go test -i ./... script: - test -z "$(go fmt ./...)" - go test - ./travis_realserver.sh swift-1.0.52/COPYING000066400000000000000000000021071365550407400137770ustar00rootroot00000000000000Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. swift-1.0.52/README.md000066400000000000000000000106571365550407400142340ustar00rootroot00000000000000Swift ===== This package provides an easy to use library for interfacing with Swift / Openstack Object Storage / Rackspace cloud files from the Go Language See here for package docs http://godoc.org/github.com/ncw/swift [![Build Status](https://api.travis-ci.org/ncw/swift.svg?branch=master)](https://travis-ci.org/ncw/swift) [![GoDoc](https://godoc.org/github.com/ncw/swift?status.svg)](https://godoc.org/github.com/ncw/swift) Install ------- Use go to install the library go get github.com/ncw/swift Usage ----- See here for full package docs - http://godoc.org/github.com/ncw/swift Here is a short example from the docs ```go import "github.com/ncw/swift" // Create a connection c := swift.Connection{ UserName: "user", ApiKey: "key", AuthUrl: "auth_url", Domain: "domain", // Name of the domain (v3 auth only) Tenant: "tenant", // Name of the tenant (v2 auth only) } // Authenticate err := c.Authenticate() if err != nil { panic(err) } // List all the containers containers, err := c.ContainerNames(nil) fmt.Println(containers) // etc... ``` Additions --------- The `rs` sub project contains a wrapper for the Rackspace specific CDN Management interface. Testing ------- To run the tests you can either use an embedded fake Swift server either use a real Openstack Swift server or a Rackspace Cloud files account. When using a real Swift server, you need to set these environment variables before running the tests export SWIFT_API_USER='user' export SWIFT_API_KEY='key' export SWIFT_AUTH_URL='https://url.of.auth.server/v1.0' And optionally these if using v2 authentication export SWIFT_TENANT='TenantName' export SWIFT_TENANT_ID='TenantId' And optionally these if using v3 authentication export SWIFT_TENANT='TenantName' export SWIFT_TENANT_ID='TenantId' export SWIFT_API_DOMAIN_ID='domain id' export SWIFT_API_DOMAIN='domain name' And optionally these if using v3 trust export SWIFT_TRUST_ID='TrustId' And optionally this if you want to skip server certificate validation export SWIFT_AUTH_INSECURE=1 And optionally this to configure the connect channel timeout, in seconds export SWIFT_CONNECTION_CHANNEL_TIMEOUT=60 And optionally this to configure the data channel timeout, in seconds export SWIFT_DATA_CHANNEL_TIMEOUT=60 Then run the tests with `go test` License ------- This is free software under the terms of MIT license (check COPYING file included in this package). Contact and support ------------------- The project website is at: - https://github.com/ncw/swift There you can file bug reports, ask for help or contribute patches. Authors ------- - Nick Craig-Wood Contributors ------------ - Brian "bojo" Jones - Janika Liiv - Yamamoto, Hirotaka - Stephen - platformpurple - Paul Querna - Livio Soares - thesyncim - lsowen - Sylvain Baubeau - Chris Kastorff - Dai HaoJun - Hua Wang - Fabian Ruff - Arturo Reuschenbach Puncernau - Petr Kotek - Stefan Majewsky - Cezar Sa Espinola - Sam Gunaratne - Richard Scothern - Michel Couillard - Christopher Waldon - dennis - hag - Alexander Neumann - eclipseo <30413512+eclipseo@users.noreply.github.com> - Yuri Per - Falk Reimann - Arthur Paim Arnold - Bruno Michel - Charles Hsu - Omar Ali - Andreas Andersen - kayrus - CodeLingo Bot - Jérémy Clerc - 4xicom <37339705+4xicom@users.noreply.github.com> - Bo - Thiago da Silva - Brandon WELSCH - Damien Tournoud - Pedro Kiefer swift-1.0.52/auth.go000066400000000000000000000216041365550407400142370ustar00rootroot00000000000000package swift import ( "bytes" "encoding/json" "net/http" "net/url" "strings" "time" ) // Auth defines the operations needed to authenticate with swift // // This encapsulates the different authentication schemes in use type Authenticator interface { // Request creates an http.Request for the auth - return nil if not needed Request(*Connection) (*http.Request, error) // Response parses the http.Response Response(resp *http.Response) error // The public storage URL - set Internal to true to read // internal/service net URL StorageUrl(Internal bool) string // The access token Token() string // The CDN url if available CdnUrl() string } // Expireser is an optional interface to read the expiration time of the token type Expireser interface { Expires() time.Time } type CustomEndpointAuthenticator interface { StorageUrlForEndpoint(endpointType EndpointType) string } type EndpointType string const ( // Use public URL as storage URL EndpointTypePublic = EndpointType("public") // Use internal URL as storage URL EndpointTypeInternal = EndpointType("internal") // Use admin URL as storage URL EndpointTypeAdmin = EndpointType("admin") ) // newAuth - create a new Authenticator from the AuthUrl // // A hint for AuthVersion can be provided func newAuth(c *Connection) (Authenticator, error) { AuthVersion := c.AuthVersion if AuthVersion == 0 { if strings.Contains(c.AuthUrl, "v3") { AuthVersion = 3 } else if strings.Contains(c.AuthUrl, "v2") { AuthVersion = 2 } else if strings.Contains(c.AuthUrl, "v1") { AuthVersion = 1 } else { return nil, newErrorf(500, "Can't find AuthVersion in AuthUrl - set explicitly") } } switch AuthVersion { case 1: return &v1Auth{}, nil case 2: return &v2Auth{ // Guess as to whether using API key or // password it will try both eventually so // this is just an optimization. useApiKey: len(c.ApiKey) >= 32, }, nil case 3: return &v3Auth{}, nil } return nil, newErrorf(500, "Auth Version %d not supported", AuthVersion) } // ------------------------------------------------------------ // v1 auth type v1Auth struct { Headers http.Header // V1 auth: the authentication headers so extensions can access them } // v1 Authentication - make request func (auth *v1Auth) Request(c *Connection) (*http.Request, error) { req, err := http.NewRequest("GET", c.AuthUrl, nil) if err != nil { return nil, err } req.Header.Set("User-Agent", c.UserAgent) req.Header.Set("X-Auth-Key", c.ApiKey) req.Header.Set("X-Auth-User", c.UserName) return req, nil } // v1 Authentication - read response func (auth *v1Auth) Response(resp *http.Response) error { auth.Headers = resp.Header return nil } // v1 Authentication - read storage url func (auth *v1Auth) StorageUrl(Internal bool) string { storageUrl := auth.Headers.Get("X-Storage-Url") if Internal { newUrl, err := url.Parse(storageUrl) if err != nil { return storageUrl } newUrl.Host = "snet-" + newUrl.Host storageUrl = newUrl.String() } return storageUrl } // v1 Authentication - read auth token func (auth *v1Auth) Token() string { return auth.Headers.Get("X-Auth-Token") } // v1 Authentication - read cdn url func (auth *v1Auth) CdnUrl() string { return auth.Headers.Get("X-CDN-Management-Url") } // ------------------------------------------------------------ // v2 Authentication type v2Auth struct { Auth *v2AuthResponse Region string useApiKey bool // if set will use API key not Password useApiKeyOk bool // if set won't change useApiKey any more notFirst bool // set after first run } // v2 Authentication - make request func (auth *v2Auth) Request(c *Connection) (*http.Request, error) { auth.Region = c.Region // Toggle useApiKey if not first run and not OK yet if auth.notFirst && !auth.useApiKeyOk { auth.useApiKey = !auth.useApiKey } auth.notFirst = true // Create a V2 auth request for the body of the connection var v2i interface{} if !auth.useApiKey { // Normal swift authentication v2 := v2AuthRequest{} v2.Auth.PasswordCredentials.UserName = c.UserName v2.Auth.PasswordCredentials.Password = c.ApiKey v2.Auth.Tenant = c.Tenant v2.Auth.TenantId = c.TenantId v2i = v2 } else { // Rackspace special with API Key v2 := v2AuthRequestRackspace{} v2.Auth.ApiKeyCredentials.UserName = c.UserName v2.Auth.ApiKeyCredentials.ApiKey = c.ApiKey v2.Auth.Tenant = c.Tenant v2.Auth.TenantId = c.TenantId v2i = v2 } body, err := json.Marshal(v2i) if err != nil { return nil, err } url := c.AuthUrl if !strings.HasSuffix(url, "/") { url += "/" } url += "tokens" req, err := http.NewRequest("POST", url, bytes.NewBuffer(body)) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/json") req.Header.Set("User-Agent", c.UserAgent) return req, nil } // v2 Authentication - read response func (auth *v2Auth) Response(resp *http.Response) error { auth.Auth = new(v2AuthResponse) err := readJson(resp, auth.Auth) // If successfully read Auth then no need to toggle useApiKey any more if err == nil { auth.useApiKeyOk = true } return err } // Finds the Endpoint Url of "type" from the v2AuthResponse using the // Region if set or defaulting to the first one if not // // Returns "" if not found func (auth *v2Auth) endpointUrl(Type string, endpointType EndpointType) string { for _, catalog := range auth.Auth.Access.ServiceCatalog { if catalog.Type == Type { for _, endpoint := range catalog.Endpoints { if auth.Region == "" || (auth.Region == endpoint.Region) { switch endpointType { case EndpointTypeInternal: return endpoint.InternalUrl case EndpointTypePublic: return endpoint.PublicUrl case EndpointTypeAdmin: return endpoint.AdminUrl default: return "" } } } } } return "" } // v2 Authentication - read storage url // // If Internal is true then it reads the private (internal / service // net) URL. func (auth *v2Auth) StorageUrl(Internal bool) string { endpointType := EndpointTypePublic if Internal { endpointType = EndpointTypeInternal } return auth.StorageUrlForEndpoint(endpointType) } // v2 Authentication - read storage url // // Use the indicated endpointType to choose a URL. func (auth *v2Auth) StorageUrlForEndpoint(endpointType EndpointType) string { return auth.endpointUrl("object-store", endpointType) } // v2 Authentication - read auth token func (auth *v2Auth) Token() string { return auth.Auth.Access.Token.Id } // v2 Authentication - read expires func (auth *v2Auth) Expires() time.Time { t, err := time.Parse(time.RFC3339, auth.Auth.Access.Token.Expires) if err != nil { return time.Time{} // return Zero if not parsed } return t } // v2 Authentication - read cdn url func (auth *v2Auth) CdnUrl() string { return auth.endpointUrl("rax:object-cdn", EndpointTypePublic) } // ------------------------------------------------------------ // V2 Authentication request // // http://docs.openstack.org/developer/keystone/api_curl_examples.html // http://docs.rackspace.com/servers/api/v2/cs-gettingstarted/content/curl_auth.html // http://docs.openstack.org/api/openstack-identity-service/2.0/content/POST_authenticate_v2.0_tokens_.html type v2AuthRequest struct { Auth struct { PasswordCredentials struct { UserName string `json:"username"` Password string `json:"password"` } `json:"passwordCredentials"` Tenant string `json:"tenantName,omitempty"` TenantId string `json:"tenantId,omitempty"` } `json:"auth"` } // V2 Authentication request - Rackspace variant // // http://docs.openstack.org/developer/keystone/api_curl_examples.html // http://docs.rackspace.com/servers/api/v2/cs-gettingstarted/content/curl_auth.html // http://docs.openstack.org/api/openstack-identity-service/2.0/content/POST_authenticate_v2.0_tokens_.html type v2AuthRequestRackspace struct { Auth struct { ApiKeyCredentials struct { UserName string `json:"username"` ApiKey string `json:"apiKey"` } `json:"RAX-KSKEY:apiKeyCredentials"` Tenant string `json:"tenantName,omitempty"` TenantId string `json:"tenantId,omitempty"` } `json:"auth"` } // V2 Authentication reply // // http://docs.openstack.org/developer/keystone/api_curl_examples.html // http://docs.rackspace.com/servers/api/v2/cs-gettingstarted/content/curl_auth.html // http://docs.openstack.org/api/openstack-identity-service/2.0/content/POST_authenticate_v2.0_tokens_.html type v2AuthResponse struct { Access struct { ServiceCatalog []struct { Endpoints []struct { InternalUrl string PublicUrl string AdminUrl string Region string TenantId string } Name string Type string } Token struct { Expires string Id string Tenant struct { Id string Name string } } User struct { DefaultRegion string `json:"RAX-AUTH:defaultRegion"` Id string Name string Roles []struct { Description string Id string Name string TenantId string } } } } swift-1.0.52/auth_v3.go000066400000000000000000000157611365550407400146560ustar00rootroot00000000000000package swift import ( "bytes" "encoding/json" "fmt" "net/http" "strings" "time" ) const ( v3AuthMethodToken = "token" v3AuthMethodPassword = "password" v3AuthMethodApplicationCredential = "application_credential" v3CatalogTypeObjectStore = "object-store" ) // V3 Authentication request // http://docs.openstack.org/developer/keystone/api_curl_examples.html // http://developer.openstack.org/api-ref-identity-v3.html type v3AuthRequest struct { Auth struct { Identity struct { Methods []string `json:"methods"` Password *v3AuthPassword `json:"password,omitempty"` Token *v3AuthToken `json:"token,omitempty"` ApplicationCredential *v3AuthApplicationCredential `json:"application_credential,omitempty"` } `json:"identity"` Scope *v3Scope `json:"scope,omitempty"` } `json:"auth"` } type v3Scope struct { Project *v3Project `json:"project,omitempty"` Domain *v3Domain `json:"domain,omitempty"` Trust *v3Trust `json:"OS-TRUST:trust,omitempty"` } type v3Domain struct { Id string `json:"id,omitempty"` Name string `json:"name,omitempty"` } type v3Project struct { Name string `json:"name,omitempty"` Id string `json:"id,omitempty"` Domain *v3Domain `json:"domain,omitempty"` } type v3Trust struct { Id string `json:"id"` } type v3User struct { Domain *v3Domain `json:"domain,omitempty"` Id string `json:"id,omitempty"` Name string `json:"name,omitempty"` Password string `json:"password,omitempty"` } type v3AuthToken struct { Id string `json:"id"` } type v3AuthPassword struct { User v3User `json:"user"` } type v3AuthApplicationCredential struct { Id string `json:"id,omitempty"` Name string `json:"name,omitempty"` Secret string `json:"secret,omitempty"` User *v3User `json:"user,omitempty"` } // V3 Authentication response type v3AuthResponse struct { Token struct { ExpiresAt string `json:"expires_at"` IssuedAt string `json:"issued_at"` Methods []string Roles []struct { Id, Name string Links struct { Self string } } Project struct { Domain struct { Id, Name string } Id, Name string } Catalog []struct { Id, Namem, Type string Endpoints []struct { Id, Region_Id, Url, Region string Interface EndpointType } } User struct { Id, Name string Domain struct { Id, Name string Links struct { Self string } } } Audit_Ids []string } } type v3Auth struct { Region string Auth *v3AuthResponse Headers http.Header } func (auth *v3Auth) Request(c *Connection) (*http.Request, error) { auth.Region = c.Region var v3i interface{} v3 := v3AuthRequest{} if (c.ApplicationCredentialId != "" || c.ApplicationCredentialName != "") && c.ApplicationCredentialSecret != "" { var user *v3User if c.ApplicationCredentialId != "" { c.ApplicationCredentialName = "" user = &v3User{} } if user == nil && c.UserId != "" { // UserID could be used without the domain information user = &v3User{ Id: c.UserId, } } if user == nil && c.UserName == "" { // Make sure that Username or UserID are provided return nil, fmt.Errorf("UserID or Name should be provided") } if user == nil && c.DomainId != "" { user = &v3User{ Name: c.UserName, Domain: &v3Domain{ Id: c.DomainId, }, } } if user == nil && c.Domain != "" { user = &v3User{ Name: c.UserName, Domain: &v3Domain{ Name: c.Domain, }, } } // Make sure that DomainID or DomainName are provided among Username if user == nil { return nil, fmt.Errorf("DomainID or Domain should be provided") } v3.Auth.Identity.Methods = []string{v3AuthMethodApplicationCredential} v3.Auth.Identity.ApplicationCredential = &v3AuthApplicationCredential{ Id: c.ApplicationCredentialId, Name: c.ApplicationCredentialName, Secret: c.ApplicationCredentialSecret, User: user, } } else if c.UserName == "" && c.UserId == "" { v3.Auth.Identity.Methods = []string{v3AuthMethodToken} v3.Auth.Identity.Token = &v3AuthToken{Id: c.ApiKey} } else { v3.Auth.Identity.Methods = []string{v3AuthMethodPassword} v3.Auth.Identity.Password = &v3AuthPassword{ User: v3User{ Name: c.UserName, Id: c.UserId, Password: c.ApiKey, }, } var domain *v3Domain if c.Domain != "" { domain = &v3Domain{Name: c.Domain} } else if c.DomainId != "" { domain = &v3Domain{Id: c.DomainId} } v3.Auth.Identity.Password.User.Domain = domain } if v3.Auth.Identity.Methods[0] != v3AuthMethodApplicationCredential { if c.TrustId != "" { v3.Auth.Scope = &v3Scope{Trust: &v3Trust{Id: c.TrustId}} } else if c.TenantId != "" || c.Tenant != "" { v3.Auth.Scope = &v3Scope{Project: &v3Project{}} if c.TenantId != "" { v3.Auth.Scope.Project.Id = c.TenantId } else if c.Tenant != "" { v3.Auth.Scope.Project.Name = c.Tenant switch { case c.TenantDomain != "": v3.Auth.Scope.Project.Domain = &v3Domain{Name: c.TenantDomain} case c.TenantDomainId != "": v3.Auth.Scope.Project.Domain = &v3Domain{Id: c.TenantDomainId} case c.Domain != "": v3.Auth.Scope.Project.Domain = &v3Domain{Name: c.Domain} case c.DomainId != "": v3.Auth.Scope.Project.Domain = &v3Domain{Id: c.DomainId} default: v3.Auth.Scope.Project.Domain = &v3Domain{Name: "Default"} } } } } v3i = v3 body, err := json.Marshal(v3i) if err != nil { return nil, err } url := c.AuthUrl if !strings.HasSuffix(url, "/") { url += "/" } url += "auth/tokens" req, err := http.NewRequest("POST", url, bytes.NewBuffer(body)) if err != nil { return nil, err } req.Header.Set("Content-Type", "application/json") req.Header.Set("User-Agent", c.UserAgent) return req, nil } func (auth *v3Auth) Response(resp *http.Response) error { auth.Auth = &v3AuthResponse{} auth.Headers = resp.Header err := readJson(resp, auth.Auth) return err } func (auth *v3Auth) endpointUrl(Type string, endpointType EndpointType) string { for _, catalog := range auth.Auth.Token.Catalog { if catalog.Type == Type { for _, endpoint := range catalog.Endpoints { if endpoint.Interface == endpointType && (auth.Region == "" || (auth.Region == endpoint.Region)) { return endpoint.Url } } } } return "" } func (auth *v3Auth) StorageUrl(Internal bool) string { endpointType := EndpointTypePublic if Internal { endpointType = EndpointTypeInternal } return auth.StorageUrlForEndpoint(endpointType) } func (auth *v3Auth) StorageUrlForEndpoint(endpointType EndpointType) string { return auth.endpointUrl("object-store", endpointType) } func (auth *v3Auth) Token() string { return auth.Headers.Get("X-Subject-Token") } func (auth *v3Auth) Expires() time.Time { t, err := time.Parse(time.RFC3339, auth.Auth.Token.ExpiresAt) if err != nil { return time.Time{} // return Zero if not parsed } return t } func (auth *v3Auth) CdnUrl() string { return "" } swift-1.0.52/bin/000077500000000000000000000000001365550407400135145ustar00rootroot00000000000000swift-1.0.52/bin/update-authors.py000077500000000000000000000022001365550407400170300ustar00rootroot00000000000000#!/usr/bin/env python """ Update the README.md file with the authors from the git log """ import re import subprocess AUTHORS = "README.md" IGNORE = [ "nick@raig-wood.com" ] def load(): """ returns a set of emails already in authors """ with open(AUTHORS) as fd: authors = fd.read() emails = set(re.findall(r"<(?!!--)(.*?)>", authors)) emails.update(IGNORE) return emails def add_email(name, email): """ adds the email passed in to the end of authors """ print "Adding %s <%s>" % (name, email) with open(AUTHORS, "a+") as fd: print >>fd, "- %s <%s>" % (name, email) subprocess.check_call(["git", "commit", "-m", "Add %s to contributors" % name, AUTHORS]) def main(): out = subprocess.check_output(["git", "log", '--reverse', '--format=%an|%ae', "master"]) previous = load() for line in out.split("\n"): line = line.strip() if line == "": continue name, email = line.split("|") if email in previous: continue previous.add(email) add_email(name, email) if __name__ == "__main__": main() swift-1.0.52/compatibility_1_0.go000066400000000000000000000014001365550407400165760ustar00rootroot00000000000000// Go 1.0 compatibility functions // +build !go1.1 package swift import ( "log" "net/http" "time" ) // Cancel the request - doesn't work under < go 1.1 func cancelRequest(transport http.RoundTripper, req *http.Request) { log.Printf("Tried to cancel a request but couldn't - recompile with go 1.1") } // Reset a timer - Doesn't work properly < go 1.1 // // This is quite hard to do properly under go < 1.1 so we do a crude // approximation and hope that everyone upgrades to go 1.1 quickly func resetTimer(t *time.Timer, d time.Duration) { t.Stop() // Very likely this doesn't actually work if we are already // selecting on t.C. However we've stopped the original timer // so won't break transfers but may not time them out :-( *t = *time.NewTimer(d) } swift-1.0.52/compatibility_1_1.go000066400000000000000000000006071365550407400166070ustar00rootroot00000000000000// Go 1.1 and later compatibility functions // // +build go1.1 package swift import ( "net/http" "time" ) // Cancel the request func cancelRequest(transport http.RoundTripper, req *http.Request) { if tr, ok := transport.(interface { CancelRequest(*http.Request) }); ok { tr.CancelRequest(req) } } // Reset a timer func resetTimer(t *time.Timer, d time.Duration) { t.Reset(d) } swift-1.0.52/compatibility_1_6.go000066400000000000000000000006421365550407400166130ustar00rootroot00000000000000// +build go1.6 package swift import ( "net/http" "time" ) const IS_AT_LEAST_GO_16 = true func SetExpectContinueTimeout(tr *http.Transport, t time.Duration) { tr.ExpectContinueTimeout = t } func AddExpectAndTransferEncoding(req *http.Request, hasContentLength bool) { if req.Body != nil { req.Header.Add("Expect", "100-continue") } if !hasContentLength { req.TransferEncoding = []string{"chunked"} } } swift-1.0.52/compatibility_not_1_6.go000066400000000000000000000004001365550407400174630ustar00rootroot00000000000000// +build !go1.6 package swift import ( "net/http" "time" ) const IS_AT_LEAST_GO_16 = false func SetExpectContinueTimeout(tr *http.Transport, t time.Duration) {} func AddExpectAndTransferEncoding(req *http.Request, hasContentLength bool) {} swift-1.0.52/dlo.go000066400000000000000000000117121365550407400140530ustar00rootroot00000000000000package swift import ( "os" "strings" ) // DynamicLargeObjectCreateFile represents an open static large object type DynamicLargeObjectCreateFile struct { largeObjectCreateFile } // DynamicLargeObjectCreateFile creates a dynamic large object // returning an object which satisfies io.Writer, io.Seeker, io.Closer // and io.ReaderFrom. The flags are as passes to the // largeObjectCreate method. func (c *Connection) DynamicLargeObjectCreateFile(opts *LargeObjectOpts) (LargeObjectFile, error) { lo, err := c.largeObjectCreate(opts) if err != nil { return nil, err } return withBuffer(opts, &DynamicLargeObjectCreateFile{ largeObjectCreateFile: *lo, }), nil } // DynamicLargeObjectCreate creates or truncates an existing dynamic // large object returning a writeable object. This sets opts.Flags to // an appropriate value before calling DynamicLargeObjectCreateFile func (c *Connection) DynamicLargeObjectCreate(opts *LargeObjectOpts) (LargeObjectFile, error) { opts.Flags = os.O_TRUNC | os.O_CREATE return c.DynamicLargeObjectCreateFile(opts) } // DynamicLargeObjectDelete deletes a dynamic large object and all of its segments. func (c *Connection) DynamicLargeObjectDelete(container string, path string) error { return c.LargeObjectDelete(container, path) } // DynamicLargeObjectMove moves a dynamic large object from srcContainer, srcObjectName to dstContainer, dstObjectName func (c *Connection) DynamicLargeObjectMove(srcContainer string, srcObjectName string, dstContainer string, dstObjectName string) error { info, headers, err := c.Object(srcContainer, srcObjectName) if err != nil { return err } segmentContainer, segmentPath := parseFullPath(headers["X-Object-Manifest"]) if err := c.createDLOManifest(dstContainer, dstObjectName, segmentContainer+"/"+segmentPath, info.ContentType, sanitizeLargeObjectMoveHeaders(headers)); err != nil { return err } if err := c.ObjectDelete(srcContainer, srcObjectName); err != nil { return err } return nil } func sanitizeLargeObjectMoveHeaders(headers Headers) Headers { sanitizedHeaders := make(map[string]string, len(headers)) for k, v := range headers { if strings.HasPrefix(k, "X-") { //Some of the fields does not effect the request e,g, X-Timestamp, X-Trans-Id, X-Openstack-Request-Id. Open stack will generate new ones anyway. sanitizedHeaders[k] = v } } return sanitizedHeaders } // createDLOManifest creates a dynamic large object manifest func (c *Connection) createDLOManifest(container string, objectName string, prefix string, contentType string, headers Headers) error { if headers == nil { headers = make(Headers) } headers["X-Object-Manifest"] = prefix manifest, err := c.ObjectCreate(container, objectName, false, "", contentType, headers) if err != nil { return err } if err := manifest.Close(); err != nil { return err } return nil } // Close satisfies the io.Closer interface func (file *DynamicLargeObjectCreateFile) Close() error { return file.Flush() } func (file *DynamicLargeObjectCreateFile) Flush() error { err := file.conn.createDLOManifest(file.container, file.objectName, file.segmentContainer+"/"+file.prefix, file.contentType, file.headers) if err != nil { return err } return file.conn.waitForSegmentsToShowUp(file.container, file.objectName, file.Size()) } func (c *Connection) getAllDLOSegments(segmentContainer, segmentPath string) ([]Object, error) { //a simple container listing works 99.9% of the time segments, err := c.ObjectsAll(segmentContainer, &ObjectsOpts{Prefix: segmentPath}) if err != nil { return nil, err } hasObjectName := make(map[string]struct{}) for _, segment := range segments { hasObjectName[segment.Name] = struct{}{} } //The container listing might be outdated (i.e. not contain all existing //segment objects yet) because of temporary inconsistency (Swift is only //eventually consistent!). Check its completeness. segmentNumber := 0 for { segmentNumber++ segmentName := getSegment(segmentPath, segmentNumber) if _, seen := hasObjectName[segmentName]; seen { continue } //This segment is missing in the container listing. Use a more reliable //request to check its existence. (HEAD requests on segments are //guaranteed to return the correct metadata, except for the pathological //case of an outage of large parts of the Swift cluster or its network, //since every segment is only written once.) segment, _, err := c.Object(segmentContainer, segmentName) switch err { case nil: //found new segment -> add it in the correct position and keep //going, more might be missing if segmentNumber <= len(segments) { segments = append(segments[:segmentNumber], segments[segmentNumber-1:]...) segments[segmentNumber-1] = segment } else { segments = append(segments, segment) } continue case ObjectNotFound: //This segment is missing. Since we upload segments sequentially, //there won't be any more segments after it. return segments, nil default: return nil, err //unexpected error } } } swift-1.0.52/dlo_test.go000066400000000000000000000136471365550407400151230ustar00rootroot00000000000000package swift import ( "bytes" "github.com/ncw/swift/swifttest" "net/http" "testing" "time" ) var srv *swifttest.SwiftServer var con *Connection var err error var filecontent = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10} var segmentContainer = "segment_container112" func initTest(t *testing.T) { con, err = initTestConnection(t) if err != nil { t.Fail() } } func teardown() { if srv != nil { srv.Close() } } func initTestConnection(t *testing.T) (*Connection, error) { //Uses /swifttest //in-memory implementation to start //a swift object store to test against srv, err = swifttest.NewSwiftServer("localhost") if err != nil { return nil, err } swiftCon := Connection{ UserName: "swifttest", ApiKey: "swifttest", AuthUrl: srv.AuthURL, Region: "", Tenant: "", ConnectTimeout: time.Second, Timeout: time.Second, Transport: new(http.Transport), Domain: "Default", AuthVersion: 1, } err = swiftCon.Authenticate() return &swiftCon, err } func TestCases(t *testing.T) { initTest(t) createContainers([]string{"c1", "c2", segmentContainer}, t) createDynamicObject("c1", "o", t) moveDynamicObject("c1", "o", "c2", "oo", t) deleteDynamicObject("c2", "oo", t) createStaticObject("c2", "o2", t) moveStaticObject("c2", "o2", "c1", "oo2", t) deleteStaticObject("c1", "oo2", t) teardown() } func createContainers(containers []string, t *testing.T) { for i := 0; i < len(containers); i++ { err = con.ContainerCreate(containers[i], nil) // Create container if err != nil { t.Errorf("Fail at create container %s", containers[i]) } } } func createDynamicObject(container, object string, t *testing.T) { metadata := map[string]string{} metadata["Custom-Field"] = "SomeValue" ops := LargeObjectOpts{ Container: container, // Name of container to place object ObjectName: object, // Name of object CheckHash: false, // If set Check the hash ContentType: "application/octet-stream", // Content-Type of the object Headers: Metadata(metadata).ObjectHeaders(), // Additional headers to upload the object with SegmentContainer: segmentContainer, // Name of the container to place segments SegmentPrefix: "sg", // Prefix to use for the segments } bigfile, err := con.DynamicLargeObjectCreate(&ops) if err != nil { t.Errorf("Fail at dynamic create Large Object") } bigfile.Write(filecontent) bigfile.Close() checkObject(container, object, t) } func checkObject(container, object string, t *testing.T) { info, header, err := con.Object(container, object) if err != nil { t.Errorf("Fail at get Large Object metadata: %s", err.Error()) } if !header.IsLargeObject() { t.Errorf("Fail: object is not a large object") } if info.Bytes != 10 { t.Errorf("Fail: mismatch content lengh") } if val, ok := header["X-Object-Meta-Custom-Field"]; !ok || val != "SomeValue" { t.Errorf("Fail: lost custom metadata header") } content, err := con.ObjectGetBytes(container, object) if err != nil { t.Errorf("Fail at read Large Object : %s", err.Error()) } if !bytes.Equal(content, filecontent) { t.Errorf("Fail: mismatch content") } } func checkNotExistObject(container, object string, t *testing.T) { _, _, err = con.Object(container, object) if err == nil || err.Error() != "Object Not Found" { t.Errorf("Fail at checkNotExistObject object: %s", err) } } func moveDynamicObject(sc, so, dc, do string, t *testing.T) { err = con.DynamicLargeObjectMove(sc, so, dc, do) if err != nil { t.Errorf("Fail at dynamic move Large Object: %s", err.Error()) } checkNotExistObject(sc, so, t) checkObject(dc, do, t) } func deleteDynamicObject(container, object string, t *testing.T) { err = con.DynamicLargeObjectDelete(container, object) if err != nil { t.Errorf("Fail at delte dynamic Large Object: %s", err.Error()) } checkNotExistObject(container, object, t) objs, err := con.ObjectsAll(segmentContainer, nil) if err != nil { t.Errorf("Fail at check delte dynamic Large Object: %s", err.Error()) } if len(objs) != 0 { t.Errorf("Fail at check delte dynamic Large Object: seg not deleted") } } func createStaticObject(container, object string, t *testing.T) { metadata := map[string]string{} metadata["Custom-Field"] = "SomeValue" ops := LargeObjectOpts{ Container: container, // Name of container to place object ObjectName: object, // Name of object CheckHash: false, // If set Check the hash ContentType: "application/octet-stream", // Content-Type of the object Headers: Metadata(metadata).ObjectHeaders(), // Additional headers to upload the object with SegmentContainer: segmentContainer, // Name of the container to place segments SegmentPrefix: "sg", // Prefix to use for the segments } bigfile, err := con.StaticLargeObjectCreate(&ops) if err != nil { t.Errorf("Fail at static create Large Object") } bigfile.Write(filecontent) bigfile.Close() checkObject(container, object, t) } func moveStaticObject(sc, so, dc, do string, t *testing.T) { err = con.StaticLargeObjectMove(sc, so, dc, do) if err != nil { t.Errorf("Fail at static move Large Object: %s", err.Error()) } checkNotExistObject(sc, so, t) checkObject(dc, do, t) } func deleteStaticObject(container, object string, t *testing.T) { err = con.StaticLargeObjectDelete(container, object) if err != nil { t.Errorf("Fail at delte dynamic Large Object: %s", err.Error()) } checkNotExistObject(container, object, t) objs, err := con.ObjectsAll(segmentContainer, nil) if err != nil { t.Errorf("Fail at check delte dynamic Large Object: %s", err.Error()) } if len(objs) != 0 { t.Errorf("Fail at check delte dynamic Large Object: seg not deleted") } } swift-1.0.52/doc.go000066400000000000000000000026471365550407400140510ustar00rootroot00000000000000/* Package swift provides an easy to use interface to Swift / Openstack Object Storage / Rackspace Cloud Files Standard Usage Most of the work is done through the Container*() and Object*() methods. All methods are safe to use concurrently in multiple go routines. Object Versioning As defined by http://docs.openstack.org/api/openstack-object-storage/1.0/content/Object_Versioning-e1e3230.html#d6e983 one can create a container which allows for version control of files. The suggested method is to create a version container for holding all non-current files, and a current container for holding the latest version that the file points to. The container and objects inside it can be used in the standard manner, however, pushing a file multiple times will result in it being copied to the version container and the new file put in it's place. If the current file is deleted, the previous file in the version container will replace it. This means that if a file is updated 5 times, it must be deleted 5 times to be completely removed from the system. Rackspace Sub Module This module specifically allows the enabling/disabling of Rackspace Cloud File CDN management on a container. This is specific to the Rackspace API and not Swift/Openstack, therefore it has been placed in a submodule. One can easily create a RsConnection and use it like the standard Connection to access and manipulate containers and objects. */ package swift swift-1.0.52/example_test.go000066400000000000000000000054751365550407400160000ustar00rootroot00000000000000// Copyright... // This example demonstrates opening a Connection and doing some basic operations. package swift_test import ( "fmt" "github.com/ncw/swift" ) func ExampleConnection() { // Create a v1 auth connection c := &swift.Connection{ // This should be your username UserName: "user", // This should be your api key ApiKey: "key", // This should be a v1 auth url, eg // Rackspace US https://auth.api.rackspacecloud.com/v1.0 // Rackspace UK https://lon.auth.api.rackspacecloud.com/v1.0 // Memset Memstore UK https://auth.storage.memset.com/v1.0 AuthUrl: "auth_url", } // Authenticate err := c.Authenticate() if err != nil { panic(err) } // List all the containers containers, err := c.ContainerNames(nil) fmt.Println(containers) // etc... // ------ or alternatively create a v2 connection ------ // Create a v2 auth connection c = &swift.Connection{ // This is the sub user for the storage - eg "admin" UserName: "user", // This should be your api key ApiKey: "key", // This should be a version2 auth url, eg // Rackspace v2 https://identity.api.rackspacecloud.com/v2.0 // Memset Memstore v2 https://auth.storage.memset.com/v2.0 AuthUrl: "v2_auth_url", // Region to use - default is use first region if unset Region: "LON", // Name of the tenant - this is likely your username Tenant: "jim", } // as above... } var container string func ExampleConnection_ObjectsWalk() { c, rollback := makeConnection(nil) defer rollback() objects := make([]string, 0) err := c.ObjectsWalk(container, nil, func(opts *swift.ObjectsOpts) (interface{}, error) { newObjects, err := c.ObjectNames(container, opts) if err == nil { objects = append(objects, newObjects...) } return newObjects, err }) fmt.Println("Found all the objects", objects, err) } func ExampleConnection_VersionContainerCreate() { c, rollback := makeConnection(nil) defer rollback() // Use the helper method to create the current and versions container. if err := c.VersionContainerCreate("cds", "cd-versions"); err != nil { fmt.Print(err.Error()) } } func ExampleConnection_VersionEnable() { c, rollback := makeConnection(nil) defer rollback() // Build the containers manually and enable them. if err := c.ContainerCreate("movie-versions", nil); err != nil { fmt.Print(err.Error()) } if err := c.ContainerCreate("movies", nil); err != nil { fmt.Print(err.Error()) } if err := c.VersionEnable("movies", "movie-versions"); err != nil { fmt.Print(err.Error()) } // Access the primary container as usual with ObjectCreate(), ObjectPut(), etc. // etc... } func ExampleConnection_VersionDisable() { c, rollback := makeConnection(nil) defer rollback() // Disable versioning on a container. Note that this does not delete the versioning container. c.VersionDisable("movies") } swift-1.0.52/go.mod000066400000000000000000000000341365550407400140470ustar00rootroot00000000000000module github.com/ncw/swift swift-1.0.52/largeobjects.go000066400000000000000000000312371365550407400157450ustar00rootroot00000000000000package swift import ( "bufio" "bytes" "crypto/rand" "crypto/sha1" "encoding/hex" "errors" "fmt" "io" "os" gopath "path" "strconv" "strings" "time" ) // NotLargeObject is returned if an operation is performed on an object which isn't large. var NotLargeObject = errors.New("Not a large object") // readAfterWriteTimeout defines the time we wait before an object appears after having been uploaded var readAfterWriteTimeout = 15 * time.Second // readAfterWriteWait defines the time to sleep between two retries var readAfterWriteWait = 200 * time.Millisecond // largeObjectCreateFile represents an open static or dynamic large object type largeObjectCreateFile struct { conn *Connection container string objectName string currentLength int64 filePos int64 chunkSize int64 segmentContainer string prefix string contentType string checkHash bool segments []Object headers Headers minChunkSize int64 } func swiftSegmentPath(path string) (string, error) { checksum := sha1.New() random := make([]byte, 32) if _, err := rand.Read(random); err != nil { return "", err } path = hex.EncodeToString(checksum.Sum(append([]byte(path), random...))) return strings.TrimLeft(strings.TrimRight("segments/"+path[0:3]+"/"+path[3:], "/"), "/"), nil } func getSegment(segmentPath string, partNumber int) string { return fmt.Sprintf("%s/%016d", segmentPath, partNumber) } func parseFullPath(manifest string) (container string, prefix string) { components := strings.SplitN(manifest, "/", 2) container = components[0] if len(components) > 1 { prefix = components[1] } return container, prefix } func (headers Headers) IsLargeObjectDLO() bool { _, isDLO := headers["X-Object-Manifest"] return isDLO } func (headers Headers) IsLargeObjectSLO() bool { _, isSLO := headers["X-Static-Large-Object"] return isSLO } func (headers Headers) IsLargeObject() bool { return headers.IsLargeObjectSLO() || headers.IsLargeObjectDLO() } func (c *Connection) getAllSegments(container string, path string, headers Headers) (string, []Object, error) { if manifest, isDLO := headers["X-Object-Manifest"]; isDLO { segmentContainer, segmentPath := parseFullPath(manifest) segments, err := c.getAllDLOSegments(segmentContainer, segmentPath) return segmentContainer, segments, err } if headers.IsLargeObjectSLO() { return c.getAllSLOSegments(container, path) } return "", nil, NotLargeObject } // LargeObjectOpts describes how a large object should be created type LargeObjectOpts struct { Container string // Name of container to place object ObjectName string // Name of object Flags int // Creation flags CheckHash bool // If set Check the hash Hash string // If set use this hash to check ContentType string // Content-Type of the object Headers Headers // Additional headers to upload the object with ChunkSize int64 // Size of chunks of the object, defaults to 10MB if not set MinChunkSize int64 // Minimum chunk size, automatically set for SLO's based on info SegmentContainer string // Name of the container to place segments SegmentPrefix string // Prefix to use for the segments NoBuffer bool // Prevents using a bufio.Writer to write segments } type LargeObjectFile interface { io.Writer io.Seeker io.Closer Size() int64 Flush() error } // largeObjectCreate creates a large object at opts.Container, opts.ObjectName. // // opts.Flags can have the following bits set // os.TRUNC - remove the contents of the large object if it exists // os.APPEND - write at the end of the large object func (c *Connection) largeObjectCreate(opts *LargeObjectOpts) (*largeObjectCreateFile, error) { var ( segmentPath string segmentContainer string segments []Object currentLength int64 err error ) if opts.SegmentPrefix != "" { segmentPath = opts.SegmentPrefix } else if segmentPath, err = swiftSegmentPath(opts.ObjectName); err != nil { return nil, err } if info, headers, err := c.Object(opts.Container, opts.ObjectName); err == nil { if opts.Flags&os.O_TRUNC != 0 { c.LargeObjectDelete(opts.Container, opts.ObjectName) } else { currentLength = info.Bytes if headers.IsLargeObject() { segmentContainer, segments, err = c.getAllSegments(opts.Container, opts.ObjectName, headers) if err != nil { return nil, err } if len(segments) > 0 { segmentPath = gopath.Dir(segments[0].Name) } } else { if err = c.ObjectMove(opts.Container, opts.ObjectName, opts.Container, getSegment(segmentPath, 1)); err != nil { return nil, err } segments = append(segments, info) } } } else if err != ObjectNotFound { return nil, err } // segmentContainer is not empty when the manifest already existed if segmentContainer == "" { if opts.SegmentContainer != "" { segmentContainer = opts.SegmentContainer } else { segmentContainer = opts.Container + "_segments" } } file := &largeObjectCreateFile{ conn: c, checkHash: opts.CheckHash, container: opts.Container, objectName: opts.ObjectName, chunkSize: opts.ChunkSize, minChunkSize: opts.MinChunkSize, headers: opts.Headers, segmentContainer: segmentContainer, prefix: segmentPath, segments: segments, currentLength: currentLength, } if file.chunkSize == 0 { file.chunkSize = 10 * 1024 * 1024 } if file.minChunkSize > file.chunkSize { file.chunkSize = file.minChunkSize } if opts.Flags&os.O_APPEND != 0 { file.filePos = currentLength } return file, nil } // LargeObjectDelete deletes the large object named by container, path func (c *Connection) LargeObjectDelete(container string, objectName string) error { _, headers, err := c.Object(container, objectName) if err != nil { return err } var objects [][]string if headers.IsLargeObject() { segmentContainer, segments, err := c.getAllSegments(container, objectName, headers) if err != nil { return err } for _, obj := range segments { objects = append(objects, []string{segmentContainer, obj.Name}) } } objects = append(objects, []string{container, objectName}) info, err := c.cachedQueryInfo() if err == nil && info.SupportsBulkDelete() && len(objects) > 0 { filenames := make([]string, len(objects)) for i, obj := range objects { filenames[i] = obj[0] + "/" + obj[1] } _, err = c.doBulkDelete(filenames, nil) // Don't fail on ObjectNotFound because eventual consistency // makes this situation normal. if err != nil && err != Forbidden && err != ObjectNotFound { return err } } else { for _, obj := range objects { if err := c.ObjectDelete(obj[0], obj[1]); err != nil { return err } } } return nil } // LargeObjectGetSegments returns all the segments that compose an object // If the object is a Dynamic Large Object (DLO), it just returns the objects // that have the prefix as indicated by the manifest. // If the object is a Static Large Object (SLO), it retrieves the JSON content // of the manifest and return all the segments of it. func (c *Connection) LargeObjectGetSegments(container string, path string) (string, []Object, error) { _, headers, err := c.Object(container, path) if err != nil { return "", nil, err } return c.getAllSegments(container, path, headers) } // Seek sets the offset for the next write operation func (file *largeObjectCreateFile) Seek(offset int64, whence int) (int64, error) { switch whence { case 0: file.filePos = offset case 1: file.filePos += offset case 2: file.filePos = file.currentLength + offset default: return -1, fmt.Errorf("invalid value for whence") } if file.filePos < 0 { return -1, fmt.Errorf("negative offset") } return file.filePos, nil } func (file *largeObjectCreateFile) Size() int64 { return file.currentLength } func withLORetry(expectedSize int64, fn func() (Headers, int64, error)) (err error) { endTimer := time.NewTimer(readAfterWriteTimeout) defer endTimer.Stop() waitingTime := readAfterWriteWait for { var headers Headers var sz int64 if headers, sz, err = fn(); err == nil { if !headers.IsLargeObjectDLO() || (expectedSize == 0 && sz > 0) || expectedSize == sz { return } } else { return } waitTimer := time.NewTimer(waitingTime) select { case <-endTimer.C: waitTimer.Stop() err = fmt.Errorf("Timeout expired while waiting for object to have size == %d, got: %d", expectedSize, sz) return case <-waitTimer.C: waitingTime *= 2 } } } func (c *Connection) waitForSegmentsToShowUp(container, objectName string, expectedSize int64) (err error) { err = withLORetry(expectedSize, func() (Headers, int64, error) { var info Object var headers Headers info, headers, err = c.objectBase(container, objectName) if err != nil { return headers, 0, err } return headers, info.Bytes, nil }) return } // Write satisfies the io.Writer interface func (file *largeObjectCreateFile) Write(buf []byte) (int, error) { var sz int64 var relativeFilePos int writeSegmentIdx := 0 for i, obj := range file.segments { if file.filePos < sz+obj.Bytes || (i == len(file.segments)-1 && file.filePos < sz+file.minChunkSize) { relativeFilePos = int(file.filePos - sz) break } writeSegmentIdx++ sz += obj.Bytes } sizeToWrite := len(buf) for offset := 0; offset < sizeToWrite; { newSegment, n, err := file.writeSegment(buf[offset:], writeSegmentIdx, relativeFilePos) if err != nil { return 0, err } if writeSegmentIdx < len(file.segments) { file.segments[writeSegmentIdx] = *newSegment } else { file.segments = append(file.segments, *newSegment) } offset += n writeSegmentIdx++ relativeFilePos = 0 } file.filePos += int64(sizeToWrite) file.currentLength = 0 for _, obj := range file.segments { file.currentLength += obj.Bytes } return sizeToWrite, nil } func (file *largeObjectCreateFile) writeSegment(buf []byte, writeSegmentIdx int, relativeFilePos int) (*Object, int, error) { var ( readers []io.Reader existingSegment *Object segmentSize int ) segmentName := getSegment(file.prefix, writeSegmentIdx+1) sizeToRead := int(file.chunkSize) if writeSegmentIdx < len(file.segments) { existingSegment = &file.segments[writeSegmentIdx] if writeSegmentIdx != len(file.segments)-1 { sizeToRead = int(existingSegment.Bytes) } if relativeFilePos > 0 { headers := make(Headers) headers["Range"] = "bytes=0-" + strconv.FormatInt(int64(relativeFilePos-1), 10) existingSegmentReader, _, err := file.conn.ObjectOpen(file.segmentContainer, segmentName, true, headers) if err != nil { return nil, 0, err } defer existingSegmentReader.Close() sizeToRead -= relativeFilePos segmentSize += relativeFilePos readers = []io.Reader{existingSegmentReader} } } if sizeToRead > len(buf) { sizeToRead = len(buf) } segmentSize += sizeToRead readers = append(readers, bytes.NewReader(buf[:sizeToRead])) if existingSegment != nil && segmentSize < int(existingSegment.Bytes) { headers := make(Headers) headers["Range"] = "bytes=" + strconv.FormatInt(int64(segmentSize), 10) + "-" tailSegmentReader, _, err := file.conn.ObjectOpen(file.segmentContainer, segmentName, true, headers) if err != nil { return nil, 0, err } defer tailSegmentReader.Close() segmentSize = int(existingSegment.Bytes) readers = append(readers, tailSegmentReader) } segmentReader := io.MultiReader(readers...) headers, err := file.conn.ObjectPut(file.segmentContainer, segmentName, segmentReader, true, "", file.contentType, nil) if err != nil { return nil, 0, err } return &Object{Name: segmentName, Bytes: int64(segmentSize), Hash: headers["Etag"]}, sizeToRead, nil } func withBuffer(opts *LargeObjectOpts, lo LargeObjectFile) LargeObjectFile { if !opts.NoBuffer { return &bufferedLargeObjectFile{ LargeObjectFile: lo, bw: bufio.NewWriterSize(lo, int(opts.ChunkSize)), } } return lo } type bufferedLargeObjectFile struct { LargeObjectFile bw *bufio.Writer } func (blo *bufferedLargeObjectFile) Close() error { err := blo.bw.Flush() if err != nil { return err } return blo.LargeObjectFile.Close() } func (blo *bufferedLargeObjectFile) Write(p []byte) (n int, err error) { return blo.bw.Write(p) } func (blo *bufferedLargeObjectFile) Seek(offset int64, whence int) (int64, error) { err := blo.bw.Flush() if err != nil { return 0, err } return blo.LargeObjectFile.Seek(offset, whence) } func (blo *bufferedLargeObjectFile) Size() int64 { return blo.LargeObjectFile.Size() + int64(blo.bw.Buffered()) } func (blo *bufferedLargeObjectFile) Flush() error { err := blo.bw.Flush() if err != nil { return err } return blo.LargeObjectFile.Flush() } swift-1.0.52/meta.go000066400000000000000000000124301365550407400142210ustar00rootroot00000000000000// Metadata manipulation in and out of Headers package swift import ( "fmt" "net/http" "strconv" "strings" "time" ) // Metadata stores account, container or object metadata. type Metadata map[string]string // Metadata gets the Metadata starting with the metaPrefix out of the Headers. // // The keys in the Metadata will be converted to lower case func (h Headers) Metadata(metaPrefix string) Metadata { m := Metadata{} metaPrefix = http.CanonicalHeaderKey(metaPrefix) for key, value := range h { if strings.HasPrefix(key, metaPrefix) { metaKey := strings.ToLower(key[len(metaPrefix):]) m[metaKey] = value } } return m } // AccountMetadata converts Headers from account to a Metadata. // // The keys in the Metadata will be converted to lower case. func (h Headers) AccountMetadata() Metadata { return h.Metadata("X-Account-Meta-") } // ContainerMetadata converts Headers from container to a Metadata. // // The keys in the Metadata will be converted to lower case. func (h Headers) ContainerMetadata() Metadata { return h.Metadata("X-Container-Meta-") } // ObjectMetadata converts Headers from object to a Metadata. // // The keys in the Metadata will be converted to lower case. func (h Headers) ObjectMetadata() Metadata { return h.Metadata("X-Object-Meta-") } // Headers convert the Metadata starting with the metaPrefix into a // Headers. // // The keys in the Metadata will be converted from lower case to http // Canonical (see http.CanonicalHeaderKey). func (m Metadata) Headers(metaPrefix string) Headers { h := Headers{} for key, value := range m { key = http.CanonicalHeaderKey(metaPrefix + key) h[key] = value } return h } // AccountHeaders converts the Metadata for the account. func (m Metadata) AccountHeaders() Headers { return m.Headers("X-Account-Meta-") } // ContainerHeaders converts the Metadata for the container. func (m Metadata) ContainerHeaders() Headers { return m.Headers("X-Container-Meta-") } // ObjectHeaders converts the Metadata for the object. func (m Metadata) ObjectHeaders() Headers { return m.Headers("X-Object-Meta-") } // Turns a number of ns into a floating point string in seconds // // Trims trailing zeros and guaranteed to be perfectly accurate func nsToFloatString(ns int64) string { if ns < 0 { return "-" + nsToFloatString(-ns) } result := fmt.Sprintf("%010d", ns) split := len(result) - 9 result, decimals := result[:split], result[split:] decimals = strings.TrimRight(decimals, "0") if decimals != "" { result += "." result += decimals } return result } // Turns a floating point string in seconds into a ns integer // // Guaranteed to be perfectly accurate func floatStringToNs(s string) (int64, error) { const zeros = "000000000" if point := strings.IndexRune(s, '.'); point >= 0 { tail := s[point+1:] if fill := 9 - len(tail); fill < 0 { tail = tail[:9] } else { tail += zeros[:fill] } s = s[:point] + tail } else if len(s) > 0 { // Make sure empty string produces an error s += zeros } return strconv.ParseInt(s, 10, 64) } // FloatStringToTime converts a floating point number string to a time.Time // // The string is floating point number of seconds since the epoch // (Unix time). The number should be in fixed point format (not // exponential), eg "1354040105.123456789" which represents the time // "2012-11-27T18:15:05.123456789Z" // // Some care is taken to preserve all the accuracy in the time.Time // (which wouldn't happen with a naive conversion through float64) so // a round trip conversion won't change the data. // // If an error is returned then time will be returned as the zero time. func FloatStringToTime(s string) (t time.Time, err error) { ns, err := floatStringToNs(s) if err != nil { return } t = time.Unix(0, ns) return } // TimeToFloatString converts a time.Time object to a floating point string // // The string is floating point number of seconds since the epoch // (Unix time). The number is in fixed point format (not // exponential), eg "1354040105.123456789" which represents the time // "2012-11-27T18:15:05.123456789Z". Trailing zeros will be dropped // from the output. // // Some care is taken to preserve all the accuracy in the time.Time // (which wouldn't happen with a naive conversion through float64) so // a round trip conversion won't change the data. func TimeToFloatString(t time.Time) string { return nsToFloatString(t.UnixNano()) } // GetModTime reads a modification time (mtime) from a Metadata object // // This is a defacto standard (used in the official python-swiftclient // amongst others) for storing the modification time (as read using // os.Stat) for an object. It is stored using the key 'mtime', which // for example when written to an object will be 'X-Object-Meta-Mtime'. // // If an error is returned then time will be returned as the zero time. func (m Metadata) GetModTime() (t time.Time, err error) { return FloatStringToTime(m["mtime"]) } // SetModTime writes an modification time (mtime) to a Metadata object // // This is a defacto standard (used in the official python-swiftclient // amongst others) for storing the modification time (as read using // os.Stat) for an object. It is stored using the key 'mtime', which // for example when written to an object will be 'X-Object-Meta-Mtime'. func (m Metadata) SetModTime(t time.Time) { m["mtime"] = TimeToFloatString(t) } swift-1.0.52/meta_test.go000066400000000000000000000111011365550407400152520ustar00rootroot00000000000000// Tests for swift metadata package swift import ( "testing" "time" ) func TestHeadersToMetadata(t *testing.T) { } func TestHeadersToAccountMetadata(t *testing.T) { } func TestHeadersToContainerMetadata(t *testing.T) { } func TestHeadersToObjectMetadata(t *testing.T) { } func TestMetadataToHeaders(t *testing.T) { } func TestMetadataToAccountHeaders(t *testing.T) { } func TestMetadataToContainerHeaders(t *testing.T) { } func TestMetadataToObjectHeaders(t *testing.T) { } func TestNsToFloatString(t *testing.T) { for _, d := range []struct { ns int64 fs string }{ {0, "0"}, {1, "0.000000001"}, {1000, "0.000001"}, {1000000, "0.001"}, {100000000, "0.1"}, {1000000000, "1"}, {10000000000, "10"}, {12345678912, "12.345678912"}, {12345678910, "12.34567891"}, {12345678900, "12.3456789"}, {12345678000, "12.345678"}, {12345670000, "12.34567"}, {12345600000, "12.3456"}, {12345000000, "12.345"}, {12340000000, "12.34"}, {12300000000, "12.3"}, {12000000000, "12"}, {10000000000, "10"}, {1347717491123123123, "1347717491.123123123"}, } { if nsToFloatString(d.ns) != d.fs { t.Error("Failed", d.ns, "!=", d.fs) } if d.ns > 0 && nsToFloatString(-d.ns) != "-"+d.fs { t.Error("Failed on negative", d.ns, "!=", d.fs) } } } func TestFloatStringToNs(t *testing.T) { for _, d := range []struct { ns int64 fs string }{ {0, "0"}, {0, "0."}, {0, ".0"}, {0, "0.0"}, {0, "0.0000000001"}, {1, "0.000000001"}, {1000, "0.000001"}, {1000000, "0.001"}, {100000000, "0.1"}, {100000000, "0.10"}, {100000000, "0.1000000001"}, {1000000000, "1"}, {1000000000, "1."}, {1000000000, "1.0"}, {10000000000, "10"}, {12345678912, "12.345678912"}, {12345678912, "12.3456789129"}, {12345678912, "12.34567891299"}, {12345678910, "12.34567891"}, {12345678900, "12.3456789"}, {12345678000, "12.345678"}, {12345670000, "12.34567"}, {12345600000, "12.3456"}, {12345000000, "12.345"}, {12340000000, "12.34"}, {12300000000, "12.3"}, {12000000000, "12"}, {10000000000, "10"}, // This is a typical value which has more bits in than a float64 {1347717491123123123, "1347717491.123123123"}, } { ns, err := floatStringToNs(d.fs) if err != nil { t.Error("Failed conversion", err) } if ns != d.ns { t.Error("Failed", d.fs, "!=", d.ns, "was", ns) } if d.ns > 0 { ns, err := floatStringToNs("-" + d.fs) if err != nil { t.Error("Failed conversion", err) } if ns != -d.ns { t.Error("Failed on negative", -d.ns, "!=", "-"+d.fs) } } } // These are expected to produce errors for _, fs := range []string{ "", " 1", "- 1", "- 1", "1.-1", "1.0.0", "1x0", } { ns, err := floatStringToNs(fs) if err == nil { t.Error("Didn't produce expected error", fs, ns) } } } func TestGetModTime(t *testing.T) { for _, d := range []struct { ns string t string }{ {"1354040105", "2012-11-27T18:15:05Z"}, {"1354040105.", "2012-11-27T18:15:05Z"}, {"1354040105.0", "2012-11-27T18:15:05Z"}, {"1354040105.000000000000", "2012-11-27T18:15:05Z"}, {"1354040105.123", "2012-11-27T18:15:05.123Z"}, {"1354040105.123456", "2012-11-27T18:15:05.123456Z"}, {"1354040105.123456789", "2012-11-27T18:15:05.123456789Z"}, {"1354040105.123456789123", "2012-11-27T18:15:05.123456789Z"}, {"0", "1970-01-01T00:00:00.000000000Z"}, } { expected, err := time.Parse(time.RFC3339, d.t) if err != nil { t.Error("Bad test", err) } m := Metadata{"mtime": d.ns} actual, err := m.GetModTime() if err != nil { t.Error("Parse error", err) } if !actual.Equal(expected) { t.Error("Expecting", expected, expected.UnixNano(), "got", actual, actual.UnixNano()) } } for _, ns := range []string{ "EMPTY", "", " 1", "- 1", "- 1", "1.-1", "1.0.0", "1x0", } { m := Metadata{} if ns != "EMPTY" { m["mtime"] = ns } actual, err := m.GetModTime() if err == nil { t.Error("Expected error not produced") } if !actual.IsZero() { t.Error("Expected output to be zero") } } } func TestSetModTime(t *testing.T) { for _, d := range []struct { ns string t string }{ {"1354040105", "2012-11-27T18:15:05Z"}, {"1354040105", "2012-11-27T18:15:05.000000Z"}, {"1354040105.123", "2012-11-27T18:15:05.123Z"}, {"1354040105.123456", "2012-11-27T18:15:05.123456Z"}, {"1354040105.123456789", "2012-11-27T18:15:05.123456789Z"}, {"0", "1970-01-01T00:00:00.000000000Z"}, } { time, err := time.Parse(time.RFC3339, d.t) if err != nil { t.Error("Bad test", err) } m := Metadata{} m.SetModTime(time) if m["mtime"] != d.ns { t.Error("mtime wrong", m, "should be", d.ns) } } } swift-1.0.52/notes.txt000066400000000000000000000026701365550407400146420ustar00rootroot00000000000000Notes on Go Swift ================= Make a builder style interface like the Google Go APIs? Advantages are that it is easy to add named methods to the service object to do specific things. Slightly less efficient. Not sure about how to return extra stuff though - in an object? Make a container struct so these could be methods on it? Make noResponse check for 204? Make storage public so it can be extended easily? Rename to go-swift to match user agent string? Reconnect on auth error - 401 when token expires isn't tested Make more api compatible with python cloudfiles? Retry operations on timeout / network errors? - also 408 error - GET requests only? Make Connection thread safe - whenever it is changed take a write lock whenever it is read from a read lock Add extra headers field to Connection (for via etc) Make errors use an error heirachy then can catch them with a type assertion Error(...) ObjectCorrupted{ Error } Make a Debug flag in connection for logging stuff Object If-Match, If-None-Match, If-Modified-Since, If-Unmodified-Since etc Object range Object create, update with X-Delete-At or X-Delete-After Large object support - check uploads are less than 5GB in normal mode? Access control CORS? Swift client retries and backs off for all types of errors Implement net error interface? type Error interface { error Timeout() bool // Is the error a timeout? Temporary() bool // Is the error temporary? } swift-1.0.52/rs/000077500000000000000000000000001365550407400133705ustar00rootroot00000000000000swift-1.0.52/rs/rs.go000066400000000000000000000042461365550407400143510ustar00rootroot00000000000000package rs import ( "errors" "net/http" "strconv" "github.com/ncw/swift" ) // RsConnection is a RackSpace specific wrapper to the core swift library which // exposes the RackSpace CDN commands via the CDN Management URL interface. type RsConnection struct { swift.Connection cdnUrl string } // manage is similar to the swift storage method, but uses the CDN Management URL for CDN specific calls. func (c *RsConnection) manage(p swift.RequestOpts) (resp *http.Response, headers swift.Headers, err error) { p.OnReAuth = func() (string, error) { if c.cdnUrl == "" { c.cdnUrl = c.Auth.CdnUrl() } if c.cdnUrl == "" { return "", errors.New("The X-CDN-Management-Url does not exist on the authenticated platform") } return c.cdnUrl, nil } if c.Authenticated() { _, err = p.OnReAuth() if err != nil { return nil, nil, err } } return c.Connection.Call(c.cdnUrl, p) } // ContainerCDNEnable enables a container for public CDN usage. // // Change the default TTL of 259200 seconds (72 hours) by passing in an integer value. // // This method can be called again to change the TTL. func (c *RsConnection) ContainerCDNEnable(container string, ttl int) (swift.Headers, error) { h := swift.Headers{"X-CDN-Enabled": "true"} if ttl > 0 { h["X-TTL"] = strconv.Itoa(ttl) } _, headers, err := c.manage(swift.RequestOpts{ Container: container, Operation: "PUT", ErrorMap: swift.ContainerErrorMap, NoResponse: true, Headers: h, }) return headers, err } // ContainerCDNDisable disables CDN access to a container. func (c *RsConnection) ContainerCDNDisable(container string) error { h := swift.Headers{"X-CDN-Enabled": "false"} _, _, err := c.manage(swift.RequestOpts{ Container: container, Operation: "PUT", ErrorMap: swift.ContainerErrorMap, NoResponse: true, Headers: h, }) return err } // ContainerCDNMeta returns the CDN metadata for a container. func (c *RsConnection) ContainerCDNMeta(container string) (swift.Headers, error) { _, headers, err := c.manage(swift.RequestOpts{ Container: container, Operation: "HEAD", ErrorMap: swift.ContainerErrorMap, NoResponse: true, Headers: swift.Headers{}, }) return headers, err } swift-1.0.52/rs/rs_test.go000066400000000000000000000037051365550407400154070ustar00rootroot00000000000000// See swift_test.go for requirements to run this test. package rs_test import ( "os" "testing" "github.com/ncw/swift/rs" ) var ( c rs.RsConnection ) const ( CONTAINER = "GoSwiftUnitTest" OBJECT = "test_object" CONTENTS = "12345" CONTENT_SIZE = int64(len(CONTENTS)) CONTENT_MD5 = "827ccb0eea8a706c4c34a16891f84e7b" ) // Test functions are run in order - this one must be first! func TestAuthenticate(t *testing.T) { UserName := os.Getenv("SWIFT_API_USER") ApiKey := os.Getenv("SWIFT_API_KEY") AuthUrl := os.Getenv("SWIFT_AUTH_URL") if UserName == "" || ApiKey == "" || AuthUrl == "" { t.Fatal("SWIFT_API_USER, SWIFT_API_KEY and SWIFT_AUTH_URL not all set") } c = rs.RsConnection{} c.UserName = UserName c.ApiKey = ApiKey c.AuthUrl = AuthUrl err := c.Authenticate() if err != nil { t.Fatal("Auth failed", err) } if !c.Authenticated() { t.Fatal("Not authenticated") } } // Setup func TestContainerCreate(t *testing.T) { err := c.ContainerCreate(CONTAINER, nil) if err != nil { t.Fatal(err) } } func TestCDNEnable(t *testing.T) { headers, err := c.ContainerCDNEnable(CONTAINER, 0) if err != nil { t.Error(err) } if _, ok := headers["X-Cdn-Uri"]; !ok { t.Error("Failed to enable CDN for container") } } func TestOnReAuth(t *testing.T) { c2 := rs.RsConnection{} c2.UserName = c.UserName c2.ApiKey = c.ApiKey c2.AuthUrl = c.AuthUrl _, err := c2.ContainerCDNEnable(CONTAINER, 0) if err != nil { t.Fatalf("Failed to reauthenticate: %v", err) } } func TestCDNMeta(t *testing.T) { headers, err := c.ContainerCDNMeta(CONTAINER) if err != nil { t.Error(err) } if _, ok := headers["X-Cdn-Uri"]; !ok { t.Error("CDN is not enabled") } } func TestCDNDisable(t *testing.T) { err := c.ContainerCDNDisable(CONTAINER) // files stick in CDN until TTL expires if err != nil { t.Error(err) } } // Teardown func TestContainerDelete(t *testing.T) { err := c.ContainerDelete(CONTAINER) if err != nil { t.Fatal(err) } } swift-1.0.52/slo.go000066400000000000000000000123731365550407400140760ustar00rootroot00000000000000package swift import ( "bytes" "encoding/json" "errors" "fmt" "io/ioutil" "net/url" "os" ) // StaticLargeObjectCreateFile represents an open static large object type StaticLargeObjectCreateFile struct { largeObjectCreateFile } var SLONotSupported = errors.New("SLO not supported") type swiftSegment struct { Path string `json:"path,omitempty"` Etag string `json:"etag,omitempty"` Size int64 `json:"size_bytes,omitempty"` // When uploading a manifest, the attributes must be named `path`, `etag` and `size_bytes` // but when querying the JSON content of a manifest with the `multipart-manifest=get` // parameter, Swift names those attributes `name`, `hash` and `bytes`. // We use all the different attributes names in this structure to be able to use // the same structure for both uploading and retrieving. Name string `json:"name,omitempty"` Hash string `json:"hash,omitempty"` Bytes int64 `json:"bytes,omitempty"` ContentType string `json:"content_type,omitempty"` LastModified string `json:"last_modified,omitempty"` } // StaticLargeObjectCreateFile creates a static large object returning // an object which satisfies io.Writer, io.Seeker, io.Closer and // io.ReaderFrom. The flags are as passed to the largeObjectCreate // method. func (c *Connection) StaticLargeObjectCreateFile(opts *LargeObjectOpts) (LargeObjectFile, error) { info, err := c.cachedQueryInfo() if err != nil || !info.SupportsSLO() { return nil, SLONotSupported } realMinChunkSize := info.SLOMinSegmentSize() if realMinChunkSize > opts.MinChunkSize { opts.MinChunkSize = realMinChunkSize } lo, err := c.largeObjectCreate(opts) if err != nil { return nil, err } return withBuffer(opts, &StaticLargeObjectCreateFile{ largeObjectCreateFile: *lo, }), nil } // StaticLargeObjectCreate creates or truncates an existing static // large object returning a writeable object. This sets opts.Flags to // an appropriate value before calling StaticLargeObjectCreateFile func (c *Connection) StaticLargeObjectCreate(opts *LargeObjectOpts) (LargeObjectFile, error) { opts.Flags = os.O_TRUNC | os.O_CREATE return c.StaticLargeObjectCreateFile(opts) } // StaticLargeObjectDelete deletes a static large object and all of its segments. func (c *Connection) StaticLargeObjectDelete(container string, path string) error { info, err := c.cachedQueryInfo() if err != nil || !info.SupportsSLO() { return SLONotSupported } return c.LargeObjectDelete(container, path) } // StaticLargeObjectMove moves a static large object from srcContainer, srcObjectName to dstContainer, dstObjectName func (c *Connection) StaticLargeObjectMove(srcContainer string, srcObjectName string, dstContainer string, dstObjectName string) error { swiftInfo, err := c.cachedQueryInfo() if err != nil || !swiftInfo.SupportsSLO() { return SLONotSupported } info, headers, err := c.Object(srcContainer, srcObjectName) if err != nil { return err } container, segments, err := c.getAllSegments(srcContainer, srcObjectName, headers) if err != nil { return err } //copy only metadata during move (other headers might not be safe for copying) headers = headers.ObjectMetadata().ObjectHeaders() if err := c.createSLOManifest(dstContainer, dstObjectName, info.ContentType, container, segments, headers); err != nil { return err } if err := c.ObjectDelete(srcContainer, srcObjectName); err != nil { return err } return nil } // createSLOManifest creates a static large object manifest func (c *Connection) createSLOManifest(container string, path string, contentType string, segmentContainer string, segments []Object, h Headers) error { sloSegments := make([]swiftSegment, len(segments)) for i, segment := range segments { sloSegments[i].Path = fmt.Sprintf("%s/%s", segmentContainer, segment.Name) sloSegments[i].Etag = segment.Hash sloSegments[i].Size = segment.Bytes } content, err := json.Marshal(sloSegments) if err != nil { return err } values := url.Values{} values.Set("multipart-manifest", "put") if _, err := c.objectPut(container, path, bytes.NewBuffer(content), false, "", contentType, h, values); err != nil { return err } return nil } func (file *StaticLargeObjectCreateFile) Close() error { return file.Flush() } func (file *StaticLargeObjectCreateFile) Flush() error { if err := file.conn.createSLOManifest(file.container, file.objectName, file.contentType, file.segmentContainer, file.segments, file.headers); err != nil { return err } return file.conn.waitForSegmentsToShowUp(file.container, file.objectName, file.Size()) } func (c *Connection) getAllSLOSegments(container, path string) (string, []Object, error) { var ( segmentList []swiftSegment segments []Object segPath string segmentContainer string ) values := url.Values{} values.Set("multipart-manifest", "get") file, _, err := c.objectOpen(container, path, true, nil, values) if err != nil { return "", nil, err } content, err := ioutil.ReadAll(file) if err != nil { return "", nil, err } json.Unmarshal(content, &segmentList) for _, segment := range segmentList { segmentContainer, segPath = parseFullPath(segment.Name[1:]) segments = append(segments, Object{ Name: segPath, Bytes: segment.Bytes, Hash: segment.Hash, }) } return segmentContainer, segments, nil } swift-1.0.52/swift.go000066400000000000000000002157521365550407400144430ustar00rootroot00000000000000package swift import ( "bufio" "bytes" "crypto/hmac" "crypto/md5" "crypto/sha1" "encoding/hex" "encoding/json" "fmt" "hash" "io" "io/ioutil" "mime" "net/http" "net/url" "os" "path" "strconv" "strings" "sync" "time" ) const ( DefaultUserAgent = "goswift/1.0" // Default user agent DefaultRetries = 3 // Default number of retries on token expiry TimeFormat = "2006-01-02T15:04:05" // Python date format for json replies parsed as UTC UploadTar = "tar" // Data format specifier for Connection.BulkUpload(). UploadTarGzip = "tar.gz" // Data format specifier for Connection.BulkUpload(). UploadTarBzip2 = "tar.bz2" // Data format specifier for Connection.BulkUpload(). allContainersLimit = 10000 // Number of containers to fetch at once allObjectsLimit = 10000 // Number objects to fetch at once allObjectsChanLimit = 1000 // ...when fetching to a channel ) // ObjectType is the type of the swift object, regular, static large, // or dynamic large. type ObjectType int // Values that ObjectType can take const ( RegularObjectType ObjectType = iota StaticLargeObjectType DynamicLargeObjectType ) // Connection holds the details of the connection to the swift server. // // You need to provide UserName, ApiKey and AuthUrl when you create a // connection then call Authenticate on it. // // The auth version in use will be detected from the AuthURL - you can // override this with the AuthVersion parameter. // // If using v2 auth you can also set Region in the Connection // structure. If you don't set Region you will get the default region // which may not be what you want. // // For reference some common AuthUrls looks like this: // // Rackspace US https://auth.api.rackspacecloud.com/v1.0 // Rackspace UK https://lon.auth.api.rackspacecloud.com/v1.0 // Rackspace v2 https://identity.api.rackspacecloud.com/v2.0 // Memset Memstore UK https://auth.storage.memset.com/v1.0 // Memstore v2 https://auth.storage.memset.com/v2.0 // // When using Google Appengine you must provide the Connection with an // appengine-specific Transport: // // import ( // "appengine/urlfetch" // "fmt" // "github.com/ncw/swift" // ) // // func handler(w http.ResponseWriter, r *http.Request) { // ctx := appengine.NewContext(r) // tr := urlfetch.Transport{Context: ctx} // c := swift.Connection{ // UserName: "user", // ApiKey: "key", // AuthUrl: "auth_url", // Transport: tr, // } // _ := c.Authenticate() // containers, _ := c.ContainerNames(nil) // fmt.Fprintf(w, "containers: %q", containers) // } // // If you don't supply a Transport, one is made which relies on // http.ProxyFromEnvironment (http://golang.org/pkg/net/http/#ProxyFromEnvironment). // This means that the connection will respect the HTTP proxy specified by the // environment variables $HTTP_PROXY and $NO_PROXY. type Connection struct { // Parameters - fill these in before calling Authenticate // They are all optional except UserName, ApiKey and AuthUrl Domain string // User's domain name DomainId string // User's domain Id UserName string // UserName for api UserId string // User Id ApiKey string // Key for api access ApplicationCredentialId string // Application Credential ID ApplicationCredentialName string // Application Credential Name ApplicationCredentialSecret string // Application Credential Secret AuthUrl string // Auth URL Retries int // Retries on error (default is 3) UserAgent string // Http User agent (default goswift/1.0) ConnectTimeout time.Duration // Connect channel timeout (default 10s) Timeout time.Duration // Data channel timeout (default 60s) Region string // Region to use eg "LON", "ORD" - default is use first region (v2,v3 auth only) AuthVersion int // Set to 1, 2 or 3 or leave at 0 for autodetect Internal bool // Set this to true to use the the internal / service network Tenant string // Name of the tenant (v2,v3 auth only) TenantId string // Id of the tenant (v2,v3 auth only) EndpointType EndpointType // Endpoint type (v2,v3 auth only) (default is public URL unless Internal is set) TenantDomain string // Name of the tenant's domain (v3 auth only), only needed if it differs from the user domain TenantDomainId string // Id of the tenant's domain (v3 auth only), only needed if it differs the from user domain TrustId string // Id of the trust (v3 auth only) Transport http.RoundTripper `json:"-" xml:"-"` // Optional specialised http.Transport (eg. for Google Appengine) // These are filled in after Authenticate is called as are the defaults for above StorageUrl string AuthToken string Expires time.Time // time the token expires, may be Zero if unknown client *http.Client Auth Authenticator `json:"-" xml:"-"` // the current authenticator authLock *sync.Mutex // lock when R/W StorageUrl, AuthToken, Auth // swiftInfo is filled after QueryInfo is called swiftInfo SwiftInfo } // setFromEnv reads the value that param points to (it must be a // pointer), if it isn't the zero value then it reads the environment // variable name passed in, parses it according to the type and writes // it to the pointer. func setFromEnv(param interface{}, name string) (err error) { val := os.Getenv(name) if val == "" { return } switch result := param.(type) { case *string: if *result == "" { *result = val } case *int: if *result == 0 { *result, err = strconv.Atoi(val) } case *bool: if *result == false { *result, err = strconv.ParseBool(val) } case *time.Duration: if *result == 0 { *result, err = time.ParseDuration(val) } case *EndpointType: if *result == EndpointType("") { *result = EndpointType(val) } default: return newErrorf(0, "can't set var of type %T", param) } return err } // ApplyEnvironment reads environment variables and applies them to // the Connection structure. It won't overwrite any parameters which // are already set in the Connection struct. // // To make a new Connection object entirely from the environment you // would do: // // c := new(Connection) // err := c.ApplyEnvironment() // if err != nil { log.Fatal(err) } // // The naming of these variables follows the official Openstack naming // scheme so it should be compatible with OpenStack rc files. // // For v1 authentication (obsolete) // ST_AUTH - Auth URL // ST_USER - UserName for api // ST_KEY - Key for api access // // For v2 authentication // OS_AUTH_URL - Auth URL // OS_USERNAME - UserName for api // OS_PASSWORD - Key for api access // OS_TENANT_NAME - Name of the tenant // OS_TENANT_ID - Id of the tenant // OS_REGION_NAME - Region to use - default is use first region // // For v3 authentication // OS_AUTH_URL - Auth URL // OS_USERNAME - UserName for api // OS_USER_ID - User Id // OS_PASSWORD - Key for api access // OS_APPLICATION_CREDENTIAL_ID - Application Credential ID // OS_APPLICATION_CREDENTIAL_NAME - Application Credential Name // OS_APPLICATION_CREDENTIAL_SECRET - Application Credential Secret // OS_USER_DOMAIN_NAME - User's domain name // OS_USER_DOMAIN_ID - User's domain Id // OS_PROJECT_NAME - Name of the project // OS_PROJECT_DOMAIN_NAME - Name of the tenant's domain, only needed if it differs from the user domain // OS_PROJECT_DOMAIN_ID - Id of the tenant's domain, only needed if it differs the from user domain // OS_TRUST_ID - If of the trust // OS_REGION_NAME - Region to use - default is use first region // // Other // OS_ENDPOINT_TYPE - Endpoint type public, internal or admin // ST_AUTH_VERSION - Choose auth version - 1, 2 or 3 or leave at 0 for autodetect // // For manual authentication // OS_STORAGE_URL - storage URL from alternate authentication // OS_AUTH_TOKEN - Auth Token from alternate authentication // // Library specific // GOSWIFT_RETRIES - Retries on error (default is 3) // GOSWIFT_USER_AGENT - HTTP User agent (default goswift/1.0) // GOSWIFT_CONNECT_TIMEOUT - Connect channel timeout with unit, eg "10s", "100ms" (default "10s") // GOSWIFT_TIMEOUT - Data channel timeout with unit, eg "10s", "100ms" (default "60s") // GOSWIFT_INTERNAL - Set this to "true" to use the the internal network (obsolete - use OS_ENDPOINT_TYPE) func (c *Connection) ApplyEnvironment() (err error) { for _, item := range []struct { result interface{} name string }{ // Environment variables - keep in same order as Connection {&c.Domain, "OS_USER_DOMAIN_NAME"}, {&c.DomainId, "OS_USER_DOMAIN_ID"}, {&c.UserName, "OS_USERNAME"}, {&c.UserId, "OS_USER_ID"}, {&c.ApiKey, "OS_PASSWORD"}, {&c.ApplicationCredentialId, "OS_APPLICATION_CREDENTIAL_ID"}, {&c.ApplicationCredentialName, "OS_APPLICATION_CREDENTIAL_NAME"}, {&c.ApplicationCredentialSecret, "OS_APPLICATION_CREDENTIAL_SECRET"}, {&c.AuthUrl, "OS_AUTH_URL"}, {&c.Retries, "GOSWIFT_RETRIES"}, {&c.UserAgent, "GOSWIFT_USER_AGENT"}, {&c.ConnectTimeout, "GOSWIFT_CONNECT_TIMEOUT"}, {&c.Timeout, "GOSWIFT_TIMEOUT"}, {&c.Region, "OS_REGION_NAME"}, {&c.AuthVersion, "ST_AUTH_VERSION"}, {&c.Internal, "GOSWIFT_INTERNAL"}, {&c.Tenant, "OS_TENANT_NAME"}, //v2 {&c.Tenant, "OS_PROJECT_NAME"}, // v3 {&c.TenantId, "OS_TENANT_ID"}, {&c.EndpointType, "OS_ENDPOINT_TYPE"}, {&c.TenantDomain, "OS_PROJECT_DOMAIN_NAME"}, {&c.TenantDomainId, "OS_PROJECT_DOMAIN_ID"}, {&c.TrustId, "OS_TRUST_ID"}, {&c.StorageUrl, "OS_STORAGE_URL"}, {&c.AuthToken, "OS_AUTH_TOKEN"}, // v1 auth alternatives {&c.ApiKey, "ST_KEY"}, {&c.UserName, "ST_USER"}, {&c.AuthUrl, "ST_AUTH"}, } { err = setFromEnv(item.result, item.name) if err != nil { return newErrorf(0, "failed to read env var %q: %v", item.name, err) } } return nil } // Error - all errors generated by this package are of this type. Other error // may be passed on from library functions though. type Error struct { StatusCode int // HTTP status code if relevant or 0 if not Text string } // Error satisfy the error interface. func (e *Error) Error() string { return e.Text } // newError make a new error from a string. func newError(StatusCode int, Text string) *Error { return &Error{ StatusCode: StatusCode, Text: Text, } } // newErrorf makes a new error from sprintf parameters. func newErrorf(StatusCode int, Text string, Parameters ...interface{}) *Error { return newError(StatusCode, fmt.Sprintf(Text, Parameters...)) } // errorMap defines http error codes to error mappings. type errorMap map[int]error var ( // Specific Errors you might want to check for equality NotModified = newError(304, "Not Modified") BadRequest = newError(400, "Bad Request") AuthorizationFailed = newError(401, "Authorization Failed") ContainerNotFound = newError(404, "Container Not Found") ContainerNotEmpty = newError(409, "Container Not Empty") ObjectNotFound = newError(404, "Object Not Found") ObjectCorrupted = newError(422, "Object Corrupted") TimeoutError = newError(408, "Timeout when reading or writing data") Forbidden = newError(403, "Operation forbidden") TooLargeObject = newError(413, "Too Large Object") RateLimit = newError(498, "Rate Limit") TooManyRequests = newError(429, "TooManyRequests") // Mappings for authentication errors authErrorMap = errorMap{ 400: BadRequest, 401: AuthorizationFailed, 403: Forbidden, } // Mappings for container errors ContainerErrorMap = errorMap{ 400: BadRequest, 403: Forbidden, 404: ContainerNotFound, 409: ContainerNotEmpty, 498: RateLimit, } // Mappings for object errors objectErrorMap = errorMap{ 304: NotModified, 400: BadRequest, 403: Forbidden, 404: ObjectNotFound, 413: TooLargeObject, 422: ObjectCorrupted, 429: TooManyRequests, 498: RateLimit, } ) // checkClose is used to check the return from Close in a defer // statement. func checkClose(c io.Closer, err *error) { cerr := c.Close() if *err == nil { *err = cerr } } // drainAndClose discards all data from rd and closes it. // If an error occurs during Read, it is discarded. func drainAndClose(rd io.ReadCloser, err *error) { if rd == nil { return } _, _ = io.Copy(ioutil.Discard, rd) cerr := rd.Close() if err != nil && *err == nil { *err = cerr } } // parseHeaders checks a response for errors and translates into // standard errors if necessary. If an error is returned, resp.Body // has been drained and closed. func (c *Connection) parseHeaders(resp *http.Response, errorMap errorMap) error { if errorMap != nil { if err, ok := errorMap[resp.StatusCode]; ok { drainAndClose(resp.Body, nil) return err } } if resp.StatusCode < 200 || resp.StatusCode > 299 { drainAndClose(resp.Body, nil) return newErrorf(resp.StatusCode, "HTTP Error: %d: %s", resp.StatusCode, resp.Status) } return nil } // readHeaders returns a Headers object from the http.Response. // // If it receives multiple values for a key (which should never // happen) it will use the first one func readHeaders(resp *http.Response) Headers { headers := Headers{} for key, values := range resp.Header { headers[key] = values[0] } return headers } // Headers stores HTTP headers (can only have one of each header like Swift). type Headers map[string]string // Does an http request using the running timer passed in func (c *Connection) doTimeoutRequest(timer *time.Timer, req *http.Request) (*http.Response, error) { // Do the request in the background so we can check the timeout type result struct { resp *http.Response err error } done := make(chan result, 1) go func() { resp, err := c.client.Do(req) done <- result{resp, err} }() // Wait for the read or the timeout select { case r := <-done: return r.resp, r.err case <-timer.C: // Kill the connection on timeout so we don't leak sockets or goroutines cancelRequest(c.Transport, req) return nil, TimeoutError } panic("unreachable") // For Go 1.0 } // Set defaults for any unset values // // Call with authLock held func (c *Connection) setDefaults() { if c.UserAgent == "" { c.UserAgent = DefaultUserAgent } if c.Retries == 0 { c.Retries = DefaultRetries } if c.ConnectTimeout == 0 { c.ConnectTimeout = 10 * time.Second } if c.Timeout == 0 { c.Timeout = 60 * time.Second } if c.Transport == nil { t := &http.Transport{ // TLSClientConfig: &tls.Config{RootCAs: pool}, // DisableCompression: true, Proxy: http.ProxyFromEnvironment, // Half of linux's default open files limit (1024). MaxIdleConnsPerHost: 512, } SetExpectContinueTimeout(t, 5*time.Second) c.Transport = t } if c.client == nil { c.client = &http.Client{ // CheckRedirect: redirectPolicyFunc, Transport: c.Transport, } } } // Authenticate connects to the Swift server. // // If you don't call it before calling one of the connection methods // then it will be called for you on the first access. func (c *Connection) Authenticate() (err error) { if c.authLock == nil { c.authLock = &sync.Mutex{} } c.authLock.Lock() defer c.authLock.Unlock() return c.authenticate() } // Internal implementation of Authenticate // // Call with authLock held func (c *Connection) authenticate() (err error) { c.setDefaults() // Flush the keepalives connection - if we are // re-authenticating then stuff has gone wrong flushKeepaliveConnections(c.Transport) if c.Auth == nil { c.Auth, err = newAuth(c) if err != nil { return } } retries := 1 again: var req *http.Request req, err = c.Auth.Request(c) if err != nil { return } if req != nil { timer := time.NewTimer(c.ConnectTimeout) defer timer.Stop() var resp *http.Response resp, err = c.doTimeoutRequest(timer, req) if err != nil { return } defer func() { drainAndClose(resp.Body, &err) // Flush the auth connection - we don't want to keep // it open if keepalives were enabled flushKeepaliveConnections(c.Transport) }() if err = c.parseHeaders(resp, authErrorMap); err != nil { // Try again for a limited number of times on // AuthorizationFailed or BadRequest. This allows us // to try some alternate forms of the request if (err == AuthorizationFailed || err == BadRequest) && retries > 0 { retries-- goto again } return } err = c.Auth.Response(resp) if err != nil { return } } if customAuth, isCustom := c.Auth.(CustomEndpointAuthenticator); isCustom && c.EndpointType != "" { c.StorageUrl = customAuth.StorageUrlForEndpoint(c.EndpointType) } else { c.StorageUrl = c.Auth.StorageUrl(c.Internal) } c.AuthToken = c.Auth.Token() if do, ok := c.Auth.(Expireser); ok { c.Expires = do.Expires() } else { c.Expires = time.Time{} } if !c.authenticated() { err = newError(0, "Response didn't have storage url and auth token") return } return } // Get an authToken and url // // The Url may be updated if it needed to authenticate using the OnReAuth function func (c *Connection) getUrlAndAuthToken(targetUrlIn string, OnReAuth func() (string, error)) (targetUrlOut, authToken string, err error) { c.authLock.Lock() defer c.authLock.Unlock() targetUrlOut = targetUrlIn if !c.authenticated() { err = c.authenticate() if err != nil { return } if OnReAuth != nil { targetUrlOut, err = OnReAuth() if err != nil { return } } } authToken = c.AuthToken return } // flushKeepaliveConnections is called to flush pending requests after an error. func flushKeepaliveConnections(transport http.RoundTripper) { if tr, ok := transport.(interface { CloseIdleConnections() }); ok { tr.CloseIdleConnections() } } // UnAuthenticate removes the authentication from the Connection. func (c *Connection) UnAuthenticate() { c.authLock.Lock() c.StorageUrl = "" c.AuthToken = "" c.authLock.Unlock() } // Authenticated returns a boolean to show if the current connection // is authenticated. // // Doesn't actually check the credentials against the server. func (c *Connection) Authenticated() bool { if c.authLock == nil { c.authLock = &sync.Mutex{} } c.authLock.Lock() defer c.authLock.Unlock() return c.authenticated() } // Internal version of Authenticated() // // Call with authLock held func (c *Connection) authenticated() bool { if c.StorageUrl == "" || c.AuthToken == "" { return false } if c.Expires.IsZero() { return true } timeUntilExpiry := c.Expires.Sub(time.Now()) return timeUntilExpiry >= 60*time.Second } // SwiftInfo contains the JSON object returned by Swift when the /info // route is queried. The object contains, among others, the Swift version, // the enabled middlewares and their configuration type SwiftInfo map[string]interface{} func (i SwiftInfo) SupportsBulkDelete() bool { _, val := i["bulk_delete"] return val } func (i SwiftInfo) SupportsSLO() bool { _, val := i["slo"] return val } func (i SwiftInfo) SLOMinSegmentSize() int64 { if slo, ok := i["slo"].(map[string]interface{}); ok { val, _ := slo["min_segment_size"].(float64) return int64(val) } return 1 } // Discover Swift configuration by doing a request against /info func (c *Connection) QueryInfo() (infos SwiftInfo, err error) { infoUrl, err := url.Parse(c.StorageUrl) if err != nil { return nil, err } infoUrl.Path = path.Join(infoUrl.Path, "..", "..", "info") resp, err := c.client.Get(infoUrl.String()) if err == nil { if resp.StatusCode != http.StatusOK { drainAndClose(resp.Body, nil) return nil, fmt.Errorf("Invalid status code for info request: %d", resp.StatusCode) } err = readJson(resp, &infos) if err == nil { c.authLock.Lock() c.swiftInfo = infos c.authLock.Unlock() } return infos, err } return nil, err } func (c *Connection) cachedQueryInfo() (infos SwiftInfo, err error) { c.authLock.Lock() infos = c.swiftInfo c.authLock.Unlock() if infos == nil { infos, err = c.QueryInfo() if err != nil { return } } return infos, nil } // RequestOpts contains parameters for Connection.storage. type RequestOpts struct { Container string ObjectName string Operation string Parameters url.Values Headers Headers ErrorMap errorMap NoResponse bool Body io.Reader Retries int // if set this is called on re-authentication to refresh the targetUrl OnReAuth func() (string, error) } // Call runs a remote command on the targetUrl, returns a // response, headers and possible error. // // operation is GET, HEAD etc // container is the name of a container // Any other parameters (if not None) are added to the targetUrl // // Returns a response or an error. If response is returned then // the resp.Body must be read completely and // resp.Body.Close() must be called on it, unless noResponse is set in // which case the body will be closed in this function // // If "Content-Length" is set in p.Headers it will be used - this can // be used to override the default chunked transfer encoding for // uploads. // // This will Authenticate if necessary, and re-authenticate if it // receives a 401 error which means the token has expired // // This method is exported so extensions can call it. func (c *Connection) Call(targetUrl string, p RequestOpts) (resp *http.Response, headers Headers, err error) { c.authLock.Lock() c.setDefaults() c.authLock.Unlock() retries := p.Retries if retries == 0 { retries = c.Retries } var req *http.Request for { var authToken string if targetUrl, authToken, err = c.getUrlAndAuthToken(targetUrl, p.OnReAuth); err != nil { return //authentication failure } var URL *url.URL URL, err = url.Parse(targetUrl) if err != nil { return } if p.Container != "" { URL.Path += "/" + p.Container if p.ObjectName != "" { URL.Path += "/" + p.ObjectName } } if p.Parameters != nil { URL.RawQuery = p.Parameters.Encode() } timer := time.NewTimer(c.ConnectTimeout) defer timer.Stop() reader := p.Body if reader != nil { reader = newWatchdogReader(reader, c.Timeout, timer) } req, err = http.NewRequest(p.Operation, URL.String(), reader) if err != nil { return } if p.Headers != nil { for k, v := range p.Headers { // Set ContentLength in req if the user passed it in in the headers if k == "Content-Length" { req.ContentLength, err = strconv.ParseInt(v, 10, 64) if err != nil { err = fmt.Errorf("Invalid %q header %q: %v", k, v, err) return } } else { req.Header.Add(k, v) } } } req.Header.Add("User-Agent", c.UserAgent) req.Header.Add("X-Auth-Token", authToken) _, hasCL := p.Headers["Content-Length"] AddExpectAndTransferEncoding(req, hasCL) resp, err = c.doTimeoutRequest(timer, req) if err != nil { if (p.Operation == "HEAD" || p.Operation == "GET") && retries > 0 { retries-- continue } return } // Check to see if token has expired if resp.StatusCode == 401 && retries > 0 { drainAndClose(resp.Body, nil) c.UnAuthenticate() retries-- } else { break } } headers = readHeaders(resp) if err = c.parseHeaders(resp, p.ErrorMap); err != nil { return } if p.NoResponse { drainAndClose(resp.Body, &err) if err != nil { return } } else { // Cancel the request on timeout cancel := func() { cancelRequest(c.Transport, req) } // Wrap resp.Body to make it obey an idle timeout resp.Body = newTimeoutReader(resp.Body, c.Timeout, cancel) } return } // storage runs a remote command on a the storage url, returns a // response, headers and possible error. // // operation is GET, HEAD etc // container is the name of a container // Any other parameters (if not None) are added to the storage url // // Returns a response or an error. If response is returned then // resp.Body.Close() must be called on it, unless noResponse is set in // which case the body will be closed in this function // // This will Authenticate if necessary, and re-authenticate if it // receives a 401 error which means the token has expired func (c *Connection) storage(p RequestOpts) (resp *http.Response, headers Headers, err error) { p.OnReAuth = func() (string, error) { return c.StorageUrl, nil } c.authLock.Lock() url := c.StorageUrl c.authLock.Unlock() return c.Call(url, p) } // readLines reads the response into an array of strings. // // Closes the response when done func readLines(resp *http.Response) (lines []string, err error) { defer drainAndClose(resp.Body, &err) reader := bufio.NewReader(resp.Body) buffer := bytes.NewBuffer(make([]byte, 0, 128)) var part []byte var prefix bool for { if part, prefix, err = reader.ReadLine(); err != nil { break } buffer.Write(part) if !prefix { lines = append(lines, buffer.String()) buffer.Reset() } } if err == io.EOF { err = nil } return } // readJson reads the response into the json type passed in // // Closes the response when done func readJson(resp *http.Response, result interface{}) (err error) { defer drainAndClose(resp.Body, &err) decoder := json.NewDecoder(resp.Body) return decoder.Decode(result) } /* ------------------------------------------------------------ */ // ContainersOpts is options for Containers() and ContainerNames() type ContainersOpts struct { Limit int // For an integer value n, limits the number of results to at most n values. Prefix string // Given a string value x, return container names matching the specified prefix. Marker string // Given a string value x, return container names greater in value than the specified marker. EndMarker string // Given a string value x, return container names less in value than the specified marker. Headers Headers // Any additional HTTP headers - can be nil } // parse the ContainerOpts func (opts *ContainersOpts) parse() (url.Values, Headers) { v := url.Values{} var h Headers if opts != nil { if opts.Limit > 0 { v.Set("limit", strconv.Itoa(opts.Limit)) } if opts.Prefix != "" { v.Set("prefix", opts.Prefix) } if opts.Marker != "" { v.Set("marker", opts.Marker) } if opts.EndMarker != "" { v.Set("end_marker", opts.EndMarker) } h = opts.Headers } return v, h } // ContainerNames returns a slice of names of containers in this account. func (c *Connection) ContainerNames(opts *ContainersOpts) ([]string, error) { v, h := opts.parse() resp, _, err := c.storage(RequestOpts{ Operation: "GET", Parameters: v, ErrorMap: ContainerErrorMap, Headers: h, }) if err != nil { return nil, err } lines, err := readLines(resp) return lines, err } // Container contains information about a container type Container struct { Name string // Name of the container Count int64 // Number of objects in the container Bytes int64 // Total number of bytes used in the container } // Containers returns a slice of structures with full information as // described in Container. func (c *Connection) Containers(opts *ContainersOpts) ([]Container, error) { v, h := opts.parse() v.Set("format", "json") resp, _, err := c.storage(RequestOpts{ Operation: "GET", Parameters: v, ErrorMap: ContainerErrorMap, Headers: h, }) if err != nil { return nil, err } var containers []Container err = readJson(resp, &containers) return containers, err } // containersAllOpts makes a copy of opts if set or makes a new one and // overrides Limit and Marker func containersAllOpts(opts *ContainersOpts) *ContainersOpts { var newOpts ContainersOpts if opts != nil { newOpts = *opts } if newOpts.Limit == 0 { newOpts.Limit = allContainersLimit } newOpts.Marker = "" return &newOpts } // ContainersAll is like Containers but it returns all the Containers // // It calls Containers multiple times using the Marker parameter // // It has a default Limit parameter but you may pass in your own func (c *Connection) ContainersAll(opts *ContainersOpts) ([]Container, error) { opts = containersAllOpts(opts) containers := make([]Container, 0) for { newContainers, err := c.Containers(opts) if err != nil { return nil, err } containers = append(containers, newContainers...) if len(newContainers) < opts.Limit { break } opts.Marker = newContainers[len(newContainers)-1].Name } return containers, nil } // ContainerNamesAll is like ContainerNames but it returns all the Containers // // It calls ContainerNames multiple times using the Marker parameter // // It has a default Limit parameter but you may pass in your own func (c *Connection) ContainerNamesAll(opts *ContainersOpts) ([]string, error) { opts = containersAllOpts(opts) containers := make([]string, 0) for { newContainers, err := c.ContainerNames(opts) if err != nil { return nil, err } containers = append(containers, newContainers...) if len(newContainers) < opts.Limit { break } opts.Marker = newContainers[len(newContainers)-1] } return containers, nil } /* ------------------------------------------------------------ */ // ObjectOpts is options for Objects() and ObjectNames() type ObjectsOpts struct { Limit int // For an integer value n, limits the number of results to at most n values. Marker string // Given a string value x, return object names greater in value than the specified marker. EndMarker string // Given a string value x, return object names less in value than the specified marker Prefix string // For a string value x, causes the results to be limited to object names beginning with the substring x. Path string // For a string value x, return the object names nested in the pseudo path Delimiter rune // For a character c, return all the object names nested in the container Headers Headers // Any additional HTTP headers - can be nil KeepMarker bool // Do not reset Marker when using ObjectsAll or ObjectNamesAll } // parse reads values out of ObjectsOpts func (opts *ObjectsOpts) parse() (url.Values, Headers) { v := url.Values{} var h Headers if opts != nil { if opts.Limit > 0 { v.Set("limit", strconv.Itoa(opts.Limit)) } if opts.Marker != "" { v.Set("marker", opts.Marker) } if opts.EndMarker != "" { v.Set("end_marker", opts.EndMarker) } if opts.Prefix != "" { v.Set("prefix", opts.Prefix) } if opts.Path != "" { v.Set("path", opts.Path) } if opts.Delimiter != 0 { v.Set("delimiter", string(opts.Delimiter)) } h = opts.Headers } return v, h } // ObjectNames returns a slice of names of objects in a given container. func (c *Connection) ObjectNames(container string, opts *ObjectsOpts) ([]string, error) { v, h := opts.parse() resp, _, err := c.storage(RequestOpts{ Container: container, Operation: "GET", Parameters: v, ErrorMap: ContainerErrorMap, Headers: h, }) if err != nil { return nil, err } return readLines(resp) } // Object contains information about an object type Object struct { Name string `json:"name"` // object name ContentType string `json:"content_type"` // eg application/directory Bytes int64 `json:"bytes"` // size in bytes ServerLastModified string `json:"last_modified"` // Last modified time, eg '2011-06-30T08:20:47.736680' as a string supplied by the server LastModified time.Time // Last modified time converted to a time.Time Hash string `json:"hash"` // MD5 hash, eg "d41d8cd98f00b204e9800998ecf8427e" SLOHash string `json:"slo_etag"` // MD5 hash of all segments' MD5 hash, eg "d41d8cd98f00b204e9800998ecf8427e" PseudoDirectory bool // Set when using delimiter to show that this directory object does not really exist SubDir string `json:"subdir"` // returned only when using delimiter to mark "pseudo directories" ObjectType ObjectType // type of this object } // Objects returns a slice of Object with information about each // object in the container. // // If Delimiter is set in the opts then PseudoDirectory may be set, // with ContentType 'application/directory'. These are not real // objects but represent directories of objects which haven't had an // object created for them. func (c *Connection) Objects(container string, opts *ObjectsOpts) ([]Object, error) { v, h := opts.parse() v.Set("format", "json") resp, _, err := c.storage(RequestOpts{ Container: container, Operation: "GET", Parameters: v, ErrorMap: ContainerErrorMap, Headers: h, }) if err != nil { return nil, err } var objects []Object err = readJson(resp, &objects) // Convert Pseudo directories and dates for i := range objects { object := &objects[i] if object.SubDir != "" { object.Name = object.SubDir object.PseudoDirectory = true object.ContentType = "application/directory" } if object.ServerLastModified != "" { // 2012-11-11T14:49:47.887250 // // Remove fractional seconds if present. This // then keeps it consistent with Object // which can only return timestamps accurate // to 1 second // // The TimeFormat will parse fractional // seconds if desired though datetime := strings.SplitN(object.ServerLastModified, ".", 2)[0] object.LastModified, err = time.Parse(TimeFormat, datetime) if err != nil { return nil, err } } if object.SLOHash != "" { object.ObjectType = StaticLargeObjectType } } return objects, err } // objectsAllOpts makes a copy of opts if set or makes a new one and // overrides Limit and Marker // Marker is not overriden if KeepMarker is set func objectsAllOpts(opts *ObjectsOpts, Limit int) *ObjectsOpts { var newOpts ObjectsOpts if opts != nil { newOpts = *opts } if newOpts.Limit == 0 { newOpts.Limit = Limit } if !newOpts.KeepMarker { newOpts.Marker = "" } return &newOpts } // A closure defined by the caller to iterate through all objects // // Call Objects or ObjectNames from here with the *ObjectOpts passed in // // Do whatever is required with the results then return them type ObjectsWalkFn func(*ObjectsOpts) (interface{}, error) // ObjectsWalk is uses to iterate through all the objects in chunks as // returned by Objects or ObjectNames using the Marker and Limit // parameters in the ObjectsOpts. // // Pass in a closure `walkFn` which calls Objects or ObjectNames with // the *ObjectsOpts passed to it and does something with the results. // // Errors will be returned from this function // // It has a default Limit parameter but you may pass in your own func (c *Connection) ObjectsWalk(container string, opts *ObjectsOpts, walkFn ObjectsWalkFn) error { opts = objectsAllOpts(opts, allObjectsChanLimit) for { objects, err := walkFn(opts) if err != nil { return err } var n int var last string switch objects := objects.(type) { case []string: n = len(objects) if n > 0 { last = objects[len(objects)-1] } case []Object: n = len(objects) if n > 0 { last = objects[len(objects)-1].Name } default: panic("Unknown type returned to ObjectsWalk") } if n < opts.Limit { break } opts.Marker = last } return nil } // ObjectsAll is like Objects but it returns an unlimited number of Objects in a slice // // It calls Objects multiple times using the Marker parameter func (c *Connection) ObjectsAll(container string, opts *ObjectsOpts) ([]Object, error) { objects := make([]Object, 0) err := c.ObjectsWalk(container, opts, func(opts *ObjectsOpts) (interface{}, error) { newObjects, err := c.Objects(container, opts) if err == nil { objects = append(objects, newObjects...) } return newObjects, err }) return objects, err } // ObjectNamesAll is like ObjectNames but it returns all the Objects // // It calls ObjectNames multiple times using the Marker parameter. Marker is // reset unless KeepMarker is set // // It has a default Limit parameter but you may pass in your own func (c *Connection) ObjectNamesAll(container string, opts *ObjectsOpts) ([]string, error) { objects := make([]string, 0) err := c.ObjectsWalk(container, opts, func(opts *ObjectsOpts) (interface{}, error) { newObjects, err := c.ObjectNames(container, opts) if err == nil { objects = append(objects, newObjects...) } return newObjects, err }) return objects, err } // Account contains information about this account. type Account struct { BytesUsed int64 // total number of bytes used Containers int64 // total number of containers Objects int64 // total number of objects } // getInt64FromHeader is a helper function to decode int64 from header. func getInt64FromHeader(resp *http.Response, header string) (result int64, err error) { value := resp.Header.Get(header) result, err = strconv.ParseInt(value, 10, 64) if err != nil { err = newErrorf(0, "Bad Header '%s': '%s': %s", header, value, err) } return } // Account returns info about the account in an Account struct. func (c *Connection) Account() (info Account, headers Headers, err error) { var resp *http.Response resp, headers, err = c.storage(RequestOpts{ Operation: "HEAD", ErrorMap: ContainerErrorMap, NoResponse: true, }) if err != nil { return } // Parse the headers into a dict // // {'Accept-Ranges': 'bytes', // 'Content-Length': '0', // 'Date': 'Tue, 05 Jul 2011 16:37:06 GMT', // 'X-Account-Bytes-Used': '316598182', // 'X-Account-Container-Count': '4', // 'X-Account-Object-Count': '1433'} if info.BytesUsed, err = getInt64FromHeader(resp, "X-Account-Bytes-Used"); err != nil { return } if info.Containers, err = getInt64FromHeader(resp, "X-Account-Container-Count"); err != nil { return } if info.Objects, err = getInt64FromHeader(resp, "X-Account-Object-Count"); err != nil { return } return } // AccountUpdate adds, replaces or remove account metadata. // // Add or update keys by mentioning them in the Headers. // // Remove keys by setting them to an empty string. func (c *Connection) AccountUpdate(h Headers) error { _, _, err := c.storage(RequestOpts{ Operation: "POST", ErrorMap: ContainerErrorMap, NoResponse: true, Headers: h, }) return err } // ContainerCreate creates a container. // // If you don't want to add Headers just pass in nil // // No error is returned if it already exists but the metadata if any will be updated. func (c *Connection) ContainerCreate(container string, h Headers) error { _, _, err := c.storage(RequestOpts{ Container: container, Operation: "PUT", ErrorMap: ContainerErrorMap, NoResponse: true, Headers: h, }) return err } // ContainerDelete deletes a container. // // May return ContainerDoesNotExist or ContainerNotEmpty func (c *Connection) ContainerDelete(container string) error { _, _, err := c.storage(RequestOpts{ Container: container, Operation: "DELETE", ErrorMap: ContainerErrorMap, NoResponse: true, }) return err } // Container returns info about a single container including any // metadata in the headers. func (c *Connection) Container(container string) (info Container, headers Headers, err error) { var resp *http.Response resp, headers, err = c.storage(RequestOpts{ Container: container, Operation: "HEAD", ErrorMap: ContainerErrorMap, NoResponse: true, }) if err != nil { return } // Parse the headers into the struct info.Name = container if info.Bytes, err = getInt64FromHeader(resp, "X-Container-Bytes-Used"); err != nil { return } if info.Count, err = getInt64FromHeader(resp, "X-Container-Object-Count"); err != nil { return } return } // ContainerUpdate adds, replaces or removes container metadata. // // Add or update keys by mentioning them in the Metadata. // // Remove keys by setting them to an empty string. // // Container metadata can only be read with Container() not with Containers(). func (c *Connection) ContainerUpdate(container string, h Headers) error { _, _, err := c.storage(RequestOpts{ Container: container, Operation: "POST", ErrorMap: ContainerErrorMap, NoResponse: true, Headers: h, }) return err } // ------------------------------------------------------------ // ObjectCreateFile represents a swift object open for writing type ObjectCreateFile struct { checkHash bool // whether we are checking the hash pipeReader *io.PipeReader // pipe for the caller to use pipeWriter *io.PipeWriter hash hash.Hash // hash being build up as we go along done chan struct{} // signals when the upload has finished resp *http.Response // valid when done has signalled err error // ditto headers Headers // ditto } // Write bytes to the object - see io.Writer func (file *ObjectCreateFile) Write(p []byte) (n int, err error) { n, err = file.pipeWriter.Write(p) if err == io.ErrClosedPipe { if file.err != nil { return 0, file.err } return 0, newError(500, "Write on closed file") } if err == nil && file.checkHash { _, _ = file.hash.Write(p) } return } // CloseWithError closes the object, aborting the upload. func (file *ObjectCreateFile) CloseWithError(err error) error { _ = file.pipeWriter.CloseWithError(err) <-file.done return nil } // Close the object and checks the md5sum if it was required. // // Also returns any other errors from the server (eg container not // found) so it is very important to check the errors on this method. func (file *ObjectCreateFile) Close() error { // Close the body err := file.pipeWriter.Close() if err != nil { return err } // Wait for the HTTP operation to complete <-file.done // Check errors if file.err != nil { return file.err } if file.checkHash { receivedMd5 := strings.ToLower(file.headers["Etag"]) calculatedMd5 := fmt.Sprintf("%x", file.hash.Sum(nil)) if receivedMd5 != calculatedMd5 { return ObjectCorrupted } } return nil } // Headers returns the response headers from the created object if the upload // has been completed. The Close() method must be called on an ObjectCreateFile // before this method. func (file *ObjectCreateFile) Headers() (Headers, error) { // error out if upload is not complete. select { case <-file.done: default: return nil, fmt.Errorf("Cannot get metadata, object upload failed or has not yet completed.") } return file.headers, nil } // Check it satisfies the interface var _ io.WriteCloser = &ObjectCreateFile{} // objectPutHeaders create a set of headers for a PUT // // It guesses the contentType from the objectName if it isn't set // // checkHash may be changed func objectPutHeaders(objectName string, checkHash *bool, Hash string, contentType string, h Headers) Headers { if contentType == "" { contentType = mime.TypeByExtension(path.Ext(objectName)) if contentType == "" { contentType = "application/octet-stream" } } // Meta stuff extraHeaders := map[string]string{ "Content-Type": contentType, } for key, value := range h { extraHeaders[key] = value } if Hash != "" { extraHeaders["Etag"] = Hash *checkHash = false // the server will do it } return extraHeaders } // ObjectCreate creates or updates the object in the container. It // returns an io.WriteCloser you should write the contents to. You // MUST call Close() on it and you MUST check the error return from // Close(). // // If checkHash is True then it will calculate the MD5 Hash of the // file as it is being uploaded and check it against that returned // from the server. If it is wrong then it will return // ObjectCorrupted on Close() // // If you know the MD5 hash of the object ahead of time then set the // Hash parameter and it will be sent to the server (as an Etag // header) and the server will check the MD5 itself after the upload, // and this will return ObjectCorrupted on Close() if it is incorrect. // // If you don't want any error protection (not recommended) then set // checkHash to false and Hash to "". // // If contentType is set it will be used, otherwise one will be // guessed from objectName using mime.TypeByExtension func (c *Connection) ObjectCreate(container string, objectName string, checkHash bool, Hash string, contentType string, h Headers) (file *ObjectCreateFile, err error) { extraHeaders := objectPutHeaders(objectName, &checkHash, Hash, contentType, h) pipeReader, pipeWriter := io.Pipe() file = &ObjectCreateFile{ hash: md5.New(), checkHash: checkHash, pipeReader: pipeReader, pipeWriter: pipeWriter, done: make(chan struct{}), } // Run the PUT in the background piping it data go func() { opts := RequestOpts{ Container: container, ObjectName: objectName, Operation: "PUT", Headers: extraHeaders, Body: pipeReader, NoResponse: true, ErrorMap: objectErrorMap, } file.resp, file.headers, file.err = c.storage(opts) // Signal finished pipeReader.Close() close(file.done) }() return } func (c *Connection) ObjectSymlinkCreate(container string, symlink string, targetAccount string, targetContainer string, targetObject string, targetEtag string) (headers Headers, err error) { EMPTY_MD5 := "d41d8cd98f00b204e9800998ecf8427e" symHeaders := Headers{} contents := bytes.NewBufferString("") if targetAccount != "" { symHeaders["X-Symlink-Target-Account"] = targetAccount } if targetEtag != "" { symHeaders["X-Symlink-Target-Etag"] = targetEtag } symHeaders["X-Symlink-Target"] = fmt.Sprintf("%s/%s", targetContainer, targetObject) _, err = c.ObjectPut(container, symlink, contents, true, EMPTY_MD5, "application/symlink", symHeaders) return } func (c *Connection) objectPut(container string, objectName string, contents io.Reader, checkHash bool, Hash string, contentType string, h Headers, parameters url.Values) (headers Headers, err error) { extraHeaders := objectPutHeaders(objectName, &checkHash, Hash, contentType, h) hash := md5.New() var body io.Reader = contents if checkHash { body = io.TeeReader(contents, hash) } _, headers, err = c.storage(RequestOpts{ Container: container, ObjectName: objectName, Operation: "PUT", Headers: extraHeaders, Body: body, NoResponse: true, ErrorMap: objectErrorMap, Parameters: parameters, }) if err != nil { return } if checkHash { receivedMd5 := strings.ToLower(headers["Etag"]) calculatedMd5 := fmt.Sprintf("%x", hash.Sum(nil)) if receivedMd5 != calculatedMd5 { err = ObjectCorrupted return } } return } // ObjectPut creates or updates the path in the container from // contents. contents should be an open io.Reader which will have all // its contents read. // // This is a low level interface. // // If checkHash is True then it will calculate the MD5 Hash of the // file as it is being uploaded and check it against that returned // from the server. If it is wrong then it will return // ObjectCorrupted. // // If you know the MD5 hash of the object ahead of time then set the // Hash parameter and it will be sent to the server (as an Etag // header) and the server will check the MD5 itself after the upload, // and this will return ObjectCorrupted if it is incorrect. // // If you don't want any error protection (not recommended) then set // checkHash to false and Hash to "". // // If contentType is set it will be used, otherwise one will be // guessed from objectName using mime.TypeByExtension func (c *Connection) ObjectPut(container string, objectName string, contents io.Reader, checkHash bool, Hash string, contentType string, h Headers) (headers Headers, err error) { return c.objectPut(container, objectName, contents, checkHash, Hash, contentType, h, nil) } // ObjectPutBytes creates an object from a []byte in a container. // // This is a simplified interface which checks the MD5. func (c *Connection) ObjectPutBytes(container string, objectName string, contents []byte, contentType string) (err error) { buf := bytes.NewBuffer(contents) h := Headers{"Content-Length": strconv.Itoa(len(contents))} _, err = c.ObjectPut(container, objectName, buf, true, "", contentType, h) return } // ObjectPutString creates an object from a string in a container. // // This is a simplified interface which checks the MD5 func (c *Connection) ObjectPutString(container string, objectName string, contents string, contentType string) (err error) { buf := strings.NewReader(contents) h := Headers{"Content-Length": strconv.Itoa(len(contents))} _, err = c.ObjectPut(container, objectName, buf, true, "", contentType, h) return } // ObjectOpenFile represents a swift object open for reading type ObjectOpenFile struct { connection *Connection // stored copy of Connection used in Open container string // stored copy of container used in Open objectName string // stored copy of objectName used in Open headers Headers // stored copy of headers used in Open resp *http.Response // http connection body io.Reader // read data from this checkHash bool // true if checking MD5 hash hash.Hash // currently accumulating MD5 bytes int64 // number of bytes read on this connection eof bool // whether we have read end of file pos int64 // current position when reading lengthOk bool // whether length is valid length int64 // length of the object if read seeked bool // whether we have seeked this file or not overSeeked bool // set if we have seeked to the end or beyond } // Read bytes from the object - see io.Reader func (file *ObjectOpenFile) Read(p []byte) (n int, err error) { if file.overSeeked { return 0, io.EOF } n, err = file.body.Read(p) file.bytes += int64(n) file.pos += int64(n) if err == io.EOF { file.eof = true } return } // Seek sets the offset for the next Read to offset, interpreted // according to whence: 0 means relative to the origin of the file, 1 // means relative to the current offset, and 2 means relative to the // end. Seek returns the new offset and an Error, if any. // // Seek uses HTTP Range headers which, if the file pointer is moved, // will involve reopening the HTTP connection. // // Note that you can't seek to the end of a file or beyond; HTTP Range // requests don't support the file pointer being outside the data, // unlike os.File // // Seek(0, 1) will return the current file pointer. func (file *ObjectOpenFile) Seek(offset int64, whence int) (newPos int64, err error) { file.overSeeked = false switch whence { case 0: // relative to start newPos = offset case 1: // relative to current newPos = file.pos + offset case 2: // relative to end if !file.lengthOk { return file.pos, newError(0, "Length of file unknown so can't seek from end") } newPos = file.length + offset if offset >= 0 { file.overSeeked = true return } default: panic("Unknown whence in ObjectOpenFile.Seek") } // If at correct position (quite likely), do nothing if newPos == file.pos { return } // Close the file... file.seeked = true err = file.Close() if err != nil { return } // ...and re-open with a Range header if file.headers == nil { file.headers = Headers{} } if newPos > 0 { file.headers["Range"] = fmt.Sprintf("bytes=%d-", newPos) } else { delete(file.headers, "Range") } newFile, _, err := file.connection.ObjectOpen(file.container, file.objectName, false, file.headers) if err != nil { return } // Update the file file.resp = newFile.resp file.body = newFile.body file.checkHash = false file.pos = newPos return } // Length gets the objects content length either from a cached copy or // from the server. func (file *ObjectOpenFile) Length() (int64, error) { if !file.lengthOk { info, _, err := file.connection.Object(file.container, file.objectName) file.length = info.Bytes file.lengthOk = (err == nil) return file.length, err } return file.length, nil } // Close the object and checks the length and md5sum if it was // required and all the object was read func (file *ObjectOpenFile) Close() (err error) { // Close the body at the end defer checkClose(file.resp.Body, &err) // If not end of file or seeked then can't check anything if !file.eof || file.seeked { return } // Check the MD5 sum if requested if file.checkHash { receivedMd5 := strings.ToLower(file.resp.Header.Get("Etag")) calculatedMd5 := fmt.Sprintf("%x", file.hash.Sum(nil)) if receivedMd5 != calculatedMd5 { err = ObjectCorrupted return } } // Check to see we read the correct number of bytes if file.lengthOk && file.length != file.bytes { err = ObjectCorrupted return } return } // Check it satisfies the interfaces var _ io.ReadCloser = &ObjectOpenFile{} var _ io.Seeker = &ObjectOpenFile{} func (c *Connection) objectOpenBase(container string, objectName string, checkHash bool, h Headers, parameters url.Values) (file *ObjectOpenFile, headers Headers, err error) { var resp *http.Response opts := RequestOpts{ Container: container, ObjectName: objectName, Operation: "GET", ErrorMap: objectErrorMap, Headers: h, Parameters: parameters, } resp, headers, err = c.storage(opts) if err != nil { return } // Can't check MD5 on an object with X-Object-Manifest or X-Static-Large-Object set if checkHash && headers.IsLargeObject() { // log.Printf("swift: turning off md5 checking on object with manifest %v", objectName) checkHash = false } file = &ObjectOpenFile{ connection: c, container: container, objectName: objectName, headers: h, resp: resp, checkHash: checkHash, body: resp.Body, } if checkHash { file.hash = md5.New() file.body = io.TeeReader(resp.Body, file.hash) } // Read Content-Length if resp.Header.Get("Content-Length") != "" { file.length, err = getInt64FromHeader(resp, "Content-Length") file.lengthOk = (err == nil) } return } func (c *Connection) objectOpen(container string, objectName string, checkHash bool, h Headers, parameters url.Values) (file *ObjectOpenFile, headers Headers, err error) { err = withLORetry(0, func() (Headers, int64, error) { file, headers, err = c.objectOpenBase(container, objectName, checkHash, h, parameters) if err != nil { return headers, 0, err } return headers, file.length, nil }) return } // ObjectOpen returns an ObjectOpenFile for reading the contents of // the object. This satisfies the io.ReadCloser and the io.Seeker // interfaces. // // You must call Close() on contents when finished // // Returns the headers of the response. // // If checkHash is true then it will calculate the md5sum of the file // as it is being received and check it against that returned from the // server. If it is wrong then it will return ObjectCorrupted. It // will also check the length returned. No checking will be done if // you don't read all the contents. // // Note that objects with X-Object-Manifest or X-Static-Large-Object // set won't ever have their md5sum's checked as the md5sum reported // on the object is actually the md5sum of the md5sums of the // parts. This isn't very helpful to detect a corrupted download as // the size of the parts aren't known without doing more operations. // If you want to ensure integrity of an object with a manifest then // you will need to download everything in the manifest separately. // // headers["Content-Type"] will give the content type if desired. func (c *Connection) ObjectOpen(container string, objectName string, checkHash bool, h Headers) (file *ObjectOpenFile, headers Headers, err error) { return c.objectOpen(container, objectName, checkHash, h, nil) } // ObjectGet gets the object into the io.Writer contents. // // Returns the headers of the response. // // If checkHash is true then it will calculate the md5sum of the file // as it is being received and check it against that returned from the // server. If it is wrong then it will return ObjectCorrupted. // // headers["Content-Type"] will give the content type if desired. func (c *Connection) ObjectGet(container string, objectName string, contents io.Writer, checkHash bool, h Headers) (headers Headers, err error) { file, headers, err := c.ObjectOpen(container, objectName, checkHash, h) if err != nil { return } defer checkClose(file, &err) _, err = io.Copy(contents, file) return } // ObjectGetBytes returns an object as a []byte. // // This is a simplified interface which checks the MD5 func (c *Connection) ObjectGetBytes(container string, objectName string) (contents []byte, err error) { var buf bytes.Buffer _, err = c.ObjectGet(container, objectName, &buf, true, nil) contents = buf.Bytes() return } // ObjectGetString returns an object as a string. // // This is a simplified interface which checks the MD5 func (c *Connection) ObjectGetString(container string, objectName string) (contents string, err error) { var buf bytes.Buffer _, err = c.ObjectGet(container, objectName, &buf, true, nil) contents = buf.String() return } // ObjectDelete deletes the object. // // May return ObjectNotFound if the object isn't found func (c *Connection) ObjectDelete(container string, objectName string) error { _, _, err := c.storage(RequestOpts{ Container: container, ObjectName: objectName, Operation: "DELETE", ErrorMap: objectErrorMap, }) return err } // ObjectTempUrl returns a temporary URL for an object func (c *Connection) ObjectTempUrl(container string, objectName string, secretKey string, method string, expires time.Time) string { mac := hmac.New(sha1.New, []byte(secretKey)) prefix, _ := url.Parse(c.StorageUrl) body := fmt.Sprintf("%s\n%d\n%s/%s/%s", method, expires.Unix(), prefix.Path, container, objectName) mac.Write([]byte(body)) sig := hex.EncodeToString(mac.Sum(nil)) return fmt.Sprintf("%s/%s/%s?temp_url_sig=%s&temp_url_expires=%d", c.StorageUrl, container, objectName, sig, expires.Unix()) } // parseResponseStatus parses string like "200 OK" and returns Error. // // For status codes beween 200 and 299, this returns nil. func parseResponseStatus(resp string, errorMap errorMap) error { code := 0 reason := resp t := strings.SplitN(resp, " ", 2) if len(t) == 2 { ncode, err := strconv.Atoi(t[0]) if err == nil { code = ncode reason = t[1] } } if errorMap != nil { if err, ok := errorMap[code]; ok { return err } } if 200 <= code && code <= 299 { return nil } return newError(code, reason) } // BulkDeleteResult stores results of BulkDelete(). // // Individual errors may (or may not) be returned by Errors. // Errors is a map whose keys are a full path of where the object was // to be deleted, and whose values are Error objects. A full path of // object looks like "/API_VERSION/USER_ACCOUNT/CONTAINER/OBJECT_PATH". type BulkDeleteResult struct { NumberNotFound int64 // # of objects not found. NumberDeleted int64 // # of deleted objects. Errors map[string]error // Mapping between object name and an error. Headers Headers // Response HTTP headers. } func (c *Connection) doBulkDelete(objects []string, h Headers) (result BulkDeleteResult, err error) { var buffer bytes.Buffer for _, s := range objects { u := url.URL{Path: s} buffer.WriteString(u.String() + "\n") } extraHeaders := Headers{ "Accept": "application/json", "Content-Type": "text/plain", "Content-Length": strconv.Itoa(buffer.Len()), } for key, value := range h { extraHeaders[key] = value } resp, headers, err := c.storage(RequestOpts{ Operation: "DELETE", Parameters: url.Values{"bulk-delete": []string{"1"}}, Headers: extraHeaders, ErrorMap: ContainerErrorMap, Body: &buffer, }) if err != nil { return } var jsonResult struct { NotFound int64 `json:"Number Not Found"` Status string `json:"Response Status"` Errors [][]string Deleted int64 `json:"Number Deleted"` } err = readJson(resp, &jsonResult) if err != nil { return } err = parseResponseStatus(jsonResult.Status, objectErrorMap) result.NumberNotFound = jsonResult.NotFound result.NumberDeleted = jsonResult.Deleted result.Headers = headers el := make(map[string]error, len(jsonResult.Errors)) for _, t := range jsonResult.Errors { if len(t) != 2 { continue } el[t[0]] = parseResponseStatus(t[1], objectErrorMap) } result.Errors = el return } // BulkDelete deletes multiple objectNames from container in one operation. // // Some servers may not accept bulk-delete requests since bulk-delete is // an optional feature of swift - these will return the Forbidden error. // // See also: // * http://docs.openstack.org/trunk/openstack-object-storage/admin/content/object-storage-bulk-delete.html // * http://docs.rackspace.com/files/api/v1/cf-devguide/content/Bulk_Delete-d1e2338.html func (c *Connection) BulkDelete(container string, objectNames []string) (result BulkDeleteResult, err error) { return c.BulkDeleteHeaders(container, objectNames, nil) } // BulkDeleteHeaders deletes multiple objectNames from container in one operation. // // Some servers may not accept bulk-delete requests since bulk-delete is // an optional feature of swift - these will return the Forbidden error. // // See also: // * http://docs.openstack.org/trunk/openstack-object-storage/admin/content/object-storage-bulk-delete.html // * http://docs.rackspace.com/files/api/v1/cf-devguide/content/Bulk_Delete-d1e2338.html func (c *Connection) BulkDeleteHeaders(container string, objectNames []string, h Headers) (result BulkDeleteResult, err error) { if len(objectNames) == 0 { result.Errors = make(map[string]error) return } fullPaths := make([]string, len(objectNames)) for i, name := range objectNames { fullPaths[i] = fmt.Sprintf("/%s/%s", container, name) } return c.doBulkDelete(fullPaths, h) } // BulkUploadResult stores results of BulkUpload(). // // Individual errors may (or may not) be returned by Errors. // Errors is a map whose keys are a full path of where an object was // to be created, and whose values are Error objects. A full path of // object looks like "/API_VERSION/USER_ACCOUNT/CONTAINER/OBJECT_PATH". type BulkUploadResult struct { NumberCreated int64 // # of created objects. Errors map[string]error // Mapping between object name and an error. Headers Headers // Response HTTP headers. } // BulkUpload uploads multiple files in one operation. // // uploadPath can be empty, a container name, or a pseudo-directory // within a container. If uploadPath is empty, new containers may be // automatically created. // // Files are read from dataStream. The format of the stream is specified // by the format parameter. Available formats are: // * UploadTar - Plain tar stream. // * UploadTarGzip - Gzip compressed tar stream. // * UploadTarBzip2 - Bzip2 compressed tar stream. // // Some servers may not accept bulk-upload requests since bulk-upload is // an optional feature of swift - these will return the Forbidden error. // // See also: // * http://docs.openstack.org/trunk/openstack-object-storage/admin/content/object-storage-extract-archive.html // * http://docs.rackspace.com/files/api/v1/cf-devguide/content/Extract_Archive-d1e2338.html func (c *Connection) BulkUpload(uploadPath string, dataStream io.Reader, format string, h Headers) (result BulkUploadResult, err error) { extraHeaders := Headers{"Accept": "application/json"} for key, value := range h { extraHeaders[key] = value } // The following code abuses Container parameter intentionally. // The best fix might be to rename Container to UploadPath. resp, headers, err := c.storage(RequestOpts{ Container: uploadPath, Operation: "PUT", Parameters: url.Values{"extract-archive": []string{format}}, Headers: extraHeaders, ErrorMap: ContainerErrorMap, Body: dataStream, }) if err != nil { return } // Detect old servers which don't support this feature if headers["Content-Type"] != "application/json" { err = Forbidden return } var jsonResult struct { Created int64 `json:"Number Files Created"` Status string `json:"Response Status"` Errors [][]string } err = readJson(resp, &jsonResult) if err != nil { return } err = parseResponseStatus(jsonResult.Status, objectErrorMap) result.NumberCreated = jsonResult.Created result.Headers = headers el := make(map[string]error, len(jsonResult.Errors)) for _, t := range jsonResult.Errors { if len(t) != 2 { continue } el[t[0]] = parseResponseStatus(t[1], objectErrorMap) } result.Errors = el return } // Object returns info about a single object including any metadata in the header. // // May return ObjectNotFound. // // Use headers.ObjectMetadata() to read the metadata in the Headers. func (c *Connection) Object(container string, objectName string) (info Object, headers Headers, err error) { err = withLORetry(0, func() (Headers, int64, error) { info, headers, err = c.objectBase(container, objectName) if err != nil { return headers, 0, err } return headers, info.Bytes, nil }) return } func (c *Connection) objectBase(container string, objectName string) (info Object, headers Headers, err error) { var resp *http.Response resp, headers, err = c.storage(RequestOpts{ Container: container, ObjectName: objectName, Operation: "HEAD", ErrorMap: objectErrorMap, NoResponse: true, }) if err != nil { return } // Parse the headers into the struct // HTTP/1.1 200 OK // Date: Thu, 07 Jun 2010 20:59:39 GMT // Server: Apache // Last-Modified: Fri, 12 Jun 2010 13:40:18 GMT // ETag: 8a964ee2a5e88be344f36c22562a6486 // Content-Length: 512000 // Content-Type: text/plain; charset=UTF-8 // X-Object-Meta-Meat: Bacon // X-Object-Meta-Fruit: Bacon // X-Object-Meta-Veggie: Bacon // X-Object-Meta-Dairy: Bacon info.Name = objectName info.ContentType = resp.Header.Get("Content-Type") if resp.Header.Get("Content-Length") != "" { if info.Bytes, err = getInt64FromHeader(resp, "Content-Length"); err != nil { return } } // Currently ceph doesn't return a Last-Modified header for DLO manifests without any segments // See ceph http://tracker.ceph.com/issues/15812 if resp.Header.Get("Last-Modified") != "" { info.ServerLastModified = resp.Header.Get("Last-Modified") if info.LastModified, err = time.Parse(http.TimeFormat, info.ServerLastModified); err != nil { return } } info.Hash = resp.Header.Get("Etag") if resp.Header.Get("X-Object-Manifest") != "" { info.ObjectType = DynamicLargeObjectType } else if resp.Header.Get("X-Static-Large-Object") != "" { info.ObjectType = StaticLargeObjectType } return } // ObjectUpdate adds, replaces or removes object metadata. // // Add or Update keys by mentioning them in the Metadata. Use // Metadata.ObjectHeaders and Headers.ObjectMetadata to convert your // Metadata to and from normal HTTP headers. // // This removes all metadata previously added to the object and // replaces it with that passed in so to delete keys, just don't // mention them the headers you pass in. // // Object metadata can only be read with Object() not with Objects(). // // This can also be used to set headers not already assigned such as // X-Delete-At or X-Delete-After for expiring objects. // // You cannot use this to change any of the object's other headers // such as Content-Type, ETag, etc. // // Refer to copying an object when you need to update metadata or // other headers such as Content-Type or CORS headers. // // May return ObjectNotFound. func (c *Connection) ObjectUpdate(container string, objectName string, h Headers) error { _, _, err := c.storage(RequestOpts{ Container: container, ObjectName: objectName, Operation: "POST", ErrorMap: objectErrorMap, NoResponse: true, Headers: h, }) return err } // urlPathEscape escapes URL path the in string using URL escaping rules // // This mimics url.PathEscape which only available from go 1.8 func urlPathEscape(in string) string { var u url.URL u.Path = in return u.String() } // ObjectCopy does a server side copy of an object to a new position // // All metadata is preserved. If metadata is set in the headers then // it overrides the old metadata on the copied object. // // The destination container must exist before the copy. // // You can use this to copy an object to itself - this is the only way // to update the content type of an object. func (c *Connection) ObjectCopy(srcContainer string, srcObjectName string, dstContainer string, dstObjectName string, h Headers) (headers Headers, err error) { // Meta stuff extraHeaders := map[string]string{ "Destination": urlPathEscape(dstContainer + "/" + dstObjectName), } for key, value := range h { extraHeaders[key] = value } _, headers, err = c.storage(RequestOpts{ Container: srcContainer, ObjectName: srcObjectName, Operation: "COPY", ErrorMap: objectErrorMap, NoResponse: true, Headers: extraHeaders, }) return } // ObjectMove does a server side move of an object to a new position // // This is a convenience method which calls ObjectCopy then ObjectDelete // // All metadata is preserved. // // The destination container must exist before the copy. func (c *Connection) ObjectMove(srcContainer string, srcObjectName string, dstContainer string, dstObjectName string) (err error) { _, err = c.ObjectCopy(srcContainer, srcObjectName, dstContainer, dstObjectName, nil) if err != nil { return } return c.ObjectDelete(srcContainer, srcObjectName) } // ObjectUpdateContentType updates the content type of an object // // This is a convenience method which calls ObjectCopy // // All other metadata is preserved. func (c *Connection) ObjectUpdateContentType(container string, objectName string, contentType string) (err error) { h := Headers{"Content-Type": contentType} _, err = c.ObjectCopy(container, objectName, container, objectName, h) return } // ------------------------------------------------------------ // VersionContainerCreate is a helper method for creating and enabling version controlled containers. // // It builds the current object container, the non-current object version container, and enables versioning. // // If the server doesn't support versioning then it will return // Forbidden however it will have created both the containers at that point. func (c *Connection) VersionContainerCreate(current, version string) error { if err := c.ContainerCreate(version, nil); err != nil { return err } if err := c.ContainerCreate(current, nil); err != nil { return err } if err := c.VersionEnable(current, version); err != nil { return err } return nil } // VersionEnable enables versioning on the current container with version as the tracking container. // // May return Forbidden if this isn't supported by the server func (c *Connection) VersionEnable(current, version string) error { h := Headers{"X-Versions-Location": version} if err := c.ContainerUpdate(current, h); err != nil { return err } // Check to see if the header was set properly _, headers, err := c.Container(current) if err != nil { return err } // If failed to set versions header, return Forbidden as the server doesn't support this if headers["X-Versions-Location"] != version { return Forbidden } return nil } // VersionDisable disables versioning on the current container. func (c *Connection) VersionDisable(current string) error { h := Headers{"X-Versions-Location": ""} if err := c.ContainerUpdate(current, h); err != nil { return err } return nil } // VersionObjectList returns a list of older versions of the object. // // Objects are returned in the format / func (c *Connection) VersionObjectList(version, object string) ([]string, error) { opts := &ObjectsOpts{ // <3-character zero-padded hexadecimal character length>/ Prefix: fmt.Sprintf("%03x", len(object)) + object + "/", } return c.ObjectNames(version, opts) } swift-1.0.52/swift_internal_test.go000066400000000000000000000355071365550407400173740ustar00rootroot00000000000000// This tests the swift package internals // // It does not require access to a swift server // // FIXME need to add more tests and to check URLs and parameters package swift import ( "fmt" "io" "net" "net/http" "os" "reflect" "testing" "time" ) const ( TEST_ADDRESS = "localhost:5324" AUTH_URL = "http://" + TEST_ADDRESS + "/v1.0" PROXY_URL = "http://" + TEST_ADDRESS + "/proxy" USERNAME = "test" APIKEY = "apikey" AUTH_TOKEN = "token" ) // Globals var ( server *SwiftServer c *Connection ) // SwiftServer implements a test swift server type SwiftServer struct { t *testing.T checks []*Check } // Used to check and reply to http transactions type Check struct { in Headers out Headers rx *string tx *string err *Error url *string } // Add a in check func (check *Check) In(in Headers) *Check { check.in = in return check } // Add an out check func (check *Check) Out(out Headers) *Check { check.out = out return check } // Add an Error check func (check *Check) Error(StatusCode int, Text string) *Check { check.err = newError(StatusCode, Text) return check } // Add a rx check func (check *Check) Rx(rx string) *Check { check.rx = &rx return check } // Add an tx check func (check *Check) Tx(tx string) *Check { check.tx = &tx return check } // Add an URL check func (check *Check) Url(url string) *Check { check.url = &url return check } // Add a check func (s *SwiftServer) AddCheck(t *testing.T) *Check { server.t = t check := &Check{ in: Headers{}, out: Headers{}, err: nil, } s.checks = append(s.checks, check) return check } // Responds to a request func (s *SwiftServer) Respond(w http.ResponseWriter, r *http.Request) { if len(s.checks) < 1 { s.t.Fatal("Unexpected http transaction") } check := s.checks[0] s.checks = s.checks[1:] // Check URL if check.url != nil && *check.url != r.URL.String() { s.t.Errorf("Expecting URL %q but got %q", *check.url, r.URL) } // Check headers for k, v := range check.in { actual := r.Header.Get(k) if actual != v { s.t.Errorf("Expecting header %q=%q but got %q", k, v, actual) } } // Write output headers h := w.Header() for k, v := range check.out { h.Set(k, v) } // Return an error if required if check.err != nil { http.Error(w, check.err.Text, check.err.StatusCode) } else { if check.tx != nil { _, err := w.Write([]byte(*check.tx)) if err != nil { s.t.Error("Write failed", err) } } } } // Checks to see all responses are used up func (s *SwiftServer) Finished() { if len(s.checks) > 0 { s.t.Error("Unused checks", s.checks) } } func handle(w http.ResponseWriter, r *http.Request) { // out, _ := httputil.DumpRequest(r, true) // os.Stdout.Write(out) server.Respond(w, r) } func NewSwiftServer() *SwiftServer { server := &SwiftServer{} http.HandleFunc("/", handle) go http.ListenAndServe(TEST_ADDRESS, nil) fmt.Print("Waiting for server to start ") for { fmt.Print(".") conn, err := net.Dial("tcp", TEST_ADDRESS) if err == nil { conn.Close() fmt.Println(" Started") break } } return server } func init() { server = NewSwiftServer() c = &Connection{ UserName: USERNAME, ApiKey: APIKEY, AuthUrl: AUTH_URL, } } // Check the error is a swift error func checkError(t *testing.T, err error, StatusCode int, Text string) { if err == nil { t.Fatal("No error returned") } err2, ok := err.(*Error) if !ok { t.Fatal("Bad error type") } if err2.StatusCode != StatusCode { t.Fatalf("Bad status code, expecting %d got %d", StatusCode, err2.StatusCode) } if err2.Text != Text { t.Fatalf("Bad error string, expecting %q got %q", Text, err2.Text) } } // FIXME copied from swift_test.go func compareMaps(t *testing.T, a, b map[string]string) { if len(a) != len(b) { t.Error("Maps different sizes", a, b) } for ka, va := range a { if vb, ok := b[ka]; !ok || va != vb { t.Error("Difference in key", ka, va, b[ka]) } } for kb, vb := range b { if va, ok := a[kb]; !ok || vb != va { t.Error("Difference in key", kb, vb, a[kb]) } } } func TestInternalError(t *testing.T) { e := newError(404, "Not Found!") if e.StatusCode != 404 || e.Text != "Not Found!" { t.Fatal("Bad error") } if e.Error() != "Not Found!" { t.Fatal("Bad error") } } func testCheckClose(rd io.ReadCloser, e error) (err error) { err = e defer checkClose(rd, &err) return } // Make a closer which returns the error of our choice type myCloser struct { err error } func (c *myCloser) Read([]byte) (int, error) { return 0, io.EOF } func (c *myCloser) Close() error { return c.err } func TestInternalCheckClose(t *testing.T) { if testCheckClose(&myCloser{nil}, nil) != nil { t.Fatal("bad 1") } if testCheckClose(&myCloser{nil}, ObjectCorrupted) != ObjectCorrupted { t.Fatal("bad 2") } if testCheckClose(&myCloser{ObjectNotFound}, nil) != ObjectNotFound { t.Fatal("bad 3") } if testCheckClose(&myCloser{ObjectNotFound}, ObjectCorrupted) != ObjectCorrupted { t.Fatal("bad 4") } } func TestInternalParseHeaders(t *testing.T) { resp := &http.Response{StatusCode: 200} if c.parseHeaders(resp, nil) != nil { t.Error("Bad 1") } if c.parseHeaders(resp, authErrorMap) != nil { t.Error("Bad 1") } resp = &http.Response{StatusCode: 299} if c.parseHeaders(resp, nil) != nil { t.Error("Bad 1") } resp = &http.Response{StatusCode: 199, Status: "BOOM"} checkError(t, c.parseHeaders(resp, nil), 199, "HTTP Error: 199: BOOM") resp = &http.Response{StatusCode: 300, Status: "BOOM"} checkError(t, c.parseHeaders(resp, nil), 300, "HTTP Error: 300: BOOM") resp = &http.Response{StatusCode: 404, Status: "BOOM"} checkError(t, c.parseHeaders(resp, nil), 404, "HTTP Error: 404: BOOM") if c.parseHeaders(resp, ContainerErrorMap) != ContainerNotFound { t.Error("Bad 1") } if c.parseHeaders(resp, objectErrorMap) != ObjectNotFound { t.Error("Bad 1") } } func TestInternalReadHeaders(t *testing.T) { resp := &http.Response{Header: http.Header{}} compareMaps(t, readHeaders(resp), Headers{}) resp = &http.Response{Header: http.Header{ "one": []string{"1"}, "two": []string{"2"}, }} compareMaps(t, readHeaders(resp), Headers{"one": "1", "two": "2"}) // FIXME this outputs a log which we should test and check resp = &http.Response{Header: http.Header{ "one": []string{"1", "11", "111"}, "two": []string{"2"}, }} compareMaps(t, readHeaders(resp), Headers{"one": "1", "two": "2"}) } func TestInternalStorage(t *testing.T) { // FIXME } // ------------------------------------------------------------ func TestInternalAuthenticate(t *testing.T) { server.AddCheck(t).In(Headers{ "User-Agent": DefaultUserAgent, "X-Auth-Key": APIKEY, "X-Auth-User": USERNAME, }).Out(Headers{ "X-Storage-Url": PROXY_URL, "X-Auth-Token": AUTH_TOKEN, }).Url("/v1.0") defer server.Finished() err := c.Authenticate() if err != nil { t.Fatal(err) } if c.StorageUrl != PROXY_URL { t.Error("Bad storage url") } if c.AuthToken != AUTH_TOKEN { t.Error("Bad auth token") } if !c.Authenticated() { t.Error("Didn't authenticate") } } func TestInternalAuthenticateDenied(t *testing.T) { server.AddCheck(t).Error(400, "Bad request") server.AddCheck(t).Error(401, "DENIED") defer server.Finished() c.UnAuthenticate() err := c.Authenticate() if err != AuthorizationFailed { t.Fatal("Expecting AuthorizationFailed", err) } // FIXME // if c.Authenticated() { // t.Fatal("Expecting not authenticated") // } } func TestInternalAuthenticateBad(t *testing.T) { server.AddCheck(t).Out(Headers{ "X-Storage-Url": PROXY_URL, }) defer server.Finished() err := c.Authenticate() checkError(t, err, 0, "Response didn't have storage url and auth token") if c.Authenticated() { t.Fatal("Expecting not authenticated") } server.AddCheck(t).Out(Headers{ "X-Auth-Token": AUTH_TOKEN, }) err = c.Authenticate() checkError(t, err, 0, "Response didn't have storage url and auth token") if c.Authenticated() { t.Fatal("Expecting not authenticated") } server.AddCheck(t) err = c.Authenticate() checkError(t, err, 0, "Response didn't have storage url and auth token") if c.Authenticated() { t.Fatal("Expecting not authenticated") } server.AddCheck(t).Out(Headers{ "X-Storage-Url": PROXY_URL, "X-Auth-Token": AUTH_TOKEN, }) err = c.Authenticate() if err != nil { t.Fatal(err) } if !c.Authenticated() { t.Fatal("Expecting authenticated") } } func testContainerNames(t *testing.T, rx string, expected []string) { server.AddCheck(t).In(Headers{ "User-Agent": DefaultUserAgent, "X-Auth-Token": AUTH_TOKEN, }).Tx(rx).Url("/proxy") containers, err := c.ContainerNames(nil) if err != nil { t.Fatal(err) } if len(containers) != len(expected) { t.Fatal("Wrong number of containers", len(containers), rx, len(expected), expected) } for i := range containers { if containers[i] != expected[i] { t.Error("Bad container", containers[i], expected[i]) } } } func TestInternalContainerNames(t *testing.T) { defer server.Finished() testContainerNames(t, "", []string{}) testContainerNames(t, "one", []string{"one"}) testContainerNames(t, "one\n", []string{"one"}) testContainerNames(t, "one\ntwo\nthree\n", []string{"one", "two", "three"}) } func TestInternalObjectPutBytes(t *testing.T) { server.AddCheck(t).In(Headers{ "User-Agent": DefaultUserAgent, "X-Auth-Token": AUTH_TOKEN, "Content-Length": "5", "Content-Type": "text/plain", }).Rx("12345") defer server.Finished() c.ObjectPutBytes("container", "object", []byte{'1', '2', '3', '4', '5'}, "text/plain") } func TestInternalObjectPutString(t *testing.T) { server.AddCheck(t).In(Headers{ "User-Agent": DefaultUserAgent, "X-Auth-Token": AUTH_TOKEN, "Content-Length": "5", "Content-Type": "text/plain", }).Rx("12345") defer server.Finished() c.ObjectPutString("container", "object", "12345", "text/plain") } func TestSetFromEnv(t *testing.T) { // String s := "" os.Setenv("POTATO", "") err := setFromEnv(&s, "POTATO") if err != nil { t.Fatal(err) } os.Setenv("POTATO", "this is a test") err = setFromEnv(&s, "POTATO") if err != nil { t.Fatal(err) } if s != "this is a test" { t.Fatal("incorrect", s) } os.Setenv("POTATO", "new") err = setFromEnv(&s, "POTATO") if err != nil { t.Fatal(err) } if s != "this is a test" { t.Fatal("was reset when it shouldn't have been") } // Integer i := 0 os.Setenv("POTATO", "42") err = setFromEnv(&i, "POTATO") if err != nil { t.Fatal(err) } if i != 42 { t.Fatal("incorrect", i) } os.Setenv("POTATO", "43") err = setFromEnv(&i, "POTATO") if err != nil { t.Fatal(err) } if i != 42 { t.Fatal("was reset when it shouldn't have been") } i = 0 os.Setenv("POTATO", "not a number") err = setFromEnv(&i, "POTATO") if err == nil { t.Fatal("expecting error but didn't get one") } // bool var b bool os.Setenv("POTATO", "1") err = setFromEnv(&b, "POTATO") if err != nil { t.Fatal(err) } if b != true { t.Fatal("incorrect", b) } // time.Duration var dt time.Duration os.Setenv("POTATO", "5s") err = setFromEnv(&dt, "POTATO") if err != nil { t.Fatal(err) } if dt != 5*time.Second { t.Fatal("incorrect", dt) } // EndpointType var e EndpointType os.Setenv("POTATO", "internal") err = setFromEnv(&e, "POTATO") if err != nil { t.Fatal(err) } if e != EndpointType("internal") { t.Fatal("incorrect", e) } // Unknown var unknown struct{} err = setFromEnv(&unknown, "POTATO") if err == nil { t.Fatal("expecting error") } os.Setenv("POTATO", "") } func TestApplyEnvironment(t *testing.T) { // We've tested all the setting logic above, so just do a quick test here c := new(Connection) os.Setenv("GOSWIFT_CONNECT_TIMEOUT", "100s") err := c.ApplyEnvironment() if err != nil { t.Fatal(err) } if c.ConnectTimeout != 100*time.Second { t.Fatal("timeout incorrect", c.ConnectTimeout) } c.ConnectTimeout = 0 os.Setenv("GOSWIFT_CONNECT_TIMEOUT", "parse error") err = c.ApplyEnvironment() if err == nil { t.Fatal("expecting error") } if c.ConnectTimeout != 0 { t.Fatal("timeout incorrect", c.ConnectTimeout) } os.Setenv("GOSWIFT_CONNECT_TIMEOUT", "") } func TestApplyEnvironmentAll(t *testing.T) { // we do this in two phases because some of the variable set the same thing for phase := 1; phase <= 2; phase++ { c := new(Connection) items := []struct { phase int result interface{} name string value string want interface{} oldValue string }{ // Copied and amended from ApplyEnvironment // Environment variables - keep in same order as Connection {1, &c.Domain, "OS_USER_DOMAIN_NAME", "os_user_domain_name", "os_user_domain_name", ""}, {1, &c.DomainId, "OS_USER_DOMAIN_ID", "os_user_domain_id", "os_user_domain_id", ""}, {1, &c.UserName, "OS_USERNAME", "os_username", "os_username", ""}, {1, &c.UserId, "OS_USER_ID", "os_user_id", "os_user_id", ""}, {1, &c.ApiKey, "OS_PASSWORD", "os_password", "os_password", ""}, {1, &c.AuthUrl, "OS_AUTH_URL", "os_auth_url", "os_auth_url", ""}, {1, &c.Retries, "GOSWIFT_RETRIES", "4", 4, ""}, {1, &c.UserAgent, "GOSWIFT_USER_AGENT", "goswift_user_agent", "goswift_user_agent", ""}, {1, &c.ConnectTimeout, "GOSWIFT_CONNECT_TIMEOUT", "98s", 98 * time.Second, ""}, {1, &c.Timeout, "GOSWIFT_TIMEOUT", "99s", 99 * time.Second, ""}, {1, &c.Region, "OS_REGION_NAME", "os_region_name", "os_region_name", ""}, {1, &c.AuthVersion, "ST_AUTH_VERSION", "3", 3, ""}, {1, &c.Internal, "GOSWIFT_INTERNAL", "true", true, ""}, {1, &c.Tenant, "OS_TENANT_NAME", "os_tenant_name", "os_tenant_name", ""}, {2, &c.Tenant, "OS_PROJECT_NAME", "os_project_name", "os_project_name", ""}, {1, &c.TenantId, "OS_TENANT_ID", "os_tenant_id", "os_tenant_id", ""}, {1, &c.EndpointType, "OS_ENDPOINT_TYPE", "internal", EndpointTypeInternal, ""}, {1, &c.TenantDomain, "OS_PROJECT_DOMAIN_NAME", "os_project_domain_name", "os_project_domain_name", ""}, {1, &c.TenantDomainId, "OS_PROJECT_DOMAIN_ID", "os_project_domain_id", "os_project_domain_id", ""}, {1, &c.TrustId, "OS_TRUST_ID", "os_trust_id", "os_trust_id", ""}, {1, &c.StorageUrl, "OS_STORAGE_URL", "os_storage_url", "os_storage_url", ""}, {1, &c.AuthToken, "OS_AUTH_TOKEN", "os_auth_token", "os_auth_token", ""}, // v1 auth alternatives {2, &c.ApiKey, "ST_KEY", "st_key", "st_key", ""}, {2, &c.UserName, "ST_USER", "st_user", "st_user", ""}, {2, &c.AuthUrl, "ST_AUTH", "st_auth", "st_auth", ""}, } for i := range items { item := &items[i] if item.phase == phase { item.oldValue = os.Getenv(item.name) // save old value os.Setenv(item.name, item.value) // set new value } } err := c.ApplyEnvironment() if err != nil { t.Fatalf("unexpected error %v", err) } for i := range items { item := &items[i] if item.phase == phase { got := reflect.Indirect(reflect.ValueOf(item.result)).Interface() if !reflect.DeepEqual(item.want, got) { t.Errorf("%s: %v != %v", item.name, item.want, got) } os.Setenv(item.name, item.oldValue) // restore old value } } } } swift-1.0.52/swift_test.go000066400000000000000000002221341365550407400154720ustar00rootroot00000000000000// This tests the swift packagae // // It can be used with a real swift server which should be set up in // the environment variables SWIFT_API_USER, SWIFT_API_KEY and // SWIFT_AUTH_URL // In case those variables are not defined, a fake Swift server // is used instead - see Testing in README.md for more info // // The functions are designed to run in order and create things the // next function tests. This means that if it goes wrong it is likely // errors will propagate. You may need to tidy up the CONTAINER to // get it to run cleanly. package swift_test import ( "archive/tar" "bytes" "crypto/md5" "crypto/rand" "crypto/tls" "encoding/json" "encoding/xml" "fmt" "io" "io/ioutil" "net/http" "net/http/httptest" "os" "reflect" "strconv" "strings" "sync" "testing" "time" "github.com/ncw/swift" "github.com/ncw/swift/swifttest" ) var ( srv *swifttest.SwiftServer m1 = swift.Metadata{"Hello": "1", "potato-Salad": "2"} m2 = swift.Metadata{"hello": "", "potato-salad": ""} skipVersionTests = false ) const ( CONTAINER = "GoSwiftUnitTest" SEGMENTS_CONTAINER = "GoSwiftUnitTest_segments" VERSIONS_CONTAINER = "GoSwiftUnitTestVersions" CURRENT_CONTAINER = "GoSwiftUnitTestCurrent" OBJECT = "test_object" OBJECT2 = "test_object2" SYMLINK_OBJECT = "test_symlink" SYMLINK_OBJECT2 = "test_symlink2" EMPTYOBJECT = "empty_test_object" CONTENTS = "12345" CONTENTS2 = "54321" CONTENT_SIZE = int64(len(CONTENTS)) CONTENT_MD5 = "827ccb0eea8a706c4c34a16891f84e7b" CONTENT2_MD5 = "01cfcd4f6b8770febfb40cb906715822" EMPTY_MD5 = "d41d8cd98f00b204e9800998ecf8427e" SECRET_KEY = "b3968d0207b54ece87cccc06515a89d4" ) type someTransport struct{ http.Transport } func makeConnection(t *testing.T) (*swift.Connection, func()) { var err error UserName := os.Getenv("SWIFT_API_USER") ApiKey := os.Getenv("SWIFT_API_KEY") AuthUrl := os.Getenv("SWIFT_AUTH_URL") Region := os.Getenv("SWIFT_REGION_NAME") EndpointType := os.Getenv("SWIFT_ENDPOINT_TYPE") Insecure := os.Getenv("SWIFT_AUTH_INSECURE") ConnectionChannelTimeout := os.Getenv("SWIFT_CONNECTION_CHANNEL_TIMEOUT") DataChannelTimeout := os.Getenv("SWIFT_DATA_CHANNEL_TIMEOUT") internalServer := false if UserName == "" || ApiKey == "" || AuthUrl == "" { srv, err = swifttest.NewSwiftServer("localhost") if err != nil && t != nil { t.Fatal("Failed to create server", err) } UserName = "swifttest" ApiKey = "swifttest" AuthUrl = srv.AuthURL internalServer = true } transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, MaxIdleConnsPerHost: 2048, } if Insecure == "1" { transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} } swift.SetExpectContinueTimeout(transport, 5*time.Second) c := swift.Connection{ UserName: UserName, ApiKey: ApiKey, AuthUrl: AuthUrl, Region: Region, Transport: transport, ConnectTimeout: 60 * time.Second, Timeout: 60 * time.Second, EndpointType: swift.EndpointType(EndpointType), } if !internalServer { if isV3Api() { c.Tenant = os.Getenv("SWIFT_TENANT") c.Domain = os.Getenv("SWIFT_API_DOMAIN") } else { c.Tenant = os.Getenv("SWIFT_TENANT") c.TenantId = os.Getenv("SWIFT_TENANT_ID") } } var timeout int64 if ConnectionChannelTimeout != "" { timeout, err = strconv.ParseInt(ConnectionChannelTimeout, 10, 32) if err == nil { c.ConnectTimeout = time.Duration(timeout) * time.Second } } if DataChannelTimeout != "" { timeout, err = strconv.ParseInt(DataChannelTimeout, 10, 32) if err == nil { c.Timeout = time.Duration(timeout) * time.Second } } return &c, func() { if srv != nil { srv.Close() } } } func makeConnectionAuth(t *testing.T) (*swift.Connection, func()) { c, rollback := makeConnection(t) err := c.Authenticate() if err != nil { t.Fatal("Auth failed", err) } return c, rollback } func makeConnectionWithContainer(t *testing.T) (*swift.Connection, func()) { c, rollback := makeConnectionAuth(t) err := c.ContainerCreate(CONTAINER, m1.ContainerHeaders()) if err != nil { t.Fatal(err) } return c, func() { c.ContainerDelete(CONTAINER) rollback() } } func makeConnectionWithObject(t *testing.T) (*swift.Connection, func()) { c, rollback := makeConnectionWithContainer(t) err := c.ObjectPutString(CONTAINER, OBJECT, CONTENTS, "") if err != nil { t.Fatal(err) } return c, func() { c.ObjectDelete(CONTAINER, OBJECT) rollback() } } func makeConnectionWithObjectHeaders(t *testing.T) (*swift.Connection, func()) { c, rollback := makeConnectionWithObject(t) err := c.ObjectUpdate(CONTAINER, OBJECT, m1.ObjectHeaders()) if err != nil { t.Fatal(err) } return c, rollback } func makeConnectionWithVersionsContainer(t *testing.T) (*swift.Connection, func()) { c, rollback := makeConnectionAuth(t) err := c.VersionContainerCreate(CURRENT_CONTAINER, VERSIONS_CONTAINER) newRollback := func() { c.ContainerDelete(CURRENT_CONTAINER) c.ContainerDelete(VERSIONS_CONTAINER) rollback() } if err != nil { if err == swift.Forbidden { skipVersionTests = true return c, newRollback } t.Fatal(err) } return c, newRollback } func makeConnectionWithVersionsObject(t *testing.T) (*swift.Connection, func()) { c, rollback := makeConnectionWithVersionsContainer(t) if err := c.ObjectPutString(CURRENT_CONTAINER, OBJECT, CONTENTS, ""); err != nil { t.Fatal(err) } // Version 2 if err := c.ObjectPutString(CURRENT_CONTAINER, OBJECT, CONTENTS2, ""); err != nil { t.Fatal(err) } // Version 3 if err := c.ObjectPutString(CURRENT_CONTAINER, OBJECT, CONTENTS2, ""); err != nil { t.Fatal(err) } return c, func() { for i := 0; i < 3; i++ { c.ObjectDelete(CURRENT_CONTAINER, OBJECT) } rollback() } } func makeConnectionWithSegmentsContainer(t *testing.T) (*swift.Connection, func()) { c, rollback := makeConnectionWithContainer(t) err := c.ContainerCreate(SEGMENTS_CONTAINER, swift.Headers{}) if err != nil { t.Fatal(err) } return c, func() { err = c.ContainerDelete(SEGMENTS_CONTAINER) if err != nil { t.Fatal(err) } rollback() } } func makeConnectionWithDLO(t *testing.T) (*swift.Connection, func()) { c, rollback := makeConnectionWithSegmentsContainer(t) opts := swift.LargeObjectOpts{ Container: CONTAINER, ObjectName: OBJECT, ContentType: "image/jpeg", } out, err := c.DynamicLargeObjectCreate(&opts) if err != nil { t.Fatal(err) } for i := 0; i < 2; i++ { _, err = fmt.Fprintf(out, "%d %s\n", i, CONTENTS) if err != nil { t.Fatal(err) } } err = out.Close() if err != nil { t.Error(err) } return c, func() { c.DynamicLargeObjectDelete(CONTAINER, OBJECT) rollback() } } func makeConnectionWithSLO(t *testing.T) (*swift.Connection, func()) { c, rollback := makeConnectionWithSegmentsContainer(t) opts := swift.LargeObjectOpts{ Container: CONTAINER, ObjectName: OBJECT, ContentType: "image/jpeg", } out, err := c.StaticLargeObjectCreate(&opts) if err != nil { if err == swift.SLONotSupported { t.Skip("SLO not supported") return c, rollback } t.Fatal(err) } for i := 0; i < 2; i++ { _, err = fmt.Fprintf(out, "%d %s\n", i, CONTENTS) if err != nil { t.Fatal(err) } } err = out.Close() if err != nil { t.Error(err) } return c, func() { c.StaticLargeObjectDelete(CONTAINER, OBJECT) rollback() } } func isV3Api() bool { AuthUrl := os.Getenv("SWIFT_AUTH_URL") return strings.Contains(AuthUrl, "v3") } func getSwinftInfo(t *testing.T) (info swift.SwiftInfo, err error) { c, rollback := makeConnectionAuth(t) defer rollback() return c.QueryInfo() } func TestTransport(t *testing.T) { c, rollback := makeConnection(t) defer rollback() tr := &someTransport{ Transport: http.Transport{ MaxIdleConnsPerHost: 2048, }, } Insecure := os.Getenv("SWIFT_AUTH_INSECURE") if Insecure == "1" { tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} } c.Transport = tr err := c.Authenticate() if err != nil { t.Fatal("Auth failed", err) } if !c.Authenticated() { t.Fatal("Not authenticated") } } // The following Test functions are run in order - this one must come before the others! func TestV1V2Authenticate(t *testing.T) { if isV3Api() { return } c, rollback := makeConnection(t) defer rollback() err := c.Authenticate() if err != nil { t.Fatal("Auth failed", err) } if !c.Authenticated() { t.Fatal("Not authenticated") } } func TestV3AuthenticateWithDomainNameAndTenantId(t *testing.T) { if !isV3Api() { return } c, rollback := makeConnection(t) defer rollback() c.Tenant = "" c.Domain = os.Getenv("SWIFT_API_DOMAIN") c.TenantId = os.Getenv("SWIFT_TENANT_ID") c.DomainId = "" err := c.Authenticate() if err != nil { t.Fatal("Auth failed", err) } if !c.Authenticated() { t.Fatal("Not authenticated") } } func TestV3TrustWithTrustId(t *testing.T) { if !isV3Api() { return } c, rollback := makeConnection(t) defer rollback() c.TrustId = os.Getenv("SWIFT_TRUST_ID") err := c.Authenticate() if err != nil { t.Fatal("Auth failed", err) } if !c.Authenticated() { t.Fatal("Not authenticated") } } func TestV3AuthenticateWithDomainIdAndTenantId(t *testing.T) { if !isV3Api() { return } c, rollback := makeConnection(t) defer rollback() c.Tenant = "" c.Domain = "" c.TenantId = os.Getenv("SWIFT_TENANT_ID") c.DomainId = os.Getenv("SWIFT_API_DOMAIN_ID") err := c.Authenticate() if err != nil { t.Fatal("Auth failed", err) } if !c.Authenticated() { t.Fatal("Not authenticated") } } func TestV3AuthenticateWithDomainNameAndTenantName(t *testing.T) { if !isV3Api() { return } c, rollback := makeConnection(t) defer rollback() c.Tenant = os.Getenv("SWIFT_TENANT") c.Domain = os.Getenv("SWIFT_API_DOMAIN") c.TenantId = "" c.DomainId = "" err := c.Authenticate() if err != nil { t.Fatal("Auth failed", err) } if !c.Authenticated() { t.Fatal("Not authenticated") } } func TestV3AuthenticateWithDomainIdAndTenantName(t *testing.T) { if !isV3Api() { return } c, rollback := makeConnection(t) defer rollback() c.Tenant = os.Getenv("SWIFT_TENANT") c.Domain = "" c.TenantId = "" c.DomainId = os.Getenv("SWIFT_API_DOMAIN_ID") err := c.Authenticate() if err != nil { t.Fatal("Auth failed", err) } if !c.Authenticated() { t.Fatal("Not authenticated") } } // Attempt to trigger a race in authenticate // // Run with -race to test func TestAuthenticateRace(t *testing.T) { c, rollback := makeConnection(t) defer rollback() var wg sync.WaitGroup for i := 0; i < 10; i++ { wg.Add(1) go func() { defer wg.Done() err := c.Authenticate() if err != nil { t.Fatal("Auth failed", err) } if !c.Authenticated() { t.Fatal("Not authenticated") } }() } wg.Wait() } // Test a connection can be serialized and unserialized with JSON func TestSerializeConnectionJson(t *testing.T) { c, rollback := makeConnectionAuth(t) defer rollback() serializedConnection, err := json.Marshal(c) if err != nil { t.Fatalf("Failed to serialize connection: %v", err) } c2 := new(swift.Connection) err = json.Unmarshal(serializedConnection, &c2) if err != nil { t.Fatalf("Failed to unserialize connection: %v", err) } if !c2.Authenticated() { t.Fatal("Should be authenticated") } _, _, err = c2.Account() if err != nil { t.Fatalf("Failed to use unserialized connection: %v", err) } } // Test a connection can be serialized and unserialized with XML func TestSerializeConnectionXml(t *testing.T) { c, rollback := makeConnectionAuth(t) defer rollback() serializedConnection, err := xml.Marshal(c) if err != nil { t.Fatalf("Failed to serialize connection: %v", err) } c2 := new(swift.Connection) err = xml.Unmarshal(serializedConnection, &c2) if err != nil { t.Fatalf("Failed to unserialize connection: %v", err) } if !c2.Authenticated() { t.Fatal("Should be authenticated") } _, _, err = c2.Account() if err != nil { t.Fatalf("Failed to use unserialized connection: %v", err) } } // Test the reauthentication logic func TestOnReAuth(t *testing.T) { c, rollback := makeConnectionAuth(t) defer rollback() c.UnAuthenticate() _, _, err := c.Account() if err != nil { t.Fatalf("Failed to reauthenticate: %v", err) } } func TestAccount(t *testing.T) { c, rollback := makeConnectionAuth(t) defer rollback() info, headers, err := c.Account() if err != nil { t.Fatal(err) } if headers["X-Account-Container-Count"] != fmt.Sprintf("%d", info.Containers) { t.Error("Bad container count") } if headers["X-Account-Bytes-Used"] != fmt.Sprintf("%d", info.BytesUsed) { t.Error("Bad bytes count") } if headers["X-Account-Object-Count"] != fmt.Sprintf("%d", info.Objects) { t.Error("Bad objects count") } } func compareMaps(t *testing.T, a, b map[string]string) { if len(a) != len(b) { t.Error("Maps different sizes", a, b) } for ka, va := range a { if vb, ok := b[ka]; !ok || va != vb { t.Error("Difference in key", ka, va, b[ka]) } } for kb, vb := range b { if va, ok := a[kb]; !ok || vb != va { t.Error("Difference in key", kb, vb, a[kb]) } } } func TestAccountUpdate(t *testing.T) { c, rollback := makeConnectionAuth(t) defer rollback() err := c.AccountUpdate(m1.AccountHeaders()) if err != nil { t.Fatal(err) } _, headers, err := c.Account() if err != nil { t.Fatal(err) } m := headers.AccountMetadata() delete(m, "temp-url-key") // remove X-Account-Meta-Temp-URL-Key if set compareMaps(t, m, map[string]string{"hello": "1", "potato-salad": "2"}) err = c.AccountUpdate(m2.AccountHeaders()) if err != nil { t.Fatal(err) } _, headers, err = c.Account() if err != nil { t.Fatal(err) } m = headers.AccountMetadata() delete(m, "temp-url-key") // remove X-Account-Meta-Temp-URL-Key if set compareMaps(t, m, map[string]string{}) } func TestContainerCreate(t *testing.T) { c, rollback := makeConnectionAuth(t) defer rollback() err := c.ContainerCreate(CONTAINER, m1.ContainerHeaders()) if err != nil { t.Fatal(err) } err = c.ContainerDelete(CONTAINER) if err != nil { t.Fatal(err) } } func TestContainer(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() info, headers, err := c.Container(CONTAINER) if err != nil { t.Fatal(err) } compareMaps(t, headers.ContainerMetadata(), map[string]string{"hello": "1", "potato-salad": "2"}) if CONTAINER != info.Name { t.Error("Bad container count") } if headers["X-Container-Bytes-Used"] != fmt.Sprintf("%d", info.Bytes) { t.Error("Bad bytes count") } if headers["X-Container-Object-Count"] != fmt.Sprintf("%d", info.Count) { t.Error("Bad objects count") } } func TestContainersAll(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() containers1, err := c.ContainersAll(nil) if err != nil { t.Fatal(err) } containers2, err := c.Containers(nil) if err != nil { t.Fatal(err) } if len(containers1) != len(containers2) { t.Fatal("Wrong length") } for i := range containers1 { if containers1[i] != containers2[i] { t.Fatal("Not the same") } } } func TestContainersAllWithLimit(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() containers1, err := c.ContainersAll(&swift.ContainersOpts{Limit: 1}) if err != nil { t.Fatal(err) } containers2, err := c.Containers(nil) if err != nil { t.Fatal(err) } if len(containers1) != len(containers2) { t.Fatal("Wrong length") } for i := range containers1 { if containers1[i] != containers2[i] { t.Fatal("Not the same") } } } func TestContainerUpdate(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() err := c.ContainerUpdate(CONTAINER, m2.ContainerHeaders()) if err != nil { t.Fatal(err) } _, headers, err := c.Container(CONTAINER) if err != nil { t.Fatal(err) } compareMaps(t, headers.ContainerMetadata(), map[string]string{}) } func TestContainerNames(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() containers, err := c.ContainerNames(nil) if err != nil { t.Fatal(err) } ok := false for _, container := range containers { if container == CONTAINER { ok = true break } } if !ok { t.Errorf("Didn't find container %q in listing %q", CONTAINER, containers) } } func TestContainerNamesAll(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() containers1, err := c.ContainerNamesAll(nil) if err != nil { t.Fatal(err) } containers2, err := c.ContainerNames(nil) if err != nil { t.Fatal(err) } if len(containers1) != len(containers2) { t.Fatal("Wrong length") } for i := range containers1 { if containers1[i] != containers2[i] { t.Fatal("Not the same") } } } func TestContainerNamesAllWithLimit(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() containers1, err := c.ContainerNamesAll(&swift.ContainersOpts{Limit: 1}) if err != nil { t.Fatal(err) } containers2, err := c.ContainerNames(nil) if err != nil { t.Fatal(err) } if len(containers1) != len(containers2) { t.Fatal("Wrong length") } for i := range containers1 { if containers1[i] != containers2[i] { t.Fatal("Not the same") } } } func TestObjectPutString(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() err := c.ObjectPutString(CONTAINER, OBJECT, CONTENTS, "") if err != nil { t.Fatal(err) } defer func() { err = c.ObjectDelete(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } }() info, _, err := c.Object(CONTAINER, OBJECT) if err != nil { t.Error(err) } if info.ContentType != "application/octet-stream" { t.Error("Bad content type", info.ContentType) } if info.Bytes != CONTENT_SIZE { t.Error("Bad length") } if info.Hash != CONTENT_MD5 { t.Error("Bad length") } } func TestObjectPut(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() headers := swift.Headers{} // Set content size incorrectly - should produce an error headers["Content-Length"] = strconv.FormatInt(CONTENT_SIZE-1, 10) contents := bytes.NewBufferString(CONTENTS) h, err := c.ObjectPut(CONTAINER, OBJECT, contents, true, CONTENT_MD5, "text/plain", headers) if err == nil { t.Fatal("Expecting error but didn't get one") } // Now set content size correctly contents = bytes.NewBufferString(CONTENTS) headers["Content-Length"] = strconv.FormatInt(CONTENT_SIZE, 10) h, err = c.ObjectPut(CONTAINER, OBJECT, contents, true, CONTENT_MD5, "text/plain", headers) if err != nil { t.Fatal(err) } defer func() { err = c.ObjectDelete(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } }() if h["Etag"] != CONTENT_MD5 { t.Errorf("Bad Etag want %q got %q", CONTENT_MD5, h["Etag"]) } // Fetch object info and compare info, _, err := c.Object(CONTAINER, OBJECT) if err != nil { t.Error(err) } if info.ContentType != "text/plain" { t.Error("Bad content type", info.ContentType) } if info.Bytes != CONTENT_SIZE { t.Error("Bad length") } if info.Hash != CONTENT_MD5 { t.Error("Bad length") } } func TestObjectPutWithReauth(t *testing.T) { if !swift.IS_AT_LEAST_GO_16 { return } c, rollback := makeConnectionWithContainer(t) defer rollback() // Simulate that our auth token expired c.AuthToken = "expiredtoken" r := strings.NewReader(CONTENTS) _, err := c.ObjectPut(CONTAINER, OBJECT, r, true, "", "text/plain", nil) if err != nil { t.Fatal(err) } info, _, err := c.Object(CONTAINER, OBJECT) if err != nil { t.Error(err) } if info.ContentType != "text/plain" { t.Error("Bad content type", info.ContentType) } if info.Bytes != CONTENT_SIZE { t.Error("Bad length") } if info.Hash != CONTENT_MD5 { t.Error("Bad length") } } func TestObjectPutStringWithReauth(t *testing.T) { if !swift.IS_AT_LEAST_GO_16 { return } c, rollback := makeConnectionWithContainer(t) defer rollback() // Simulate that our auth token expired c.AuthToken = "expiredtoken" err := c.ObjectPutString(CONTAINER, OBJECT, CONTENTS, "") if err != nil { t.Fatal(err) } info, _, err := c.Object(CONTAINER, OBJECT) if err != nil { t.Error(err) } if info.ContentType != "application/octet-stream" { t.Error("Bad content type", info.ContentType) } if info.Bytes != CONTENT_SIZE { t.Error("Bad length") } if info.Hash != CONTENT_MD5 { t.Error("Bad length") } } func TestObjectEmpty(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() err := c.ObjectPutString(CONTAINER, EMPTYOBJECT, "", "") if err != nil { t.Fatal(err) } defer func() { err = c.ObjectDelete(CONTAINER, EMPTYOBJECT) if err != nil { t.Error(err) } }() info, _, err := c.Object(CONTAINER, EMPTYOBJECT) if err != nil { t.Error(err) } if info.ContentType != "application/octet-stream" { t.Error("Bad content type", info.ContentType) } if info.Bytes != 0 { t.Errorf("Bad length want 0 got %v", info.Bytes) } if info.Hash != EMPTY_MD5 { t.Errorf("Bad MD5 want %v got %v", EMPTY_MD5, info.Hash) } } func TestSymlinkObject(t *testing.T) { info, err := getSwinftInfo(t) if err != nil { t.Fatal(err) } if _, ok := info["symlink"]; !ok { // skip, symlink not supported t.Skip("skip, symlink not supported") return } c, rollback := makeConnectionWithContainer(t) defer rollback() // write target objects err = c.ObjectPutBytes(CONTAINER, OBJECT, []byte(CONTENTS), "text/potato") if err != nil { t.Fatal(err) } defer func() { err = c.ObjectDelete(CONTAINER, OBJECT) if err != nil { t.Error(err) } }() // test dynamic link _, err = c.ObjectSymlinkCreate(CONTAINER, SYMLINK_OBJECT, "", CONTAINER, OBJECT, "") if err != nil { t.Fatal(err) } defer func() { err = c.ObjectDelete(CONTAINER, SYMLINK_OBJECT) if err != nil { t.Error(err) } }() md, _, err := c.Object(CONTAINER, SYMLINK_OBJECT) if err != nil { t.Error(err) } if md.ContentType != "text/potato" { t.Error("Bad content type", md.ContentType) } if md.Bytes != CONTENT_SIZE { t.Errorf("Bad length want 5 got %v", md.Bytes) } if md.Hash != CONTENT_MD5 { t.Errorf("Bad MD5 want %v got %v", CONTENT_MD5, md.Hash) } } func TestStaticSymlinkObject(t *testing.T) { info, err := getSwinftInfo(t) if err != nil { t.Fatal(err) } if sym, ok := info["symlink"].(map[string]interface{}); ok { if _, ok := sym["static_links"]; !ok { t.Skip("skip, static symlink not supported") return } } else { t.Skip("skip, symlink not supported") return } c, rollback := makeConnectionWithContainer(t) defer rollback() // write target objects err = c.ObjectPutBytes(CONTAINER, OBJECT2, []byte(CONTENTS2), "text/tomato") if err != nil { t.Fatal(err) } defer func() { err = c.ObjectDelete(CONTAINER, OBJECT2) if err != nil { t.Error(err) } }() // test static link // first with the wrong target etag _, err = c.ObjectSymlinkCreate(CONTAINER, SYMLINK_OBJECT2, "", CONTAINER, OBJECT2, CONTENT_MD5) if err == nil { t.Error("Symlink with wrong target etag should have failed") } _, err = c.ObjectSymlinkCreate(CONTAINER, SYMLINK_OBJECT2, "", CONTAINER, OBJECT2, CONTENT2_MD5) if err != nil { t.Fatal(err) } defer func() { err = c.ObjectDelete(CONTAINER, SYMLINK_OBJECT2) if err != nil { t.Error(err) } }() md, _, err := c.Object(CONTAINER, SYMLINK_OBJECT2) if err != nil { t.Error(err) } if md.ContentType != "text/tomato" { t.Error("Bad content type", md.ContentType) } if md.Bytes != CONTENT_SIZE { t.Errorf("Bad length want 5 got %v", md.Bytes) } if md.Hash != CONTENT2_MD5 { t.Errorf("Bad MD5 want %v got %v", CONTENT2_MD5, md.Hash) } } func TestObjectPutBytes(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() err := c.ObjectPutBytes(CONTAINER, OBJECT, []byte(CONTENTS), "") if err != nil { t.Fatal(err) } defer func() { err = c.ObjectDelete(CONTAINER, OBJECT) if err != nil { t.Error(err) } }() info, _, err := c.Object(CONTAINER, OBJECT) if err != nil { t.Error(err) } if info.ContentType != "application/octet-stream" { t.Error("Bad content type", info.ContentType) } if info.Bytes != CONTENT_SIZE { t.Error("Bad length") } if info.Hash != CONTENT_MD5 { t.Error("Bad length") } } func TestObjectPutMimeType(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() err := c.ObjectPutString(CONTAINER, "test.jpg", CONTENTS, "") if err != nil { t.Fatal(err) } defer func() { err = c.ObjectDelete(CONTAINER, "test.jpg") if err != nil { t.Error(err) } }() info, _, err := c.Object(CONTAINER, "test.jpg") if err != nil { t.Error(err) } if info.ContentType != "image/jpeg" { t.Error("Bad content type", info.ContentType) } } func TestObjectCreate(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() out, err := c.ObjectCreate(CONTAINER, OBJECT2, true, "", "", nil) if err != nil { t.Fatal(err) } defer func() { err = c.ObjectDelete(CONTAINER, OBJECT2) if err != nil { t.Error(err) } }() buf := &bytes.Buffer{} hash := md5.New() out2 := io.MultiWriter(out, buf, hash) for i := 0; i < 100; i++ { fmt.Fprintf(out2, "%d %s\n", i, CONTENTS) } // Ensure Headers fails if called prematurely _, err = out.Headers() if err == nil { t.Error("Headers should fail if called before Close()") } err = out.Close() if err != nil { t.Error(err) } expected := buf.String() contents, err := c.ObjectGetString(CONTAINER, OBJECT2) if err != nil { t.Error(err) } if contents != expected { t.Errorf("Contents wrong, expected %q, got: %q", expected, contents) } // Ensure Headers succeeds when called after a good upload headers, err := out.Headers() if err != nil { t.Error(err) } if len(headers) < 1 { t.Error("The Headers returned by Headers() should not be empty") } // Test writing on closed file n, err := out.Write([]byte{0}) if err == nil || n != 0 { t.Error("Expecting error and n == 0 writing on closed file", err, n) } // Now with hash instead out, err = c.ObjectCreate(CONTAINER, OBJECT2, false, fmt.Sprintf("%x", hash.Sum(nil)), "", nil) if err != nil { t.Fatal(err) } _, err = out.Write(buf.Bytes()) if err != nil { t.Error(err) } err = out.Close() if err != nil { t.Error(err) } contents, err = c.ObjectGetString(CONTAINER, OBJECT2) if err != nil { t.Error(err) } if contents != expected { t.Errorf("Contents wrong, expected %q, got: %q", expected, contents) } // Now with bad hash out, err = c.ObjectCreate(CONTAINER, OBJECT2, false, CONTENT_MD5, "", nil) if err != nil { t.Fatal(err) } // FIXME: work around bug which produces 503 not 422 for empty corrupted files fmt.Fprintf(out, "Sausage") err = out.Close() if err != swift.ObjectCorrupted { t.Error("Expecting object corrupted not", err) } } func TestObjectCreateAbort(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() out, err := c.ObjectCreate(CONTAINER, OBJECT2, true, "", "", nil) if err != nil { t.Fatal(err) } defer func() { _ = c.ObjectDelete(CONTAINER, OBJECT2) // Ignore error }() expectedContents := "foo" _, err = out.Write([]byte(expectedContents)) if err != nil { t.Error(err) } errAbort := fmt.Errorf("abort") err = out.CloseWithError(errAbort) if err != nil { t.Errorf("Unexpected error %#v", err) } _, err = c.ObjectGetString(CONTAINER, OBJECT2) if err != swift.ObjectNotFound { t.Errorf("Unexpected error: %#v", err) } } func TestObjectGetString(t *testing.T) { c, rollback := makeConnectionWithObject(t) defer rollback() contents, err := c.ObjectGetString(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } if contents != CONTENTS { t.Error("Contents wrong") } } func TestObjectGetBytes(t *testing.T) { c, rollback := makeConnectionWithObject(t) defer rollback() contents, err := c.ObjectGetBytes(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } if string(contents) != CONTENTS { t.Error("Contents wrong") } } func TestObjectOpen(t *testing.T) { c, rollback := makeConnectionWithObject(t) defer rollback() file, _, err := c.ObjectOpen(CONTAINER, OBJECT, true, nil) if err != nil { t.Fatal(err) } var buf bytes.Buffer n, err := io.Copy(&buf, file) if err != nil { t.Fatal(err) } if n != CONTENT_SIZE { t.Fatal("Wrong length", n, CONTENT_SIZE) } if buf.String() != CONTENTS { t.Error("Contents wrong") } err = file.Close() if err != nil { t.Fatal(err) } } func TestObjectOpenPartial(t *testing.T) { c, rollback := makeConnectionWithObject(t) defer rollback() file, _, err := c.ObjectOpen(CONTAINER, OBJECT, true, nil) if err != nil { t.Fatal(err) } var buf bytes.Buffer n, err := io.CopyN(&buf, file, 1) if err != nil { t.Fatal(err) } if n != 1 { t.Fatal("Wrong length", n, CONTENT_SIZE) } if buf.String() != CONTENTS[:1] { t.Error("Contents wrong") } err = file.Close() if err != nil { t.Fatal(err) } } func TestObjectOpenLength(t *testing.T) { c, rollback := makeConnectionWithObject(t) defer rollback() file, _, err := c.ObjectOpen(CONTAINER, OBJECT, true, nil) if err != nil { t.Fatal(err) } // FIXME ideally this would check both branches of the Length() code n, err := file.Length() if err != nil { t.Fatal(err) } if n != CONTENT_SIZE { t.Fatal("Wrong length", n, CONTENT_SIZE) } err = file.Close() if err != nil { t.Fatal(err) } } func TestObjectOpenNotModified(t *testing.T) { c, rollback := makeConnectionWithObject(t) defer rollback() _, _, err := c.ObjectOpen(CONTAINER, OBJECT, true, swift.Headers{ "If-None-Match": CONTENT_MD5, }) if err != swift.NotModified { t.Fatal(err) } } func TestObjectOpenSeek(t *testing.T) { c, rollback := makeConnectionWithObject(t) defer rollback() plan := []struct { whence int offset int64 result int64 }{ {-1, 0, 0}, {-1, 0, 1}, {-1, 0, 2}, {0, 0, 0}, {0, 0, 0}, {0, 1, 1}, {0, 2, 2}, {1, 0, 3}, {1, -2, 2}, {1, 1, 4}, {2, -1, 4}, {2, -3, 2}, {2, -2, 3}, {2, -5, 0}, {2, -4, 1}, } file, _, err := c.ObjectOpen(CONTAINER, OBJECT, true, nil) if err != nil { t.Fatal(err) } for _, p := range plan { if p.whence >= 0 { var result int64 result, err = file.Seek(p.offset, p.whence) if err != nil { t.Fatal(err, p) } if result != p.result { t.Fatal("Seek result was", result, "expecting", p.result, p) } } var buf bytes.Buffer var n int64 n, err = io.CopyN(&buf, file, 1) if err != nil { t.Fatal(err, p) } if n != 1 { t.Fatal("Wrong length", n, p) } actual := buf.String() expected := CONTENTS[p.result : p.result+1] if actual != expected { t.Error("Contents wrong, expecting", expected, "got", actual, p) } } err = file.Close() if err != nil { t.Fatal(err) } } // Test seeking to the end to find the file size func TestObjectOpenSeekEnd(t *testing.T) { c, rollback := makeConnectionWithObject(t) defer rollback() file, _, err := c.ObjectOpen(CONTAINER, OBJECT, true, nil) if err != nil { t.Fatal(err) } n, err := file.Seek(0, 2) // seek to end if err != nil { t.Fatal(err) } if n != CONTENT_SIZE { t.Fatal("Wrong offset", n) } // Now check reading returns EOF buf := make([]byte, 16) nn, err := io.ReadFull(file, buf) if err != io.EOF { t.Fatal(err) } if nn != 0 { t.Fatal("wrong length", n) } // Now seek back to start and check we can read the file n, err = file.Seek(0, 0) // seek to start if err != nil { t.Fatal(err) } if n != 0 { t.Fatal("Wrong offset", n) } // read file and check contents buf, err = ioutil.ReadAll(file) if err != nil { t.Fatal(err) } if string(buf) != CONTENTS { t.Fatal("wrong contents", string(buf)) } } func TestObjectUpdate(t *testing.T) { c, rollback := makeConnectionWithObject(t) defer rollback() err := c.ObjectUpdate(CONTAINER, OBJECT, m1.ObjectHeaders()) if err != nil { t.Fatal(err) } } func checkTime(t *testing.T, when time.Time, low, high int) { dt := time.Now().Sub(when) if dt < time.Duration(low)*time.Second || dt > time.Duration(high)*time.Second { t.Errorf("Time is wrong: dt=%q, when=%q", dt, when) } } func TestObject(t *testing.T) { c, rollback := makeConnectionWithObjectHeaders(t) defer rollback() object, headers, err := c.Object(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } compareMaps(t, headers.ObjectMetadata(), map[string]string{"hello": "1", "potato-salad": "2"}) if object.Name != OBJECT || object.Bytes != CONTENT_SIZE || object.ContentType != "application/octet-stream" || object.Hash != CONTENT_MD5 || object.PseudoDirectory != false || object.SubDir != "" { t.Error("Bad object info", object) } checkTime(t, object.LastModified, -10, 10) } func TestObjectUpdate2(t *testing.T) { c, rollback := makeConnectionWithObjectHeaders(t) defer rollback() err := c.ObjectUpdate(CONTAINER, OBJECT, m2.ObjectHeaders()) if err != nil { t.Fatal(err) } _, headers, err := c.Object(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } compareMaps(t, headers.ObjectMetadata(), map[string]string{"hello": "", "potato-salad": ""}) } func TestContainers(t *testing.T) { c, rollback := makeConnectionWithObjectHeaders(t) defer rollback() containers, err := c.Containers(nil) if err != nil { t.Fatal(err) } ok := false for _, container := range containers { if container.Name == CONTAINER { ok = true // Container may or may not have the file contents in it // Swift updates may be behind if container.Count == 0 && container.Bytes == 0 { break } if container.Count == 1 && container.Bytes == CONTENT_SIZE { break } t.Errorf("Bad size of Container %q: %q", CONTAINER, container) break } } if !ok { t.Errorf("Didn't find container %q in listing %q", CONTAINER, containers) } } func TestObjectNames(t *testing.T) { c, rollback := makeConnectionWithObjectHeaders(t) defer rollback() objects, err := c.ObjectNames(CONTAINER, nil) if err != nil { t.Fatal(err) } if len(objects) != 1 || objects[0] != OBJECT { t.Error("Incorrect listing", objects) } } func TestObjectNamesAll(t *testing.T) { c, rollback := makeConnectionWithObjectHeaders(t) defer rollback() objects, err := c.ObjectNamesAll(CONTAINER, nil) if err != nil { t.Fatal(err) } if len(objects) != 1 || objects[0] != OBJECT { t.Error("Incorrect listing", objects) } } func TestObjectNamesAllWithLimit(t *testing.T) { c, rollback := makeConnectionWithObjectHeaders(t) defer rollback() objects, err := c.ObjectNamesAll(CONTAINER, &swift.ObjectsOpts{Limit: 1}) if err != nil { t.Fatal(err) } if len(objects) != 1 || objects[0] != OBJECT { t.Error("Incorrect listing", objects) } } func TestObjectsWalk(t *testing.T) { c, rollback := makeConnectionWithObjectHeaders(t) defer rollback() objects := make([]string, 0) err := c.ObjectsWalk(container, nil, func(opts *swift.ObjectsOpts) (interface{}, error) { newObjects, err := c.ObjectNames(CONTAINER, opts) if err == nil { objects = append(objects, newObjects...) } return newObjects, err }) if err != nil { t.Fatal(err) } if len(objects) != 1 || objects[0] != OBJECT { t.Error("Incorrect listing", objects) } } func TestObjects(t *testing.T) { c, rollback := makeConnectionWithObjectHeaders(t) defer rollback() objects, err := c.Objects(CONTAINER, &swift.ObjectsOpts{Delimiter: '/'}) if err != nil { t.Fatal(err) } if len(objects) != 1 { t.Fatal("Should only be 1 object") } object := objects[0] if object.Name != OBJECT || object.Bytes != CONTENT_SIZE || object.ContentType != "application/octet-stream" || object.Hash != CONTENT_MD5 || object.PseudoDirectory != false || object.SubDir != "" { t.Error("Bad object info", object) } checkTime(t, object.LastModified, -10, 10) } func TestObjectsDirectory(t *testing.T) { c, rollback := makeConnectionWithObjectHeaders(t) defer rollback() err := c.ObjectPutString(CONTAINER, "directory", "", "application/directory") if err != nil { t.Fatal(err) } defer c.ObjectDelete(CONTAINER, "directory") // Look for the directory object and check we aren't confusing // it with a pseudo directory object objects, err := c.Objects(CONTAINER, &swift.ObjectsOpts{Delimiter: '/'}) if err != nil { t.Fatal(err) } if len(objects) != 2 { t.Fatal("Should only be 2 objects") } found := false for i := range objects { object := objects[i] if object.Name == "directory" { found = true if object.Bytes != 0 || object.ContentType != "application/directory" || object.Hash != "d41d8cd98f00b204e9800998ecf8427e" || object.PseudoDirectory != false || object.SubDir != "" { t.Error("Bad object info", object) } checkTime(t, object.LastModified, -10, 10) } } if !found { t.Error("Didn't find directory object") } } func TestObjectsPseudoDirectory(t *testing.T) { c, rollback := makeConnectionWithObjectHeaders(t) defer rollback() err := c.ObjectPutString(CONTAINER, "directory/puppy.jpg", "cute puppy", "") if err != nil { t.Fatal(err) } defer c.ObjectDelete(CONTAINER, "directory/puppy.jpg") // Look for the pseudo directory objects, err := c.Objects(CONTAINER, &swift.ObjectsOpts{Delimiter: '/'}) if err != nil { t.Fatal(err) } if len(objects) != 2 { t.Fatal("Should only be 2 objects", objects) } found := false for i := range objects { object := objects[i] if object.Name == "directory/" { found = true if object.Bytes != 0 || object.ContentType != "application/directory" || object.Hash != "" || object.PseudoDirectory != true || object.SubDir != "directory/" && object.LastModified.IsZero() { t.Error("Bad object info", object) } } } if !found { t.Error("Didn't find directory object", objects) } // Look in the pseudo directory now objects, err = c.Objects(CONTAINER, &swift.ObjectsOpts{Delimiter: '/', Path: "directory/"}) if err != nil { t.Fatal(err) } if len(objects) != 1 { t.Fatal("Should only be 1 object", objects) } object := objects[0] if object.Name != "directory/puppy.jpg" || object.Bytes != 10 || object.ContentType != "image/jpeg" || object.Hash != "87a12ea22fca7f54f0cefef1da535489" || object.PseudoDirectory != false || object.SubDir != "" { t.Error("Bad object info", object) } checkTime(t, object.LastModified, -10, 10) } func TestObjectsAll(t *testing.T) { c, rollback := makeConnectionWithObjectHeaders(t) defer rollback() objects, err := c.ObjectsAll(CONTAINER, nil) if err != nil { t.Fatal(err) } if len(objects) != 1 || objects[0].Name != OBJECT { t.Error("Incorrect listing", objects) } } func TestObjectsAllWithLimit(t *testing.T) { c, rollback := makeConnectionWithObjectHeaders(t) defer rollback() objects, err := c.ObjectsAll(CONTAINER, &swift.ObjectsOpts{Limit: 1}) if err != nil { t.Fatal(err) } if len(objects) != 1 || objects[0].Name != OBJECT { t.Error("Incorrect listing", objects) } } func TestObjectNamesWithPath(t *testing.T) { c, rollback := makeConnectionWithObjectHeaders(t) defer rollback() objects, err := c.ObjectNames(CONTAINER, &swift.ObjectsOpts{Delimiter: '/', Path: ""}) if err != nil { t.Fatal(err) } if len(objects) != 1 || objects[0] != OBJECT { t.Error("Bad listing with path", objects) } // fmt.Println(objects) objects, err = c.ObjectNames(CONTAINER, &swift.ObjectsOpts{Delimiter: '/', Path: "Downloads/"}) if err != nil { t.Fatal(err) } if len(objects) != 0 { t.Error("Bad listing with path", objects) } } func TestObjectCopy(t *testing.T) { c, rollback := makeConnectionWithObjectHeaders(t) defer rollback() _, err := c.ObjectCopy(CONTAINER, OBJECT, CONTAINER, OBJECT2, nil) if err != nil { t.Fatal(err) } err = c.ObjectDelete(CONTAINER, OBJECT2) if err != nil { t.Fatal(err) } } func TestObjectCopyDifficultName(t *testing.T) { c, rollback := makeConnectionWithObjectHeaders(t) defer rollback() const dest = OBJECT + "?param %30%31%32 £100" _, err := c.ObjectCopy(CONTAINER, OBJECT, CONTAINER, dest, nil) if err != nil { t.Fatal(err) } err = c.ObjectDelete(CONTAINER, dest) if err != nil { t.Fatal(err) } } func TestObjectCopyWithMetadata(t *testing.T) { c, rollback := makeConnectionWithObjectHeaders(t) defer rollback() m := swift.Metadata{} m["copy-special-metadata"] = "hello" m["hello"] = "9" h := m.ObjectHeaders() h["Content-Type"] = "image/jpeg" _, err := c.ObjectCopy(CONTAINER, OBJECT, CONTAINER, OBJECT2, h) if err != nil { t.Fatal(err) } defer func() { err = c.ObjectDelete(CONTAINER, OBJECT2) if err != nil { t.Fatal(err) } }() // Re-read the metadata to see if it is correct _, headers, err := c.Object(CONTAINER, OBJECT2) if err != nil { t.Fatal(err) } if headers["Content-Type"] != "image/jpeg" { t.Error("Didn't change content type") } compareMaps(t, headers.ObjectMetadata(), map[string]string{"hello": "9", "potato-salad": "2", "copy-special-metadata": "hello"}) } func TestObjectMove(t *testing.T) { c, rollback := makeConnectionWithObjectHeaders(t) defer rollback() err := c.ObjectMove(CONTAINER, OBJECT, CONTAINER, OBJECT2) if err != nil { t.Fatal(err) } testExistenceAfterDelete(t, c, CONTAINER, OBJECT) _, _, err = c.Object(CONTAINER, OBJECT2) if err != nil { t.Fatal(err) } err = c.ObjectMove(CONTAINER, OBJECT2, CONTAINER, OBJECT) if err != nil { t.Fatal(err) } testExistenceAfterDelete(t, c, CONTAINER, OBJECT2) _, headers, err := c.Object(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } compareMaps(t, headers.ObjectMetadata(), map[string]string{"hello": "1", "potato-salad": "2"}) } func TestObjectUpdateContentType(t *testing.T) { c, rollback := makeConnectionWithObjectHeaders(t) defer rollback() err := c.ObjectUpdateContentType(CONTAINER, OBJECT, "text/potato") if err != nil { t.Fatal(err) } // Re-read the metadata to see if it is correct _, headers, err := c.Object(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } if headers["Content-Type"] != "text/potato" { t.Error("Didn't change content type") } compareMaps(t, headers.ObjectMetadata(), map[string]string{"hello": "1", "potato-salad": "2"}) } func TestVersionContainerCreate(t *testing.T) { c, rollback := makeConnectionAuth(t) defer rollback() err := c.VersionContainerCreate(CURRENT_CONTAINER, VERSIONS_CONTAINER) defer func() { c.ContainerDelete(CURRENT_CONTAINER) c.ContainerDelete(VERSIONS_CONTAINER) }() if err != nil { if err == swift.Forbidden { t.Log("Server doesn't support Versions - skipping test") return } t.Fatal(err) } } func TestVersionObjectAdd(t *testing.T) { c, rollback := makeConnectionWithVersionsContainer(t) defer rollback() if skipVersionTests { t.Log("Server doesn't support Versions - skipping test") return } // Version 1 if err := c.ObjectPutString(CURRENT_CONTAINER, OBJECT, CONTENTS, ""); err != nil { t.Fatal(err) } defer func() { err := c.ObjectDelete(CURRENT_CONTAINER, OBJECT) if err != nil { t.Fatal(err) } }() if contents, err := c.ObjectGetString(CURRENT_CONTAINER, OBJECT); err != nil { t.Fatal(err) } else if contents != CONTENTS { t.Error("Contents wrong") } // Version 2 if err := c.ObjectPutString(CURRENT_CONTAINER, OBJECT, CONTENTS2, ""); err != nil { t.Fatal(err) } defer func() { err := c.ObjectDelete(CURRENT_CONTAINER, OBJECT) if err != nil { t.Fatal(err) } }() if contents, err := c.ObjectGetString(CURRENT_CONTAINER, OBJECT); err != nil { t.Fatal(err) } else if contents != CONTENTS2 { t.Error("Contents wrong") } // Version 3 if err := c.ObjectPutString(CURRENT_CONTAINER, OBJECT, CONTENTS2, ""); err != nil { t.Fatal(err) } defer func() { err := c.ObjectDelete(CURRENT_CONTAINER, OBJECT) if err != nil { t.Fatal(err) } }() } func TestVersionObjectList(t *testing.T) { c, rollback := makeConnectionWithVersionsObject(t) defer rollback() if skipVersionTests { t.Log("Server doesn't support Versions - skipping test") return } list, err := c.VersionObjectList(VERSIONS_CONTAINER, OBJECT) if err != nil { t.Fatal(err) } if len(list) != 2 { t.Error("Version list should return 2 objects") } } func TestVersionObjectDelete(t *testing.T) { c, rollback := makeConnectionWithVersionsObject(t) defer rollback() if skipVersionTests { t.Log("Server doesn't support Versions - skipping test") return } // Delete Version 3 if err := c.ObjectDelete(CURRENT_CONTAINER, OBJECT); err != nil { t.Fatal(err) } // Delete Version 2 if err := c.ObjectDelete(CURRENT_CONTAINER, OBJECT); err != nil { t.Fatal(err) } // Contents should be reverted to Version 1 if contents, err := c.ObjectGetString(CURRENT_CONTAINER, OBJECT); err != nil { t.Fatal(err) } else if contents != CONTENTS { t.Error("Contents wrong") } } func TestVersionDeleteContent(t *testing.T) { c, rollback := makeConnectionWithVersionsObject(t) defer rollback() if skipVersionTests { t.Log("Server doesn't support Versions - skipping test") return } // Delete Version 3 if err := c.ObjectDelete(CURRENT_CONTAINER, OBJECT); err != nil { t.Fatal(err) } // Delete Version 2 if err := c.ObjectDelete(CURRENT_CONTAINER, OBJECT); err != nil { t.Fatal(err) } // Delete Version 1 if err := c.ObjectDelete(CURRENT_CONTAINER, OBJECT); err != nil { t.Fatal(err) } if err := c.ObjectDelete(CURRENT_CONTAINER, OBJECT); err != swift.ObjectNotFound { t.Fatalf("Expecting Object not found error, got: %v", err) } } // Check for non existence after delete // May have to do it a few times to wait for swift to be consistent. func testExistenceAfterDelete(t *testing.T, c *swift.Connection, container, object string) { for i := 10; i <= 0; i-- { _, _, err := c.Object(container, object) if err == swift.ObjectNotFound { break } if i == 0 { t.Fatalf("Expecting object %q/%q not found not: err=%v", container, object, err) } time.Sleep(1 * time.Second) } } func TestObjectDelete(t *testing.T) { c, rollback := makeConnectionWithObject(t) defer rollback() err := c.ObjectDelete(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } testExistenceAfterDelete(t, c, CONTAINER, OBJECT) err = c.ObjectDelete(CONTAINER, OBJECT) if err != swift.ObjectNotFound { t.Fatal("Expecting Object not found", err) } } func TestBulkDelete(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() result, err := c.BulkDelete(CONTAINER, []string{OBJECT}) if err == swift.Forbidden { t.Log("Server doesn't support BulkDelete - skipping test") return } if err != nil { t.Fatal(err) } if result.NumberNotFound != 1 { t.Error("Expected 1, actual:", result.NumberNotFound) } if result.NumberDeleted != 0 { t.Error("Expected 0, actual:", result.NumberDeleted) } err = c.ObjectPutString(CONTAINER, OBJECT, CONTENTS, "") if err != nil { t.Fatal(err) } result, err = c.BulkDelete(CONTAINER, []string{OBJECT2, OBJECT}) if err != nil { t.Fatal(err) } if result.NumberNotFound != 1 { t.Error("Expected 1, actual:", result.NumberNotFound) } if result.NumberDeleted != 1 { t.Error("Expected 1, actual:", result.NumberDeleted) } t.Log("Errors:", result.Errors) } func TestBulkUpload(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() buffer := new(bytes.Buffer) ds := tar.NewWriter(buffer) var files = []struct{ Name, Body string }{ {OBJECT, CONTENTS}, {OBJECT2, CONTENTS2}, } for _, file := range files { hdr := &tar.Header{ Name: file.Name, Size: int64(len(file.Body)), } if err := ds.WriteHeader(hdr); err != nil { t.Fatal(err) } if _, err := ds.Write([]byte(file.Body)); err != nil { t.Fatal(err) } } if err := ds.Close(); err != nil { t.Fatal(err) } result, err := c.BulkUpload(CONTAINER, buffer, swift.UploadTar, nil) if err == swift.Forbidden { t.Log("Server doesn't support BulkUpload - skipping test") return } if err != nil { t.Fatal(err) } defer func() { err = c.ObjectDelete(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } err = c.ObjectDelete(CONTAINER, OBJECT2) if err != nil { t.Fatal(err) } }() if result.NumberCreated != 2 { t.Error("Expected 2, actual:", result.NumberCreated) } t.Log("Errors:", result.Errors) _, _, err = c.Object(CONTAINER, OBJECT) if err != nil { t.Error("Expecting object to be found") } _, _, err = c.Object(CONTAINER, OBJECT2) if err != nil { t.Error("Expecting object to be found") } } func TestObjectDifficultName(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() const name = `hello? sausage/êé/Hello, 世界/ " ' @ < > & ?/` err := c.ObjectPutString(CONTAINER, name, CONTENTS, "") if err != nil { t.Fatal(err) } defer func() { err = c.ObjectDelete(CONTAINER, name) if err != nil { t.Fatal(err) } }() objects, err := c.ObjectNamesAll(CONTAINER, nil) if err != nil { t.Error(err) } found := false for _, object := range objects { if object == name { found = true break } } if !found { t.Errorf("Couldn't find %q in listing %q", name, objects) } } func TestTempUrl(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() err := c.ObjectPutBytes(CONTAINER, OBJECT, []byte(CONTENTS), "") if err != nil { t.Fatal(err) } defer func() { err = c.ObjectDelete(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } }() m := swift.Metadata{} m["temp-url-key"] = SECRET_KEY err = c.AccountUpdate(m.AccountHeaders()) if err != nil { t.Fatal(err) } expiresTime := time.Now().Add(20 * time.Minute) tempUrl := c.ObjectTempUrl(CONTAINER, OBJECT, SECRET_KEY, "GET", expiresTime) resp, err := http.Get(tempUrl) if err != nil { t.Fatal("Failed to retrieve file from temporary url") } defer resp.Body.Close() if resp.StatusCode == 401 { t.Log("Server doesn't support tempurl") } else if resp.StatusCode != 200 { t.Fatal("HTTP Error retrieving file from temporary url", resp.StatusCode) } else { var content []byte if content, err = ioutil.ReadAll(resp.Body); err != nil || string(content) != CONTENTS { t.Error("Bad content", err) } resp, err = http.Post(tempUrl, "image/jpeg", bytes.NewReader([]byte(CONTENTS))) if err != nil { t.Fatal("Failed to retrieve file from temporary url") } defer resp.Body.Close() if resp.StatusCode != 401 { t.Fatal("Expecting server to forbid access to object") } } } func TestQueryInfo(t *testing.T) { c, rollback := makeConnectionAuth(t) defer rollback() infos, err := c.QueryInfo() if err != nil { t.Log("Server doesn't support querying info") return } if _, ok := infos["swift"]; !ok { t.Fatal("No 'swift' section found in configuration") } } func TestDLOCreate(t *testing.T) { c, rollback := makeConnectionWithSegmentsContainer(t) defer rollback() opts := swift.LargeObjectOpts{ Container: CONTAINER, ObjectName: OBJECT, ContentType: "image/jpeg", } out, err := c.DynamicLargeObjectCreate(&opts) if err != nil { t.Fatal(err) } defer func() { err = c.DynamicLargeObjectDelete(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } }() buf := &bytes.Buffer{} multi := io.MultiWriter(buf, out) for i := 0; i < 2; i++ { _, err = fmt.Fprintf(multi, "%d %s\n", i, CONTENTS) if err != nil { t.Fatal(err) } } err = out.Close() if err != nil { t.Error(err) } expected := buf.String() contents, err := c.ObjectGetString(CONTAINER, OBJECT) if err != nil { t.Error(err) } if contents != expected { t.Errorf("Contents wrong, expected %q, got: %q", expected, contents) } info, _, err := c.Object(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } if info.ObjectType != swift.DynamicLargeObjectType { t.Errorf("Wrong ObjectType, expected %d, got: %d", swift.DynamicLargeObjectType, info.ObjectType) } if info.Bytes != int64(len(expected)) { t.Errorf("Wrong Bytes size, expected %d, got: %d", len(expected), info.Bytes) } } func TestDLOInsert(t *testing.T) { c, rollback := makeConnectionWithDLO(t) defer rollback() opts := swift.LargeObjectOpts{ Container: CONTAINER, ObjectName: OBJECT, CheckHash: true, ContentType: "image/jpeg", } out, err := c.DynamicLargeObjectCreateFile(&opts) if err != nil { t.Fatal(err) } buf := &bytes.Buffer{} multi := io.MultiWriter(buf, out) _, err = fmt.Fprintf(multi, "%d%s\n", 0, CONTENTS) if err != nil { t.Fatal(err) } fmt.Fprintf(buf, "\n%d %s\n", 1, CONTENTS) err = out.Close() if err != nil { t.Error(err) } expected := buf.String() contents, err := c.ObjectGetString(CONTAINER, OBJECT) if err != nil { t.Error(err) } if contents != expected { t.Errorf("Contents wrong, expected %q, got: %q", expected, contents) } } func TestDLOAppend(t *testing.T) { c, rollback := makeConnectionWithDLO(t) defer rollback() opts := swift.LargeObjectOpts{ Container: CONTAINER, ObjectName: OBJECT, Flags: os.O_APPEND, CheckHash: true, ContentType: "image/jpeg", } out, err := c.DynamicLargeObjectCreateFile(&opts) if err != nil { t.Fatal(err) } contents, err := c.ObjectGetString(CONTAINER, OBJECT) buf := bytes.NewBuffer([]byte(contents)) multi := io.MultiWriter(buf, out) for i := 0; i < 2; i++ { _, err = fmt.Fprintf(multi, "%d %s\n", i+10, CONTENTS) if err != nil { t.Fatal(err) } } err = out.Close() if err != nil { t.Error(err) } expected := buf.String() contents, err = c.ObjectGetString(CONTAINER, OBJECT) if err != nil { t.Error(err) } if contents != expected { t.Errorf("Contents wrong, expected %q, got: %q", expected, contents) } } func TestDLOTruncate(t *testing.T) { c, rollback := makeConnectionWithDLO(t) defer rollback() opts := swift.LargeObjectOpts{ Container: CONTAINER, ObjectName: OBJECT, Flags: os.O_TRUNC, CheckHash: true, ContentType: "image/jpeg", } out, err := c.DynamicLargeObjectCreateFile(&opts) if err != nil { t.Fatal(err) } buf := &bytes.Buffer{} multi := io.MultiWriter(buf, out) _, err = fmt.Fprintf(multi, "%s", CONTENTS) if err != nil { t.Fatal(err) } err = out.Close() if err != nil { t.Error(err) } expected := buf.String() contents, err := c.ObjectGetString(CONTAINER, OBJECT) if err != nil { t.Error(err) } if contents != expected { t.Errorf("Contents wrong, expected %q, got: %q", expected, contents) } } func TestDLOMove(t *testing.T) { c, rollback := makeConnectionWithDLO(t) defer rollback() contents, err := c.ObjectGetString(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } err = c.DynamicLargeObjectMove(CONTAINER, OBJECT, CONTAINER, OBJECT2) if err != nil { t.Fatal(err) } defer func() { err = c.DynamicLargeObjectDelete(CONTAINER, OBJECT2) if err != nil { t.Fatal(err) } }() contents2, err := c.ObjectGetString(CONTAINER, OBJECT2) if err != nil { t.Fatal(err) } if contents2 != contents { t.Error("Contents wrong") } } func TestDLONoSegmentContainer(t *testing.T) { c, rollback := makeConnectionWithDLO(t) defer rollback() opts := swift.LargeObjectOpts{ Container: CONTAINER, ObjectName: OBJECT, ContentType: "image/jpeg", SegmentContainer: CONTAINER, } out, err := c.DynamicLargeObjectCreate(&opts) if err != nil { t.Fatal(err) } buf := &bytes.Buffer{} multi := io.MultiWriter(buf, out) for i := 0; i < 2; i++ { _, err = fmt.Fprintf(multi, "%d %s\n", i, CONTENTS) if err != nil { t.Fatal(err) } } err = out.Close() if err != nil { t.Error(err) } expected := buf.String() contents, err := c.ObjectGetString(CONTAINER, OBJECT) if err != nil { t.Error(err) } if contents != expected { t.Errorf("Contents wrong, expected %q, got: %q", expected, contents) } } func TestDLOCreateMissingSegmentsInList(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() if srv == nil { t.Skipf("This test only runs with the fake swift server as it's needed to simulate eventual consistency problems.") return } listURL := "/v1/AUTH_" + swifttest.TEST_ACCOUNT + "/" + SEGMENTS_CONTAINER srv.SetOverride(listURL, func(w http.ResponseWriter, r *http.Request, recorder *httptest.ResponseRecorder) { for k, v := range recorder.HeaderMap { w.Header().Set(k, v[0]) } w.WriteHeader(recorder.Code) w.Write([]byte("null\n")) }) defer srv.UnsetOverride(listURL) headers := swift.Headers{} err := c.ContainerCreate(SEGMENTS_CONTAINER, headers) if err != nil { t.Fatal(err) } defer func() { err = c.ContainerDelete(SEGMENTS_CONTAINER) if err != nil { t.Fatal(err) } }() opts := swift.LargeObjectOpts{ Container: CONTAINER, ObjectName: OBJECT, ContentType: "image/jpeg", } out, err := c.DynamicLargeObjectCreate(&opts) if err != nil { t.Fatal(err) } defer func() { err = c.DynamicLargeObjectDelete(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } }() buf := &bytes.Buffer{} multi := io.MultiWriter(buf, out) for i := 0; i < 2; i++ { _, err = fmt.Fprintf(multi, "%d %s\n", i, CONTENTS) if err != nil { t.Fatal(err) } } err = out.Close() if err != nil { t.Error(err) } expected := buf.String() contents, err := c.ObjectGetString(CONTAINER, OBJECT) if err != nil { t.Error(err) } if contents != expected { t.Errorf("Contents wrong, expected %q, got: %q", expected, contents) } } func TestDLOCreateIncorrectSize(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() if srv == nil { t.Skipf("This test only runs with the fake swift server as it's needed to simulate eventual consistency problems.") return } listURL := "/v1/AUTH_" + swifttest.TEST_ACCOUNT + "/" + CONTAINER + "/" + OBJECT headCount := 0 expectedHeadCount := 5 srv.SetOverride(listURL, func(w http.ResponseWriter, r *http.Request, recorder *httptest.ResponseRecorder) { for k, v := range recorder.HeaderMap { w.Header().Set(k, v[0]) } if r.Method == "HEAD" { headCount++ if headCount < expectedHeadCount { w.Header().Set("Content-Length", "7") } } w.WriteHeader(recorder.Code) w.Write(recorder.Body.Bytes()) }) defer srv.UnsetOverride(listURL) headers := swift.Headers{} err := c.ContainerCreate(SEGMENTS_CONTAINER, headers) if err != nil { t.Fatal(err) } defer func() { err = c.ContainerDelete(SEGMENTS_CONTAINER) if err != nil { t.Fatal(err) } }() opts := swift.LargeObjectOpts{ Container: CONTAINER, ObjectName: OBJECT, ContentType: "image/jpeg", } out, err := c.DynamicLargeObjectCreate(&opts) if err != nil { t.Fatal(err) } defer func() { err = c.DynamicLargeObjectDelete(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } }() buf := &bytes.Buffer{} multi := io.MultiWriter(buf, out) for i := 0; i < 2; i++ { _, err = fmt.Fprintf(multi, "%d %s\n", i, CONTENTS) if err != nil { t.Fatal(err) } } err = out.Close() if err != nil { t.Error(err) } if headCount != expectedHeadCount { t.Errorf("Unexpected HEAD requests count, expected %d, got: %d", expectedHeadCount, headCount) } expected := buf.String() contents, err := c.ObjectGetString(CONTAINER, OBJECT) if err != nil { t.Error(err) } if contents != expected { t.Errorf("Contents wrong, expected %q, got: %q", expected, contents) } } func TestDLOConcurrentWrite(t *testing.T) { c, rollback := makeConnectionWithSegmentsContainer(t) defer rollback() nConcurrency := 5 nChunks := 100 var chunkSize int64 = 1024 writeFn := func(i int) { objName := fmt.Sprintf("%s_concurrent_dlo_%d", OBJECT, i) opts := swift.LargeObjectOpts{ Container: CONTAINER, ObjectName: objName, ContentType: "image/jpeg", } out, err := c.DynamicLargeObjectCreate(&opts) if err != nil { t.Fatal(err) } defer func() { err = c.DynamicLargeObjectDelete(CONTAINER, objName) if err != nil { t.Fatal(err) } }() buf := &bytes.Buffer{} for j := 0; j < nChunks; j++ { var data []byte var n int data, err = ioutil.ReadAll(io.LimitReader(rand.Reader, chunkSize)) if err != nil { t.Fatal(err) } multi := io.MultiWriter(buf, out) n, err = multi.Write(data) if err != nil { t.Fatal(err) } if int64(n) != chunkSize { t.Fatalf("expected to write %d, got: %d", chunkSize, n) } } err = out.Close() if err != nil { t.Error(err) } expected := buf.String() contents, err := c.ObjectGetString(CONTAINER, objName) if err != nil { t.Error(err) } if contents != expected { t.Error("Contents wrong") } } wg := sync.WaitGroup{} for i := 0; i < nConcurrency; i++ { wg.Add(1) go func(i int) { defer wg.Done() writeFn(i) }(i) } wg.Wait() } func TestDLOSegmentation(t *testing.T) { c, rollback := makeConnectionWithSegmentsContainer(t) defer rollback() opts := swift.LargeObjectOpts{ Container: CONTAINER, ObjectName: OBJECT, ContentType: "image/jpeg", ChunkSize: 6, NoBuffer: true, } testSegmentation(t, c, func() swift.LargeObjectFile { out, err := c.DynamicLargeObjectCreate(&opts) if err != nil { t.Fatal(err) } return out }, []segmentTest{ { writes: []string{"0", "1", "2", "3", "4", "5", "6", "7", "8"}, expectedSegs: []string{"0", "1", "2", "3", "4", "5", "6", "7", "8"}, expectedValue: "012345678", }, { writes: []string{"012345", "012345"}, expectedSegs: []string{"012345", "012345"}, expectedValue: "012345012345", }, { writes: []string{"0123456", "0123456"}, expectedSegs: []string{"012345", "6", "012345", "6"}, expectedValue: "01234560123456", }, { writes: []string{"0123456", "0123456"}, seeks: []int{-4, 0}, expectedSegs: []string{"012012", "3456"}, expectedValue: "0120123456", }, { writes: []string{"0123456", "0123456", "abcde"}, seeks: []int{0, -11, 0}, expectedSegs: []string{"012abc", "d", "e12345", "6"}, expectedValue: "012abcde123456", }, { writes: []string{"0123456", "ab"}, seeks: []int{-4, 0}, expectedSegs: []string{"012ab5", "6"}, expectedValue: "012ab56", }, }) } func TestDLOSegmentationBuffered(t *testing.T) { c, rollback := makeConnectionWithSegmentsContainer(t) defer rollback() opts := swift.LargeObjectOpts{ Container: CONTAINER, ObjectName: OBJECT, ContentType: "image/jpeg", ChunkSize: 6, } testSegmentation(t, c, func() swift.LargeObjectFile { out, err := c.DynamicLargeObjectCreate(&opts) if err != nil { t.Fatal(err) } return out }, []segmentTest{ { writes: []string{"0", "1", "2", "3", "4", "5", "6", "7", "8"}, expectedSegs: []string{"012345", "678"}, expectedValue: "012345678", }, { writes: []string{"012345", "012345"}, expectedSegs: []string{"012345", "012345"}, expectedValue: "012345012345", }, { writes: []string{"0123456", "0123456"}, expectedSegs: []string{"012345", "6", "012345", "6"}, expectedValue: "01234560123456", }, { writes: []string{"0123456", "0123456"}, seeks: []int{-4, 0}, expectedSegs: []string{"012012", "3456"}, expectedValue: "0120123456", }, { writes: []string{"0123456", "0123456", "abcde"}, seeks: []int{0, -11, 0}, expectedSegs: []string{"012abc", "d", "e12345", "6"}, expectedValue: "012abcde123456", }, { writes: []string{"0123456", "ab"}, seeks: []int{-4, 0}, expectedSegs: []string{"012ab5", "6"}, expectedValue: "012ab56", }, }) } func TestSLOCreate(t *testing.T) { c, rollback := makeConnectionWithSegmentsContainer(t) defer rollback() opts := swift.LargeObjectOpts{ Container: CONTAINER, ObjectName: OBJECT, ContentType: "image/jpeg", } out, err := c.StaticLargeObjectCreate(&opts) if err != nil { if err == swift.SLONotSupported { t.Skip("SLO not supported") return } t.Fatal(err) } defer func() { err = c.StaticLargeObjectDelete(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } }() buf := &bytes.Buffer{} multi := io.MultiWriter(buf, out) for i := 0; i < 2; i++ { _, err = fmt.Fprintf(multi, "%d %s\n", i, CONTENTS) if err != nil { t.Fatal(err) } } err = out.Close() if err != nil { t.Error(err) } expected := buf.String() contents, err := c.ObjectGetString(CONTAINER, OBJECT) if err != nil { t.Error(err) } if contents != expected { t.Errorf("Contents wrong, expected %q, got: %q", expected, contents) } info, _, err := c.Object(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } if info.ObjectType != swift.StaticLargeObjectType { t.Errorf("Wrong ObjectType, expected %d, got: %d", swift.StaticLargeObjectType, info.ObjectType) } if info.Bytes != int64(len(expected)) { t.Errorf("Wrong Bytes size, expected %d, got: %d", len(expected), info.Bytes) } } func TestSLOInsert(t *testing.T) { c, rollback := makeConnectionWithSLO(t) defer rollback() opts := swift.LargeObjectOpts{ Container: CONTAINER, ObjectName: OBJECT, ContentType: "image/jpeg", } out, err := c.StaticLargeObjectCreateFile(&opts) if err != nil { t.Fatal(err) } buf := &bytes.Buffer{} multi := io.MultiWriter(buf, out) _, err = fmt.Fprintf(multi, "%d%s\n", 0, CONTENTS) if err != nil { t.Fatal(err) } fmt.Fprintf(buf, "\n%d %s\n", 1, CONTENTS) err = out.Close() if err != nil { t.Error(err) } expected := buf.String() contents, err := c.ObjectGetString(CONTAINER, OBJECT) if err != nil { t.Error(err) } if contents != expected { t.Errorf("Contents wrong, expected %q, got: %q", expected, contents) } } func TestSLOAppend(t *testing.T) { c, rollback := makeConnectionWithSLO(t) defer rollback() opts := swift.LargeObjectOpts{ Container: CONTAINER, ObjectName: OBJECT, Flags: os.O_APPEND, CheckHash: true, ContentType: "image/jpeg", } out, err := c.StaticLargeObjectCreateFile(&opts) if err != nil { t.Fatal(err) } contents, err := c.ObjectGetString(CONTAINER, OBJECT) buf := bytes.NewBuffer([]byte(contents)) multi := io.MultiWriter(buf, out) for i := 0; i < 2; i++ { _, err = fmt.Fprintf(multi, "%d %s\n", i+10, CONTENTS) if err != nil { t.Fatal(err) } } err = out.Close() if err != nil { t.Error(err) } expected := buf.String() contents, err = c.ObjectGetString(CONTAINER, OBJECT) if err != nil { t.Error(err) } if contents != expected { t.Errorf("Contents wrong, expected %q, got: %q", expected, contents) } } func TestSLOMove(t *testing.T) { c, rollback := makeConnectionWithSLO(t) defer rollback() contents, err := c.ObjectGetString(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } err = c.StaticLargeObjectMove(CONTAINER, OBJECT, CONTAINER, OBJECT2) if err != nil { t.Fatal(err) } defer func() { err = c.StaticLargeObjectDelete(CONTAINER, OBJECT2) if err != nil { t.Fatal(err) } }() contents2, err := c.ObjectGetString(CONTAINER, OBJECT2) if err != nil { t.Fatal(err) } if contents2 != contents { t.Error("Contents wrong") } } func TestSLONoSegmentContainer(t *testing.T) { c, rollback := makeConnectionWithSLO(t) defer rollback() opts := swift.LargeObjectOpts{ Container: CONTAINER, ObjectName: OBJECT, ContentType: "image/jpeg", SegmentContainer: CONTAINER, } out, err := c.StaticLargeObjectCreate(&opts) if err != nil { t.Fatal(err) } buf := &bytes.Buffer{} multi := io.MultiWriter(buf, out) for i := 0; i < 2; i++ { _, err = fmt.Fprintf(multi, "%d %s\n", i, CONTENTS) if err != nil { t.Fatal(err) } } err = out.Close() if err != nil { t.Error(err) } expected := buf.String() contents, err := c.ObjectGetString(CONTAINER, OBJECT) if err != nil { t.Error(err) } if contents != expected { t.Errorf("Contents wrong, expected %q, got: %q", expected, contents) } err = c.StaticLargeObjectDelete(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } } func TestSLOMinChunkSize(t *testing.T) { c, rollback := makeConnectionWithSegmentsContainer(t) defer rollback() if srv == nil { t.Skipf("This test only runs with the fake swift server as it's needed to simulate min segment size.") return } srv.SetOverride("/info", func(w http.ResponseWriter, r *http.Request, recorder *httptest.ResponseRecorder) { w.Write([]byte(`{"slo": {"min_segment_size": 4}}`)) }) defer srv.UnsetOverride("/info") c.QueryInfo() opts := swift.LargeObjectOpts{ Container: CONTAINER, ObjectName: OBJECT, ContentType: "image/jpeg", ChunkSize: 6, MinChunkSize: 0, NoBuffer: true, } testSLOSegmentation(t, c, func() swift.LargeObjectFile { out, err := c.StaticLargeObjectCreate(&opts) if err != nil { t.Fatal(err) } return out }) } func TestSLOSegmentation(t *testing.T) { c, rollback := makeConnectionWithSegmentsContainer(t) defer rollback() opts := swift.LargeObjectOpts{ Container: CONTAINER, ObjectName: OBJECT, ContentType: "image/jpeg", ChunkSize: 6, MinChunkSize: 4, NoBuffer: true, } testSLOSegmentation(t, c, func() swift.LargeObjectFile { out, err := c.StaticLargeObjectCreate(&opts) if err != nil { if err == swift.SLONotSupported { t.Skip("SLO not supported") } t.Fatal(err) } return out }) } func TestSLOSegmentationBuffered(t *testing.T) { c, rollback := makeConnectionWithSegmentsContainer(t) defer rollback() opts := swift.LargeObjectOpts{ Container: CONTAINER, ObjectName: OBJECT, ContentType: "image/jpeg", ChunkSize: 6, MinChunkSize: 4, } testSegmentation(t, c, func() swift.LargeObjectFile { out, err := c.StaticLargeObjectCreate(&opts) if err != nil { if err == swift.SLONotSupported { t.Skip("SLO not supported") } t.Fatal(err) } return out }, []segmentTest{ { writes: []string{"0", "1", "2", "3", "4", "5", "6", "7", "8"}, expectedSegs: []string{"012345", "678"}, expectedValue: "012345678", }, { writes: []string{"012345", "012345"}, expectedSegs: []string{"012345", "012345"}, expectedValue: "012345012345", }, { writes: []string{"0123456", "0123456"}, expectedSegs: []string{"012345", "601234", "56"}, expectedValue: "01234560123456", }, { writes: []string{"0123456", "0123456"}, seeks: []int{-4, 0}, expectedSegs: []string{"012012", "3456"}, expectedValue: "0120123456", }, { writes: []string{"0123456", "0123456", "abcde"}, seeks: []int{0, -11, 0}, expectedSegs: []string{"012abc", "de1234", "56"}, expectedValue: "012abcde123456", }, { writes: []string{"0123456", "ab"}, seeks: []int{-4, 0}, expectedSegs: []string{"012ab5", "6"}, expectedValue: "012ab56", }, }) } func testSLOSegmentation(t *testing.T, c *swift.Connection, createObj func() swift.LargeObjectFile) { testCases := []segmentTest{ { writes: []string{"0", "1", "2", "3", "4", "5", "6", "7", "8"}, expectedSegs: []string{"0123", "4567", "8"}, expectedValue: "012345678", }, { writes: []string{"012345", "012345"}, expectedSegs: []string{"012345", "012345"}, expectedValue: "012345012345", }, { writes: []string{"0123456", "0123456"}, expectedSegs: []string{"012345", "601234", "56"}, expectedValue: "01234560123456", }, { writes: []string{"0123456", "0123456"}, seeks: []int{-4, 0}, expectedSegs: []string{"012012", "3456"}, expectedValue: "0120123456", }, { writes: []string{"0123456", "0123456", "abcde"}, seeks: []int{0, -11, 0}, expectedSegs: []string{"012abc", "de1234", "56"}, expectedValue: "012abcde123456", }, { writes: []string{"0123456", "ab"}, seeks: []int{-4, 0}, expectedSegs: []string{"012ab5", "6"}, expectedValue: "012ab56", }, } testSegmentation(t, c, createObj, testCases) } type segmentTest struct { writes []string seeks []int expectedSegs []string expectedValue string } func testSegmentation(t *testing.T, c *swift.Connection, createObj func() swift.LargeObjectFile, testCases []segmentTest) { var err error runTestCase := func(tCase segmentTest) { out := createObj() defer func() { err = c.LargeObjectDelete(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } }() for i, data := range tCase.writes { _, err = fmt.Fprint(out, data) if err != nil { t.Error(err) } if i < len(tCase.seeks)-1 { _, err = out.Seek(int64(tCase.seeks[i]), os.SEEK_CUR) if err != nil { t.Error(err) } } } err = out.Close() if err != nil { t.Error(err) } contents, err := c.ObjectGetString(CONTAINER, OBJECT) if err != nil { t.Error(err) } if contents != tCase.expectedValue { t.Errorf("Contents wrong, expected %q, got: %q", tCase.expectedValue, contents) } container, objects, err := c.LargeObjectGetSegments(CONTAINER, OBJECT) if err != nil { t.Error(err) } if container != SEGMENTS_CONTAINER { t.Errorf("Segments container wrong, expected %q, got: %q", SEGMENTS_CONTAINER, container) } _, headers, err := c.Object(CONTAINER, OBJECT) if err != nil { t.Fatal(err) } if headers.IsLargeObjectSLO() { var info swift.SwiftInfo info, err = c.QueryInfo() if err != nil { t.Fatal(err) } if info.SLOMinSegmentSize() > 4 { t.Log("Skipping checking segments because SLO min segment size imposed by server is larger than wanted for tests.") return } } var segContents []string for _, obj := range objects { var value string value, err = c.ObjectGetString(SEGMENTS_CONTAINER, obj.Name) if err != nil { t.Error(err) } segContents = append(segContents, value) } if !reflect.DeepEqual(segContents, tCase.expectedSegs) { t.Errorf("Segments wrong, expected %#v, got: %#v", tCase.expectedSegs, segContents) } } for _, tCase := range testCases { runTestCase(tCase) } } func TestContainerDelete(t *testing.T) { c, rollback := makeConnectionWithContainer(t) defer rollback() err := c.ContainerDelete(CONTAINER) if err != nil { t.Fatal(err) } err = c.ContainerDelete(CONTAINER) if err != swift.ContainerNotFound { t.Fatal("Expecting container not found", err) } _, _, err = c.Container(CONTAINER) if err != swift.ContainerNotFound { t.Fatal("Expecting container not found", err) } } func TestUnAuthenticate(t *testing.T) { c, rollback := makeConnectionAuth(t) defer rollback() c.UnAuthenticate() if c.Authenticated() { t.Fatal("Shouldn't be authenticated") } // Test re-authenticate err := c.Authenticate() if err != nil { t.Fatal("ReAuth failed", err) } if !c.Authenticated() { t.Fatal("Not authenticated") } } swift-1.0.52/swifttest/000077500000000000000000000000001365550407400150005ustar00rootroot00000000000000swift-1.0.52/swifttest/server.go000066400000000000000000000772311365550407400166470ustar00rootroot00000000000000// This implements a very basic Swift server // Everything is stored in memory // // This comes from the https://github.com/mitchellh/goamz // and was adapted for Swift // package swifttest import ( "archive/tar" "bytes" "compress/bzip2" "compress/gzip" "crypto/hmac" "crypto/md5" "crypto/rand" "crypto/sha1" "encoding/hex" "encoding/json" "fmt" "io" "io/ioutil" "log" "mime" "net" "net/http" "net/http/httptest" "net/url" "path" "reflect" "regexp" "sort" "strconv" "strings" "sync" "sync/atomic" "testing" "time" ) var ( DEBUG = false TEST_ACCOUNT = "swifttest" ) type HandlerOverrideFunc func(w http.ResponseWriter, r *http.Request, recorder *httptest.ResponseRecorder) type SwiftServer struct { // `sync/atomic` expects the first word in an allocated struct to be 64-bit // aligned on both ARM and x86-32. // See https://golang.org/pkg/sync/atomic/#pkg-note-BUG for more details. reqId int64 sync.RWMutex t *testing.T mu sync.Mutex Listener net.Listener AuthURL string URL string Accounts map[string]*account Sessions map[string]*session override map[string]HandlerOverrideFunc } // The Folder type represents a container stored in an account type Folder struct { Count int64 `json:"count"` Bytes int64 `json:"bytes"` Name string `json:"name"` } // The Key type represents an item stored in an container. type Key struct { Key string `json:"name"` LastModified string `json:"last_modified"` Size int64 `json:"bytes"` // ETag gives the hex-encoded MD5 sum of the contents, // surrounded with double-quotes. ETag string `json:"hash"` ContentType string `json:"content_type"` // Owner Owner } type Subdir struct { Subdir string `json:"subdir"` } type AutoExtractResponse struct { CreatedFiles int64 `json:"Number Files Created"` Status string `json:"Response Status"` Errors [][]string `json:"Errors"` } type swiftError struct { statusCode int Code string Message string } type action struct { srv *SwiftServer w http.ResponseWriter req *http.Request reqId string user *account } type session struct { username string } type metadata struct { meta http.Header // metadata to return with requests. } type swiftaccount struct { BytesUsed int64 // total number of bytes used Containers int64 // total number of containers Objects int64 // total number of objects } type account struct { sync.RWMutex swiftaccount metadata password string ContainersLock sync.RWMutex Containers map[string]*container } type object struct { sync.RWMutex metadata name string mtime time.Time checksum []byte // also held as ETag in meta. data []byte content_type string } type container struct { // `sync/atomic` expects the first word in an allocated struct to be 64-bit // aligned on both ARM and x86-32. // See https://golang.org/pkg/sync/atomic/#pkg-note-BUG for more details. bytes int64 sync.RWMutex metadata name string ctime time.Time objects map[string]*object } type segment struct { Path string `json:"path,omitempty"` Hash string `json:"hash,omitempty"` Size int64 `json:"size_bytes,omitempty"` // When uploading a manifest, the attributes must be named `path`, `hash` and `size` // but when querying the JSON content of a manifest with the `multipart-manifest=get` // parameter, Swift names those attributes `name`, `etag` and `bytes`. // We use all the different attributes names in this structure to be able to use // the same structure for both uploading and retrieving. Name string `json:"name,omitempty"` Etag string `json:"etag,omitempty"` Bytes int64 `json:"bytes,omitempty"` ContentType string `json:"content_type,omitempty"` LastModified string `json:"last_modified,omitempty"` } // A resource encapsulates the subject of an HTTP request. // The resource referred to may or may not exist // when the request is made. type resource interface { put(a *action) interface{} get(a *action) interface{} post(a *action) interface{} delete(a *action) interface{} copy(a *action) interface{} } type objectResource struct { name string version string container *container // always non-nil. object *object // may be nil. } type containerResource struct { name string container *container // non-nil if the container already exists. } var responseParams = map[string]bool{ "content-type": true, "content-language": true, "expires": true, "cache-control": true, "content-disposition": true, "content-encoding": true, } func fatalf(code int, codeStr string, errf string, a ...interface{}) { panic(&swiftError{ statusCode: code, Code: codeStr, Message: fmt.Sprintf(errf, a...), }) } func (m metadata) setMetadata(a *action, resource string) { for key, values := range a.req.Header { key = http.CanonicalHeaderKey(key) if metaHeaders[key] || strings.HasPrefix(key, "X-"+strings.Title(resource)+"-Meta-") { if values[0] != "" || resource == "object" { m.meta[key] = values } else { m.meta.Del(key) } } } } func (m metadata) getMetadata(a *action) { h := a.w.Header() for name, d := range m.meta { h[name] = d } } func (c *container) list(delimiter string, marker string, prefix string, parent string) (resp []interface{}) { var tmp orderedObjects c.RLock() defer c.RUnlock() // first get all matching objects and arrange them in alphabetical order. for _, obj := range c.objects { if strings.HasPrefix(obj.name, prefix) { tmp = append(tmp, obj) } } sort.Sort(tmp) var prefixes []string for _, obj := range tmp { if !strings.HasPrefix(obj.name, prefix) { continue } isPrefix := false name := obj.name if parent != "" { if path.Dir(obj.name) != path.Clean(parent) { continue } } else if delimiter != "" { if i := strings.Index(obj.name[len(prefix):], delimiter); i >= 0 { name = obj.name[:len(prefix)+i+len(delimiter)] if prefixes != nil && prefixes[len(prefixes)-1] == name { continue } isPrefix = true } } if name <= marker { continue } if isPrefix { prefixes = append(prefixes, name) resp = append(resp, Subdir{ Subdir: name, }) } else { resp = append(resp, obj) } } return } // GET on a container lists the objects in the container. func (r containerResource) get(a *action) interface{} { if r.container == nil { fatalf(404, "NoSuchContainer", "The specified container does not exist") } r.container.RLock() delimiter := a.req.Form.Get("delimiter") marker := a.req.Form.Get("marker") prefix := a.req.Form.Get("prefix") format := a.req.URL.Query().Get("format") parent := a.req.Form.Get("path") a.w.Header().Set("X-Container-Bytes-Used", strconv.Itoa(int(r.container.bytes))) a.w.Header().Set("X-Container-Object-Count", strconv.Itoa(len(r.container.objects))) r.container.getMetadata(a) if a.req.Method == "HEAD" { r.container.RUnlock() return nil } r.container.RUnlock() objects := r.container.list(delimiter, marker, prefix, parent) if format == "json" { a.w.Header().Set("Content-Type", "application/json") var resp []interface{} for _, item := range objects { if obj, ok := item.(*object); ok { resp = append(resp, obj.Key()) } else { resp = append(resp, item) } } return resp } else { for _, item := range objects { if obj, ok := item.(*object); ok { a.w.Write([]byte(obj.name + "\n")) } else if subdir, ok := item.(Subdir); ok { a.w.Write([]byte(subdir.Subdir + "\n")) } } return nil } } // orderedContainers holds a slice of containers that can be sorted // by name. type orderedContainers []*container func (s orderedContainers) Len() int { return len(s) } func (s orderedContainers) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s orderedContainers) Less(i, j int) bool { return s[i].name < s[j].name } func (r containerResource) delete(a *action) interface{} { b := r.container if b == nil { fatalf(404, "NoSuchContainer", "The specified container does not exist") } if len(b.objects) > 0 { fatalf(409, "Conflict", "The container you tried to delete is not empty") } a.user.Lock() delete(a.user.Containers, b.name) a.user.swiftaccount.Containers-- a.user.Unlock() return nil } func (r containerResource) put(a *action) interface{} { if r.container == nil { if !validContainerName(r.name) { fatalf(400, "InvalidContainerName", "The specified container is not valid") } r.container = &container{ name: r.name, objects: make(map[string]*object), metadata: metadata{ meta: make(http.Header), }, } r.container.setMetadata(a, "container") a.user.Lock() a.user.Containers[r.name] = r.container a.user.swiftaccount.Containers++ a.user.Unlock() } if format := a.req.URL.Query().Get("extract-archive"); format != "" { _, _, objectName, _ := a.srv.parseURL(a.req.URL) data, err := ioutil.ReadAll(a.req.Body) if err != nil { fatalf(400, "TODO", "read error") } if a.req.ContentLength >= 0 && int64(len(data)) != a.req.ContentLength { fatalf(400, "IncompleteBody", "You did not provide the number of bytes specified by the Content-Length HTTP header") } dataReader := bytes.NewReader(data) var reader *tar.Reader switch format { case "tar": reader = tar.NewReader(dataReader) case "tar.gz": gzr, err := gzip.NewReader(dataReader) if err != nil { fatalf(400, "TODO", "Invalid tar.gz") } defer gzr.Close() reader = tar.NewReader(gzr) case "tar.bz2": bzr := bzip2.NewReader(dataReader) reader = tar.NewReader(bzr) default: fatalf(400, "TODO", "Invalid format %s", format) } resp := AutoExtractResponse{} for { header, err := reader.Next() if err == io.EOF { break } else if err != nil { //return location, err } if header == nil { continue } if header.Typeflag == tar.TypeDir { continue } var fullPath string if objectName != "" { fullPath = objectName + "/" + header.Name } else { fullPath = header.Name } obj := r.container.objects[fullPath] if obj == nil { // new object obj = &object{ name: fullPath, metadata: metadata{ meta: make(http.Header), }, } atomic.AddInt64(&a.user.Objects, 1) } else { atomic.AddInt64(&r.container.bytes, -header.Size) atomic.AddInt64(&a.user.BytesUsed, -header.Size) } // Default content_type obj.content_type = "application/octet-stream" // handle extended attributes records := getPAXRecords(header) for k, v := range records { ks := strings.SplitN(k, "SCHILY.xattr.user.", 2) if len(ks) < 2 { continue } if ks[1] == "mime_type" { obj.content_type = v } if strings.HasPrefix(ks[1], "meta.") { meta := strings.TrimLeft(ks[1], "meta.") obj.meta["X-Object-Meta-"+strings.Title(meta)] = []string{v} } } sum := md5.New() objData, err := ioutil.ReadAll(io.TeeReader(reader, sum)) if err != nil { errArr := []string{fullPath, fmt.Sprintf("read error: %v", err)} resp.Errors = append(resp.Errors, errArr) continue } gotHash := sum.Sum(nil) obj.data = objData obj.checksum = gotHash obj.mtime = time.Now().UTC() r.container.Lock() r.container.objects[fullPath] = obj r.container.bytes += header.Size r.container.Unlock() atomic.AddInt64(&a.user.BytesUsed, header.Size) atomic.AddInt64(&resp.CreatedFiles, 1) } resp.Status = "201 Accepted" status := 201 if len(resp.Errors) > 0 { resp.Status = "400 Error" status = 400 } a.w.Header().Set("Content-Type", "application/json") a.w.WriteHeader(status) jsonMarshal(a.w, resp) } return nil } func (r containerResource) post(a *action) interface{} { if r.container == nil { fatalf(400, "Method", "The resource could not be found.") } else { r.container.RLock() defer r.container.RUnlock() r.container.setMetadata(a, "container") a.w.WriteHeader(201) jsonMarshal(a.w, Folder{ Count: int64(len(r.container.objects)), Bytes: r.container.bytes, Name: r.container.name, }) } return nil } func (containerResource) copy(a *action) interface{} { return notAllowed() } func getPAXRecords(h *tar.Header) map[string]string { rHeader := reflect.ValueOf(h) // Try PAXRecords - go 1.10 paxField := rHeader.Elem().FieldByName("PAXRecords") if paxField.IsValid() { return paxField.Interface().(map[string]string) } // Try Xattrs - go 1.3 xAttrsField := rHeader.Elem().FieldByName("Xattrs") if xAttrsField.IsValid() { return xAttrsField.Interface().(map[string]string) } return map[string]string{} } // validContainerName returns whether name is a valid bucket name. // Here are the rules, from: // http://docs.openstack.org/api/openstack-object-storage/1.0/content/ch_object-storage-dev-api-storage.html // // Container names cannot exceed 256 bytes and cannot contain the / character. // func validContainerName(name string) bool { if len(name) == 0 || len(name) > 256 { return false } for _, r := range name { switch { case r == '/': return false default: } } return true } // orderedObjects holds a slice of objects that can be sorted // by name. type orderedObjects []*object func (s orderedObjects) Len() int { return len(s) } func (s orderedObjects) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s orderedObjects) Less(i, j int) bool { return s[i].name < s[j].name } func (obj *object) Key() Key { return Key{ Key: obj.name, LastModified: obj.mtime.Format("2006-01-02T15:04:05"), Size: int64(len(obj.data)), ETag: fmt.Sprintf("%x", obj.checksum), ContentType: obj.content_type, } } var metaHeaders = map[string]bool{ "Content-Type": true, "Content-Encoding": true, "Content-Disposition": true, "X-Object-Manifest": true, "X-Static-Large-Object": true, } var rangeRegexp = regexp.MustCompile("(bytes=)?([0-9]*)-([0-9]*)") // GET on an object gets the contents of the object. func (objr objectResource) get(a *action) interface{} { var ( etag []byte reader io.Reader start int end int = -1 ) obj := objr.object if obj == nil { fatalf(404, "Not Found", "The resource could not be found.") } obj.RLock() defer obj.RUnlock() h := a.w.Header() // add metadata obj.getMetadata(a) if r := a.req.Header.Get("Range"); r != "" { m := rangeRegexp.FindStringSubmatch(r) if m[2] != "" { start, _ = strconv.Atoi(m[2]) } if m[3] != "" { end, _ = strconv.Atoi(m[3]) } } max := func(a int, b int) int { if a > b { return a } return b } if manifest, ok := obj.meta["X-Object-Manifest"]; ok { var segments []io.Reader components := strings.SplitN(manifest[0], "/", 2) a.user.RLock() segContainer := a.user.Containers[components[0]] a.user.RUnlock() prefix := components[1] resp := segContainer.list("", "", prefix, "") sum := md5.New() cursor := 0 size := 0 for _, item := range resp { if obj, ok := item.(*object); ok { length := len(obj.data) size += length sum.Write([]byte(hex.EncodeToString(obj.checksum))) if start >= cursor+length { continue } segments = append(segments, bytes.NewReader(obj.data[max(0, start-cursor):])) cursor += length } } etag = sum.Sum(nil) if end == -1 { end = size - 1 } reader = io.LimitReader(io.MultiReader(segments...), int64(end-start+1)) } else if value, ok := obj.meta["X-Static-Large-Object"]; ok && value[0] == "True" && a.req.URL.Query().Get("multipart-manifest") != "get" { var segments []io.Reader var segmentList []segment json.Unmarshal(obj.data, &segmentList) cursor := 0 size := 0 sum := md5.New() for _, segment := range segmentList { components := strings.SplitN(segment.Name[1:], "/", 2) a.user.RLock() segContainer := a.user.Containers[components[0]] a.user.RUnlock() objectName := components[1] segObject := segContainer.objects[objectName] length := len(segObject.data) size += length sum.Write([]byte(hex.EncodeToString(segObject.checksum))) if start >= cursor+length { continue } segments = append(segments, bytes.NewReader(segObject.data[max(0, start-cursor):])) cursor += length } etag = sum.Sum(nil) if end == -1 { end = size - 1 } reader = io.LimitReader(io.MultiReader(segments...), int64(end-start+1)) } else { if end == -1 { end = len(obj.data) - 1 } etag = obj.checksum reader = bytes.NewReader(obj.data[start : end+1]) } etagHex := hex.EncodeToString(etag) if a.req.Header.Get("If-None-Match") == etagHex { a.w.WriteHeader(http.StatusNotModified) return nil } h.Set("Content-Length", fmt.Sprint(end-start+1)) h.Set("ETag", etagHex) h.Set("Last-Modified", obj.mtime.Format(http.TimeFormat)) if a.req.Method == "HEAD" { return nil } // TODO avoid holding the lock when writing data. _, err := io.Copy(a.w, reader) if err != nil { // we can't do much except just log the fact. log.Printf("error writing data: %v", err) } return nil } // PUT on an object creates the object. func (objr objectResource) put(a *action) interface{} { var expectHash []byte if c := a.req.Header.Get("ETag"); c != "" { var err error expectHash, err = hex.DecodeString(c) if err != nil || len(expectHash) != md5.Size { fatalf(400, "InvalidDigest", "The ETag you specified was invalid") } } sum := md5.New() // TODO avoid holding lock while reading data. data, err := ioutil.ReadAll(io.TeeReader(a.req.Body, sum)) if err != nil { fatalf(400, "TODO", "read error") } gotHash := sum.Sum(nil) if expectHash != nil && bytes.Compare(gotHash, expectHash) != 0 { fatalf(422, "Bad ETag", "The ETag you specified did not match what we received") } if a.req.ContentLength >= 0 && int64(len(data)) != a.req.ContentLength { fatalf(400, "IncompleteBody", "You did not provide the number of bytes specified by the Content-Length HTTP header") } // TODO is this correct, or should we erase all previous metadata? obj := objr.object if obj == nil { obj = &object{ name: objr.name, metadata: metadata{ meta: make(http.Header), }, } atomic.AddInt64(&a.user.Objects, 1) } else { atomic.AddInt64(&objr.container.bytes, -int64(len(obj.data))) atomic.AddInt64(&a.user.BytesUsed, -int64(len(obj.data))) } var content_type string if content_type = a.req.Header.Get("Content-Type"); content_type == "" { content_type = mime.TypeByExtension(obj.name) if content_type == "" { content_type = "application/octet-stream" } } if a.req.URL.Query().Get("multipart-manifest") == "put" { // TODO: check the content of the SLO a.req.Header.Set("X-Static-Large-Object", "True") var segments []segment json.Unmarshal(data, &segments) for i := range segments { segments[i].Name = "/" + segments[i].Path segments[i].Path = "" segments[i].Hash = segments[i].Etag segments[i].Etag = "" segments[i].Bytes = segments[i].Size segments[i].Size = 0 } data, _ = json.Marshal(segments) sum = md5.New() sum.Write(data) gotHash = sum.Sum(nil) } // PUT request has been successful - save data and metadata obj.setMetadata(a, "object") obj.content_type = content_type obj.data = data obj.checksum = gotHash obj.mtime = time.Now().UTC() objr.container.Lock() objr.container.objects[objr.name] = obj objr.container.bytes += int64(len(data)) objr.container.Unlock() atomic.AddInt64(&a.user.BytesUsed, int64(len(data))) h := a.w.Header() h.Set("ETag", hex.EncodeToString(obj.checksum)) return nil } func (objr objectResource) delete(a *action) interface{} { if objr.object == nil { fatalf(404, "NoSuchKey", "The specified key does not exist.") } objr.container.Lock() defer objr.container.Unlock() objr.object.Lock() defer objr.object.Unlock() objr.container.bytes -= int64(len(objr.object.data)) delete(objr.container.objects, objr.name) atomic.AddInt64(&a.user.BytesUsed, -int64(len(objr.object.data))) atomic.AddInt64(&a.user.Objects, -1) return nil } func (objr objectResource) post(a *action) interface{} { objr.object.Lock() defer objr.object.Unlock() obj := objr.object obj.setMetadata(a, "object") return nil } func (objr objectResource) copy(a *action) interface{} { if objr.object == nil { fatalf(404, "NoSuchKey", "The specified key does not exist.") } obj := objr.object obj.RLock() defer obj.RUnlock() destination := a.req.Header.Get("Destination") if destination == "" { fatalf(400, "Bad Request", "You must provide a Destination header") } var ( obj2 *object objr2 objectResource ) destURL, _ := url.Parse("/v1/AUTH_" + TEST_ACCOUNT + "/" + destination) r := a.srv.resourceForURL(destURL) switch t := r.(type) { case objectResource: objr2 = t if objr2.object == nil { obj2 = &object{ name: objr2.name, metadata: metadata{ meta: make(http.Header), }, } atomic.AddInt64(&a.user.Objects, 1) } else { obj2 = objr2.object atomic.AddInt64(&objr2.container.bytes, -int64(len(obj2.data))) atomic.AddInt64(&a.user.BytesUsed, -int64(len(obj2.data))) } default: fatalf(400, "Bad Request", "Destination must point to a valid object path") } if objr2.container.name != objr2.container.name && obj2.name != obj.name { obj2.Lock() defer obj2.Unlock() } obj2.content_type = obj.content_type obj2.data = obj.data obj2.checksum = obj.checksum obj2.mtime = time.Now() for key, values := range obj.metadata.meta { obj2.metadata.meta[key] = values } obj2.setMetadata(a, "object") objr2.container.Lock() objr2.container.objects[objr2.name] = obj2 objr2.container.bytes += int64(len(obj.data)) objr2.container.Unlock() atomic.AddInt64(&a.user.BytesUsed, int64(len(obj.data))) return nil } func (s *SwiftServer) serveHTTP(w http.ResponseWriter, req *http.Request) { // ignore error from ParseForm as it's usually spurious. req.ParseForm() if fn := s.override[req.URL.Path]; fn != nil { originalRW := w recorder := httptest.NewRecorder() w = recorder defer func() { fn(originalRW, req, recorder) }() } if DEBUG { log.Printf("swifttest %q %q", req.Method, req.URL) } a := &action{ srv: s, w: w, req: req, reqId: fmt.Sprintf("%09X", atomic.LoadInt64(&s.reqId)), } atomic.AddInt64(&s.reqId, 1) var r resource defer func() { switch err := recover().(type) { case *swiftError: if DEBUG { fmt.Printf("\t%d - %s\n", err.statusCode, err.Message) } w.Header().Set("Content-Type", `text/plain; charset=utf-8`) http.Error(w, err.Message, err.statusCode) case nil: default: if DEBUG { fmt.Printf("\tpanic %s\n", err) } panic(err) } }() var resp interface{} if req.URL.String() == "/v1.0" { username := req.Header.Get("x-auth-user") key := req.Header.Get("x-auth-key") s.Lock() defer s.Unlock() if acct, ok := s.Accounts[username]; ok { if acct.password == key { r := make([]byte, 16) _, _ = rand.Read(r) id := fmt.Sprintf("%X", r) w.Header().Set("X-Storage-Url", s.URL+"/AUTH_"+username) w.Header().Set("X-Auth-Token", "AUTH_tk"+string(id)) w.Header().Set("X-Storage-Token", "AUTH_tk"+string(id)) s.Sessions[id] = &session{ username: username, } return } } panic(notAuthorized()) } if req.URL.String() == "/info" { jsonMarshal(w, &map[string]interface{}{ "swift": map[string]interface{}{ "version": "1.2", }, "tempurl": map[string]interface{}{ "methods": []string{"GET", "HEAD", "PUT"}, }, "slo": map[string]interface{}{ "max_manifest_segments": 1000, "max_manifest_size": 2097152, "min_segment_size": 1, }, }) return } r = s.resourceForURL(req.URL) key := req.Header.Get("x-auth-token") signature := req.URL.Query().Get("temp_url_sig") expires := req.URL.Query().Get("temp_url_expires") if key == "" && signature != "" && expires != "" { accountName, _, _, _ := s.parseURL(req.URL) secretKey := "" s.RLock() if account, ok := s.Accounts[accountName]; ok { secretKey = account.meta.Get("X-Account-Meta-Temp-Url-Key") } s.RUnlock() get_hmac := func(method string) string { mac := hmac.New(sha1.New, []byte(secretKey)) body := fmt.Sprintf("%s\n%s\n%s", method, expires, req.URL.Path) mac.Write([]byte(body)) return hex.EncodeToString(mac.Sum(nil)) } if req.Method == "HEAD" { if signature != get_hmac("GET") && signature != get_hmac("POST") && signature != get_hmac("PUT") { panic(notAuthorized()) } } else if signature != get_hmac(req.Method) { panic(notAuthorized()) } } else { s.RLock() session, ok := s.Sessions[key[7:]] if !ok { s.RUnlock() panic(notAuthorized()) return } a.user = s.Accounts[session.username] s.RUnlock() } switch req.Method { case "PUT": resp = r.put(a) case "GET", "HEAD": resp = r.get(a) case "DELETE": resp = r.delete(a) case "POST": resp = r.post(a) case "COPY": resp = r.copy(a) default: fatalf(400, "MethodNotAllowed", "unknown http request method %q", req.Method) } content_type := req.Header.Get("Content-Type") if resp != nil && req.Method != "HEAD" { if strings.HasPrefix(content_type, "application/json") || req.URL.Query().Get("format") == "json" { jsonMarshal(w, resp) } else { switch r := resp.(type) { case string: w.Write([]byte(r)) default: w.Write(resp.([]byte)) } } } } func (s *SwiftServer) SetOverride(path string, fn HandlerOverrideFunc) { s.override[path] = fn } func (s *SwiftServer) UnsetOverride(path string) { delete(s.override, path) } func jsonMarshal(w io.Writer, x interface{}) { if err := json.NewEncoder(w).Encode(x); err != nil { panic(fmt.Errorf("error marshalling %#v: %v", x, err)) } } var pathRegexp = regexp.MustCompile("/v1/AUTH_([a-zA-Z0-9]+)(/([^/]+)(/(.*))?)?") func (srv *SwiftServer) parseURL(u *url.URL) (account string, container string, object string, err error) { m := pathRegexp.FindStringSubmatch(u.Path) if m == nil { return "", "", "", fmt.Errorf("Couldn't parse the specified URI") } account = m[1] container = m[3] object = m[5] return } // resourceForURL returns a resource object for the given URL. func (srv *SwiftServer) resourceForURL(u *url.URL) (r resource) { accountName, containerName, objectName, err := srv.parseURL(u) if err != nil { fatalf(404, "InvalidURI", err.Error()) } srv.RLock() account, ok := srv.Accounts[accountName] if !ok { //srv.RUnlock() fatalf(404, "NoSuchAccount", "The specified account does not exist") } srv.RUnlock() account.RLock() if containerName == "" { account.RUnlock() return rootResource{} } account.RUnlock() b := containerResource{ name: containerName, container: account.Containers[containerName], } if objectName == "" { return b } if b.container == nil { fatalf(404, "NoSuchContainer", "The specified container does not exist") } objr := objectResource{ name: objectName, version: u.Query().Get("versionId"), container: b.container, } objr.container.RLock() defer objr.container.RUnlock() if obj := objr.container.objects[objr.name]; obj != nil { objr.object = obj } return objr } // nullResource has error stubs for all resource methods. type nullResource struct{} func notAllowed() interface{} { fatalf(400, "MethodNotAllowed", "The specified method is not allowed against this resource") return nil } func notAuthorized() interface{} { fatalf(401, "Unauthorized", "This server could not verify that you are authorized to access the document you requested.") return nil } func (nullResource) put(a *action) interface{} { return notAllowed() } func (nullResource) get(a *action) interface{} { return notAllowed() } func (nullResource) post(a *action) interface{} { return notAllowed() } func (nullResource) delete(a *action) interface{} { return notAllowed() } func (nullResource) copy(a *action) interface{} { return notAllowed() } type rootResource struct{} func (rootResource) put(a *action) interface{} { return notAllowed() } func (rootResource) get(a *action) interface{} { marker := a.req.Form.Get("marker") prefix := a.req.Form.Get("prefix") format := a.req.URL.Query().Get("format") h := a.w.Header() h.Set("X-Account-Bytes-Used", strconv.Itoa(int(atomic.LoadInt64(&a.user.BytesUsed)))) h.Set("X-Account-Container-Count", strconv.Itoa(int(atomic.LoadInt64(&a.user.swiftaccount.Containers)))) h.Set("X-Account-Object-Count", strconv.Itoa(int(atomic.LoadInt64(&a.user.Objects)))) a.user.RLock() defer a.user.RUnlock() // add metadata a.user.metadata.getMetadata(a) if a.req.Method == "HEAD" { return nil } var tmp orderedContainers // first get all matching objects and arrange them in alphabetical order. for _, container := range a.user.Containers { if strings.HasPrefix(container.name, prefix) { tmp = append(tmp, container) } } sort.Sort(tmp) resp := make([]Folder, 0) for _, container := range tmp { if container.name <= marker { continue } if format == "json" { resp = append(resp, Folder{ Count: int64(len(container.objects)), Bytes: container.bytes, Name: container.name, }) } else { a.w.Write([]byte(container.name + "\n")) } } if format == "json" { return resp } else { return nil } } func (r rootResource) post(a *action) interface{} { a.user.Lock() a.user.metadata.setMetadata(a, "account") a.user.Unlock() return nil } func (r rootResource) delete(a *action) interface{} { if a.req.URL.Query().Get("bulk-delete") == "1" { data, err := ioutil.ReadAll(a.req.Body) if err != nil { fatalf(400, "Bad Request", "read error") } var nb, notFound int for _, obj := range strings.Fields(string(data)) { parts := strings.SplitN(obj, "/", 3) if len(parts) < 3 { fatalf(403, "Operation forbidden", "Bulk delete is not supported for containers") } b := containerResource{ name: parts[1], container: a.user.Containers[parts[1]], } if b.container == nil { notFound++ continue } objr := objectResource{ name: parts[2], container: b.container, } objr.container.RLock() if obj := objr.container.objects[objr.name]; obj != nil { objr.object = obj } objr.container.RUnlock() if objr.object == nil { notFound++ continue } objr.container.Lock() objr.object.Lock() objr.container.bytes -= int64(len(objr.object.data)) delete(objr.container.objects, objr.name) objr.object.Unlock() objr.container.Unlock() atomic.AddInt64(&a.user.BytesUsed, -int64(len(objr.object.data))) atomic.AddInt64(&a.user.Objects, -1) nb++ } accept := a.req.Header.Get("Accept") if strings.HasPrefix(accept, "application/json") { a.w.Header().Set("Content-Type", "application/json") resp := map[string]interface{}{ "Number Deleted": nb, "Number Not Found": notFound, "Errors": []string{}, "Response Status": "200 OK", "Response Body": "", } jsonMarshal(a.w, resp) return nil } resp := fmt.Sprintf("Number Deleted: %d\nNumber Not Found: %d\nErrors: \nResponse Status: 200 OK\n", nb, notFound) a.w.Write([]byte(resp)) return nil } return notAllowed() } func (rootResource) copy(a *action) interface{} { return notAllowed() } func NewSwiftServer(address string) (*SwiftServer, error) { if strings.Index(address, ":") == -1 { address += ":0" } l, err := net.Listen("tcp", address) if err != nil { return nil, fmt.Errorf("cannot listen on %s: %v", address, err) } server := &SwiftServer{ Listener: l, AuthURL: "http://" + l.Addr().String() + "/v1.0", URL: "http://" + l.Addr().String() + "/v1", Accounts: make(map[string]*account), Sessions: make(map[string]*session), override: make(map[string]HandlerOverrideFunc), } server.Accounts[TEST_ACCOUNT] = &account{ password: TEST_ACCOUNT, metadata: metadata{ meta: make(http.Header), }, Containers: make(map[string]*container), } go http.Serve(l, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { server.serveHTTP(w, req) })) return server, nil } func (srv *SwiftServer) Close() { srv.Listener.Close() } swift-1.0.52/timeout_reader.go000066400000000000000000000024771365550407400163150ustar00rootroot00000000000000package swift import ( "io" "time" ) // An io.ReadCloser which obeys an idle timeout type timeoutReader struct { reader io.ReadCloser timeout time.Duration cancel func() } // Returns a wrapper around the reader which obeys an idle // timeout. The cancel function is called if the timeout happens func newTimeoutReader(reader io.ReadCloser, timeout time.Duration, cancel func()) *timeoutReader { return &timeoutReader{ reader: reader, timeout: timeout, cancel: cancel, } } // Read reads up to len(p) bytes into p // // Waits at most for timeout for the read to complete otherwise returns a timeout func (t *timeoutReader) Read(p []byte) (int, error) { // FIXME limit the amount of data read in one chunk so as to not exceed the timeout? // Do the read in the background type result struct { n int err error } done := make(chan result, 1) go func() { n, err := t.reader.Read(p) done <- result{n, err} }() // Wait for the read or the timeout timer := time.NewTimer(t.timeout) defer timer.Stop() select { case r := <-done: return r.n, r.err case <-timer.C: t.cancel() return 0, TimeoutError } panic("unreachable") // for Go 1.0 } // Close the channel func (t *timeoutReader) Close() error { return t.reader.Close() } // Check it satisfies the interface var _ io.ReadCloser = &timeoutReader{} swift-1.0.52/timeout_reader_test.go000066400000000000000000000036531365550407400173510ustar00rootroot00000000000000// This tests TimeoutReader package swift import ( "io" "io/ioutil" "sync" "testing" "time" ) // An io.ReadCloser for testing type testReader struct { sync.Mutex n int delay time.Duration closed bool } // Returns n bytes with at time.Duration delay func newTestReader(n int, delay time.Duration) *testReader { return &testReader{ n: n, delay: delay, } } // Returns 1 byte at a time after delay func (t *testReader) Read(p []byte) (n int, err error) { if t.n <= 0 { return 0, io.EOF } time.Sleep(t.delay) p[0] = 'A' t.Lock() t.n-- t.Unlock() return 1, nil } // Close the channel func (t *testReader) Close() error { t.Lock() t.closed = true t.Unlock() return nil } func TestTimeoutReaderNoTimeout(t *testing.T) { test := newTestReader(3, 10*time.Millisecond) cancelled := false cancel := func() { cancelled = true } tr := newTimeoutReader(test, 100*time.Millisecond, cancel) b, err := ioutil.ReadAll(tr) if err != nil || string(b) != "AAA" { t.Fatalf("Bad read %s %s", err, b) } if cancelled { t.Fatal("Cancelled when shouldn't have been") } if test.n != 0 { t.Fatal("Didn't read all") } if test.closed { t.Fatal("Shouldn't be closed") } tr.Close() if !test.closed { t.Fatal("Should be closed") } } func TestTimeoutReaderTimeout(t *testing.T) { // Return those bytes slowly so we get an idle timeout test := newTestReader(3, 100*time.Millisecond) cancelled := false cancel := func() { cancelled = true } tr := newTimeoutReader(test, 10*time.Millisecond, cancel) _, err := ioutil.ReadAll(tr) if err != TimeoutError { t.Fatal("Expecting TimeoutError, got", err) } if !cancelled { t.Fatal("Not cancelled when should have been") } test.Lock() n := test.n test.Unlock() if n == 0 { t.Fatal("Read all") } if n != 3 { t.Fatal("Didn't read any") } if test.closed { t.Fatal("Shouldn't be closed") } tr.Close() if !test.closed { t.Fatal("Should be closed") } } swift-1.0.52/travis_realserver.sh000077500000000000000000000011471365550407400170500ustar00rootroot00000000000000#!/bin/bash set -e if [ "${TRAVIS_PULL_REQUEST}" = "true" ]; then exit 0 fi if [ "${TEST_REAL_SERVER}" = "rackspace" ] && [ ! -z "${RACKSPACE_APIKEY}" ]; then echo "Running tests pointing to Rackspace" export SWIFT_API_KEY=$RACKSPACE_APIKEY export SWIFT_API_USER=$RACKSPACE_USER export SWIFT_AUTH_URL=$RACKSPACE_AUTH go test ./... fi if [ "${TEST_REAL_SERVER}" = "memset" ] && [ ! -z "${MEMSET_APIKEY}" ]; then echo "Running tests pointing to Memset" export SWIFT_API_KEY=$MEMSET_APIKEY export SWIFT_API_USER=$MEMSET_USER export SWIFT_AUTH_URL=$MEMSET_AUTH go test fi swift-1.0.52/watchdog_reader.go000066400000000000000000000024561365550407400164240ustar00rootroot00000000000000package swift import ( "io" "time" ) var watchdogChunkSize = 1 << 20 // 1 MiB // An io.Reader which resets a watchdog timer whenever data is read type watchdogReader struct { timeout time.Duration reader io.Reader timer *time.Timer chunkSize int } // Returns a new reader which will kick the watchdog timer whenever data is read func newWatchdogReader(reader io.Reader, timeout time.Duration, timer *time.Timer) *watchdogReader { return &watchdogReader{ timeout: timeout, reader: reader, timer: timer, chunkSize: watchdogChunkSize, } } // Read reads up to len(p) bytes into p func (t *watchdogReader) Read(p []byte) (int, error) { //read from underlying reader in chunks not larger than t.chunkSize //while resetting the watchdog timer before every read; the small chunk //size ensures that the timer does not fire when reading a large amount of //data from a slow connection start := 0 end := len(p) for start < end { length := end - start if length > t.chunkSize { length = t.chunkSize } resetTimer(t.timer, t.timeout) n, err := t.reader.Read(p[start : start+length]) start += n if n == 0 || err != nil { return start, err } } resetTimer(t.timer, t.timeout) return start, nil } // Check it satisfies the interface var _ io.Reader = &watchdogReader{} swift-1.0.52/watchdog_reader_test.go000066400000000000000000000075231365550407400174630ustar00rootroot00000000000000// This tests WatchdogReader package swift import ( "bytes" "io" "io/ioutil" "testing" "time" ) // Uses testReader from timeout_reader_test.go func testWatchdogReaderTimeout(t *testing.T, initialTimeout, watchdogTimeout time.Duration, expectedTimeout bool) { test := newTestReader(3, 10*time.Millisecond) timer, firedChan := setupTimer(initialTimeout) wr := newWatchdogReader(test, watchdogTimeout, timer) b, err := ioutil.ReadAll(wr) if err != nil || string(b) != "AAA" { t.Fatalf("Bad read %s %s", err, b) } checkTimer(t, firedChan, expectedTimeout) } func setupTimer(initialTimeout time.Duration) (timer *time.Timer, fired <-chan bool) { timer = time.NewTimer(initialTimeout) firedChan := make(chan bool) started := make(chan bool) go func() { started <- true select { case <-timer.C: firedChan <- true } }() <-started return timer, firedChan } func checkTimer(t *testing.T, firedChan <-chan bool, expectedTimeout bool) { fired := false select { case fired = <-firedChan: default: } if expectedTimeout { if !fired { t.Fatal("Timer should have fired") } } else { if fired { t.Fatal("Timer should not have fired") } } } func TestWatchdogReaderNoTimeout(t *testing.T) { testWatchdogReaderTimeout(t, 100*time.Millisecond, 100*time.Millisecond, false) } func TestWatchdogReaderTimeout(t *testing.T) { testWatchdogReaderTimeout(t, 5*time.Millisecond, 5*time.Millisecond, true) } func TestWatchdogReaderNoTimeoutShortInitial(t *testing.T) { testWatchdogReaderTimeout(t, 5*time.Millisecond, 100*time.Millisecond, false) } func TestWatchdogReaderTimeoutLongInitial(t *testing.T) { testWatchdogReaderTimeout(t, 100*time.Millisecond, 5*time.Millisecond, true) } //slowReader simulates reading from a slow network connection by introducing a delay //in each Read() proportional to the amount of bytes read. type slowReader struct { reader io.Reader delayPerByte time.Duration } func (r *slowReader) Read(p []byte) (n int, err error) { n, err = r.reader.Read(p) if n > 0 { time.Sleep(time.Duration(n) * r.delayPerByte) } return } //This test verifies that the watchdogReader's timeout is not triggered by data //that comes in very slowly. (It should only be triggered if no data arrives at //all.) func TestWatchdogReaderOnSlowNetwork(t *testing.T) { byteString := make([]byte, 8*watchdogChunkSize) reader := &slowReader{ reader: bytes.NewReader(byteString), //reading everything at once would take 100 ms, which is longer than the //watchdog timeout below delayPerByte: 200 * time.Millisecond / time.Duration(len(byteString)), } timer, firedChan := setupTimer(100 * time.Millisecond) wr := newWatchdogReader(reader, 190*time.Millisecond, timer) //use io.ReadFull instead of ioutil.ReadAll here because ReadAll already does //some chunking that would keep this testcase from failing b := make([]byte, len(byteString)) n, err := io.ReadFull(wr, b) if err != nil || n != len(b) || !bytes.Equal(b, byteString) { t.Fatalf("Bad read %s %d", err, n) } checkTimer(t, firedChan, false) } //This test verifies that the watchdogReader's chunking logic does not mess up //the byte strings that are read. func TestWatchdogReaderValidity(t *testing.T) { byteString := []byte("abcdefghij") //make a reader with a non-standard chunk size (1 MiB would be much too huge //to comfortably look at the bytestring that comes out of the reader) wr := &watchdogReader{ reader: bytes.NewReader(byteString), chunkSize: 3, //len(byteString) % chunkSize != 0 to be extra rude :) //don't care about the timeout stuff here timeout: 5 * time.Minute, timer: time.NewTimer(5 * time.Minute), } b := make([]byte, len(byteString)) n, err := io.ReadFull(wr, b) if err != nil || n != len(b) { t.Fatalf("Read error: %s", err) } if !bytes.Equal(b, byteString) { t.Fatalf("Bad read: %#v != %#v", string(b), string(byteString)) } }