distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/0000755000175000017500000000000012502424227020557 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/.mailmap0000644000175000017500000000061712502424227022204 0ustar tianontianonStephen J Day Stephen Day Stephen J Day Stephen Day Olivier Gambier Olivier Gambier Brian Bland Brian Bland Josh Hawn Josh Hawn distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/notifications/0000755000175000017500000000000012502424227023430 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/notifications/bridge.go0000644000175000017500000001064612502424227025222 0ustar tianontianonpackage notifications import ( "net/http" "time" "code.google.com/p/go-uuid/uuid" "github.com/docker/distribution" "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest" ) type bridge struct { ub URLBuilder actor ActorRecord source SourceRecord request RequestRecord sink Sink } var _ Listener = &bridge{} // URLBuilder defines a subset of url builder to be used by the event listener. type URLBuilder interface { BuildManifestURL(name, tag string) (string, error) BuildBlobURL(name string, dgst digest.Digest) (string, error) } // NewBridge returns a notification listener that writes records to sink, // using the actor and source. Any urls populated in the events created by // this bridge will be created using the URLBuilder. // TODO(stevvooe): Update this to simply take a context.Context object. func NewBridge(ub URLBuilder, source SourceRecord, actor ActorRecord, request RequestRecord, sink Sink) Listener { return &bridge{ ub: ub, actor: actor, source: source, request: request, sink: sink, } } // NewRequestRecord builds a RequestRecord for use in NewBridge from an // http.Request, associating it with a request id. func NewRequestRecord(id string, r *http.Request) RequestRecord { return RequestRecord{ ID: id, Addr: r.RemoteAddr, Host: r.Host, Method: r.Method, UserAgent: r.UserAgent(), } } func (b *bridge) ManifestPushed(repo distribution.Repository, sm *manifest.SignedManifest) error { return b.createManifestEventAndWrite(EventActionPush, repo, sm) } func (b *bridge) ManifestPulled(repo distribution.Repository, sm *manifest.SignedManifest) error { return b.createManifestEventAndWrite(EventActionPull, repo, sm) } func (b *bridge) ManifestDeleted(repo distribution.Repository, sm *manifest.SignedManifest) error { return b.createManifestEventAndWrite(EventActionDelete, repo, sm) } func (b *bridge) LayerPushed(repo distribution.Repository, layer distribution.Layer) error { return b.createLayerEventAndWrite(EventActionPush, repo, layer) } func (b *bridge) LayerPulled(repo distribution.Repository, layer distribution.Layer) error { return b.createLayerEventAndWrite(EventActionPull, repo, layer) } func (b *bridge) LayerDeleted(repo distribution.Repository, layer distribution.Layer) error { return b.createLayerEventAndWrite(EventActionDelete, repo, layer) } func (b *bridge) createManifestEventAndWrite(action string, repo distribution.Repository, sm *manifest.SignedManifest) error { manifestEvent, err := b.createManifestEvent(action, repo, sm) if err != nil { return err } return b.sink.Write(*manifestEvent) } func (b *bridge) createManifestEvent(action string, repo distribution.Repository, sm *manifest.SignedManifest) (*Event, error) { event := b.createEvent(action) event.Target.MediaType = manifest.ManifestMediaType event.Target.Repository = repo.Name() p, err := sm.Payload() if err != nil { return nil, err } event.Target.Length = int64(len(p)) event.Target.Digest, err = digest.FromBytes(p) if err != nil { return nil, err } // TODO(stevvooe): Currently, the is the "tag" url: once the digest url is // implemented, this should be replaced. event.Target.URL, err = b.ub.BuildManifestURL(sm.Name, sm.Tag) if err != nil { return nil, err } return event, nil } func (b *bridge) createLayerEventAndWrite(action string, repo distribution.Repository, layer distribution.Layer) error { event, err := b.createLayerEvent(action, repo, layer) if err != nil { return err } return b.sink.Write(*event) } func (b *bridge) createLayerEvent(action string, repo distribution.Repository, layer distribution.Layer) (*Event, error) { event := b.createEvent(action) event.Target.MediaType = layerMediaType event.Target.Repository = repo.Name() event.Target.Length = layer.Length() dgst := layer.Digest() event.Target.Digest = dgst var err error event.Target.URL, err = b.ub.BuildBlobURL(repo.Name(), dgst) if err != nil { return nil, err } return event, nil } // createEvent creates an event with actor and source populated. func (b *bridge) createEvent(action string) *Event { event := createEvent(action) event.Source = b.source event.Actor = b.actor event.Request = b.request return event } // createEvent returns a new event, timestamped, with the specified action. func createEvent(action string) *Event { return &Event{ ID: uuid.New(), Timestamp: time.Now(), Action: action, } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/notifications/listener_test.go0000644000175000017500000001077512502424227026655 0ustar tianontianonpackage notifications import ( "io" "reflect" "testing" "github.com/docker/distribution" "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest" "github.com/docker/distribution/registry/storage" "github.com/docker/distribution/registry/storage/driver/inmemory" "github.com/docker/distribution/testutil" "github.com/docker/libtrust" "golang.org/x/net/context" ) func TestListener(t *testing.T) { registry := storage.NewRegistryWithDriver(inmemory.New()) tl := &testListener{ ops: make(map[string]int), } ctx := context.Background() repository, err := registry.Repository(ctx, "foo/bar") if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } repository = Listen(repository, tl) // Now take the registry through a number of operations checkExerciseRepository(t, repository) expectedOps := map[string]int{ "manifest:push": 1, "manifest:pull": 2, // "manifest:delete": 0, // deletes not supported for now "layer:push": 2, "layer:pull": 2, // "layer:delete": 0, // deletes not supported for now } if !reflect.DeepEqual(tl.ops, expectedOps) { t.Fatalf("counts do not match:\n%v\n !=\n%v", tl.ops, expectedOps) } } type testListener struct { ops map[string]int } func (tl *testListener) ManifestPushed(repo distribution.Repository, sm *manifest.SignedManifest) error { tl.ops["manifest:push"]++ return nil } func (tl *testListener) ManifestPulled(repo distribution.Repository, sm *manifest.SignedManifest) error { tl.ops["manifest:pull"]++ return nil } func (tl *testListener) ManifestDeleted(repo distribution.Repository, sm *manifest.SignedManifest) error { tl.ops["manifest:delete"]++ return nil } func (tl *testListener) LayerPushed(repo distribution.Repository, layer distribution.Layer) error { tl.ops["layer:push"]++ return nil } func (tl *testListener) LayerPulled(repo distribution.Repository, layer distribution.Layer) error { tl.ops["layer:pull"]++ return nil } func (tl *testListener) LayerDeleted(repo distribution.Repository, layer distribution.Layer) error { tl.ops["layer:delete"]++ return nil } // checkExerciseRegistry takes the registry through all of its operations, // carrying out generic checks. func checkExerciseRepository(t *testing.T, repository distribution.Repository) { // TODO(stevvooe): This would be a nice testutil function. Basically, it // takes the registry through a common set of operations. This could be // used to make cross-cutting updates by changing internals that affect // update counts. Basically, it would make writing tests a lot easier. tag := "thetag" m := manifest.Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, }, Name: repository.Name(), Tag: tag, } layers := repository.Layers() for i := 0; i < 2; i++ { rs, ds, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating test layer: %v", err) } dgst := digest.Digest(ds) upload, err := layers.Upload() if err != nil { t.Fatalf("error creating layer upload: %v", err) } // Use the resumes, as well! upload, err = layers.Resume(upload.UUID()) if err != nil { t.Fatalf("error resuming layer upload: %v", err) } io.Copy(upload, rs) if _, err := upload.Finish(dgst); err != nil { t.Fatalf("unexpected error finishing upload: %v", err) } m.FSLayers = append(m.FSLayers, manifest.FSLayer{ BlobSum: dgst, }) // Then fetch the layers if _, err := layers.Fetch(dgst); err != nil { t.Fatalf("error fetching layer: %v", err) } } pk, err := libtrust.GenerateECP256PrivateKey() if err != nil { t.Fatalf("unexpected error generating key: %v", err) } sm, err := manifest.Sign(&m, pk) if err != nil { t.Fatalf("unexpected error signing manifest: %v", err) } manifests := repository.Manifests() if err := manifests.Put(sm); err != nil { t.Fatalf("unexpected error putting the manifest: %v", err) } p, err := sm.Payload() if err != nil { t.Fatalf("unexpected error getting manifest payload: %v", err) } dgst, err := digest.FromBytes(p) if err != nil { t.Fatalf("unexpected error digesting manifest payload: %v", err) } fetchedByManifest, err := manifests.Get(dgst) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } if fetchedByManifest.Tag != sm.Tag { t.Fatalf("retrieved unexpected manifest: %v", err) } fetched, err := manifests.GetByTag(tag) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } if fetched.Tag != fetchedByManifest.Tag { t.Fatalf("retrieved unexpected manifest: %v", err) } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/notifications/metrics.go0000644000175000017500000001004512502424227025425 0ustar tianontianonpackage notifications import ( "expvar" "fmt" "net/http" "sync" ) // EndpointMetrics track various actions taken by the endpoint, typically by // number of events. The goal of this to export it via expvar but we may find // some other future solution to be better. type EndpointMetrics struct { Pending int // events pending in queue Events int // total events incoming Successes int // total events written successfully Failures int // total events failed Errors int // total events errored Statuses map[string]int // status code histogram, per call event } // safeMetrics guards the metrics implementation with a lock and provides a // safe update function. type safeMetrics struct { EndpointMetrics sync.Mutex // protects statuses map } // newSafeMetrics returns safeMetrics with map allocated. func newSafeMetrics() *safeMetrics { var sm safeMetrics sm.Statuses = make(map[string]int) return &sm } // httpStatusListener returns the listener for the http sink that updates the // relevent counters. func (sm *safeMetrics) httpStatusListener() httpStatusListener { return &endpointMetricsHTTPStatusListener{ safeMetrics: sm, } } // eventQueueListener returns a listener that maintains queue related counters. func (sm *safeMetrics) eventQueueListener() eventQueueListener { return &endpointMetricsEventQueueListener{ safeMetrics: sm, } } // endpointMetricsHTTPStatusListener increments counters related to http sinks // for the relevent events. type endpointMetricsHTTPStatusListener struct { *safeMetrics } var _ httpStatusListener = &endpointMetricsHTTPStatusListener{} func (emsl *endpointMetricsHTTPStatusListener) success(status int, events ...Event) { emsl.safeMetrics.Lock() defer emsl.safeMetrics.Unlock() emsl.Statuses[fmt.Sprintf("%d %s", status, http.StatusText(status))] += len(events) emsl.Successes += len(events) } func (emsl *endpointMetricsHTTPStatusListener) failure(status int, events ...Event) { emsl.safeMetrics.Lock() defer emsl.safeMetrics.Unlock() emsl.Statuses[fmt.Sprintf("%d %s", status, http.StatusText(status))] += len(events) emsl.Failures += len(events) } func (emsl *endpointMetricsHTTPStatusListener) err(err error, events ...Event) { emsl.safeMetrics.Lock() defer emsl.safeMetrics.Unlock() emsl.Errors += len(events) } // endpointMetricsEventQueueListener maintains the incoming events counter and // the queues pending count. type endpointMetricsEventQueueListener struct { *safeMetrics } func (eqc *endpointMetricsEventQueueListener) ingress(events ...Event) { eqc.Lock() defer eqc.Unlock() eqc.Events += len(events) eqc.Pending += len(events) } func (eqc *endpointMetricsEventQueueListener) egress(events ...Event) { eqc.Lock() defer eqc.Unlock() eqc.Pending -= len(events) } // endpoints is global registry of endpoints used to report metrics to expvar var endpoints struct { registered []*Endpoint mu sync.Mutex } // register places the endpoint into expvar so that stats are tracked. func register(e *Endpoint) { endpoints.mu.Lock() defer endpoints.mu.Unlock() endpoints.registered = append(endpoints.registered, e) } func init() { // NOTE(stevvooe): Setup registry metrics structure to report to expvar. // Ideally, we do more metrics through logging but we need some nice // realtime metrics for queue state for now. registry := expvar.Get("registry") if registry == nil { registry = expvar.NewMap("registry") } var notifications expvar.Map notifications.Init() notifications.Set("endpoints", expvar.Func(func() interface{} { endpoints.mu.Lock() defer endpoints.mu.Unlock() var names []interface{} for _, v := range endpoints.registered { var epjson struct { Name string `json:"name"` URL string `json:"url"` EndpointConfig Metrics EndpointMetrics } epjson.Name = v.Name() epjson.URL = v.URL() epjson.EndpointConfig = v.EndpointConfig v.ReadMetrics(&epjson.Metrics) names = append(names, epjson) } return names })) registry.(*expvar.Map).Set("notifications", ¬ifications) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/notifications/endpoint.go0000644000175000017500000000404512502424227025602 0ustar tianontianonpackage notifications import ( "net/http" "time" ) // EndpointConfig covers the optional configuration parameters for an active // endpoint. type EndpointConfig struct { Headers http.Header Timeout time.Duration Threshold int Backoff time.Duration } // defaults set any zero-valued fields to a reasonable default. func (ec *EndpointConfig) defaults() { if ec.Timeout <= 0 { ec.Timeout = time.Second } if ec.Threshold <= 0 { ec.Threshold = 10 } if ec.Backoff <= 0 { ec.Backoff = time.Second } } // Endpoint is a reliable, queued, thread-safe sink that notify external http // services when events are written. Writes are non-blocking and always // succeed for callers but events may be queued internally. type Endpoint struct { Sink url string name string EndpointConfig metrics *safeMetrics } // NewEndpoint returns a running endpoint, ready to receive events. func NewEndpoint(name, url string, config EndpointConfig) *Endpoint { var endpoint Endpoint endpoint.name = name endpoint.url = url endpoint.EndpointConfig = config endpoint.defaults() endpoint.metrics = newSafeMetrics() // Configures the inmemory queue, retry, http pipeline. endpoint.Sink = newHTTPSink( endpoint.url, endpoint.Timeout, endpoint.Headers, endpoint.metrics.httpStatusListener()) endpoint.Sink = newRetryingSink(endpoint.Sink, endpoint.Threshold, endpoint.Backoff) endpoint.Sink = newEventQueue(endpoint.Sink, endpoint.metrics.eventQueueListener()) register(&endpoint) return &endpoint } // Name returns the name of the endpoint, generally used for debugging. func (e *Endpoint) Name() string { return e.name } // URL returns the url of the endpoint. func (e *Endpoint) URL() string { return e.url } // ReadMetrics populates em with metrics from the endpoint. func (e *Endpoint) ReadMetrics(em *EndpointMetrics) { e.metrics.Lock() defer e.metrics.Unlock() *em = e.metrics.EndpointMetrics // Map still need to copied in a threadsafe manner. em.Statuses = make(map[string]int) for k, v := range e.metrics.Statuses { em.Statuses[k] = v } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/notifications/http.go0000644000175000017500000000717512502424227024750 0ustar tianontianonpackage notifications import ( "bytes" "encoding/json" "fmt" "net/http" "sync" "time" ) // httpSink implements a single-flight, http notification endpoint. This is // very lightweight in that it only makes an attempt at an http request. // Reliability should be provided by the caller. type httpSink struct { url string mu sync.Mutex closed bool client *http.Client listeners []httpStatusListener // TODO(stevvooe): Allow one to configure the media type accepted by this // sink and choose the serialization based on that. } // newHTTPSink returns an unreliable, single-flight http sink. Wrap in other // sinks for increased reliability. func newHTTPSink(u string, timeout time.Duration, headers http.Header, listeners ...httpStatusListener) *httpSink { return &httpSink{ url: u, listeners: listeners, client: &http.Client{ Transport: &headerRoundTripper{ Transport: http.DefaultTransport.(*http.Transport), headers: headers, }, Timeout: timeout, }, } } // httpStatusListener is called on various outcomes of sending notifications. type httpStatusListener interface { success(status int, events ...Event) failure(status int, events ...Event) err(err error, events ...Event) } // Accept makes an attempt to notify the endpoint, returning an error if it // fails. It is the caller's responsibility to retry on error. The events are // accepted or rejected as a group. func (hs *httpSink) Write(events ...Event) error { hs.mu.Lock() defer hs.mu.Unlock() if hs.closed { return ErrSinkClosed } envelope := Envelope{ Events: events, } // TODO(stevvooe): It is not ideal to keep re-encoding the request body on // retry but we are going to do it to keep the code simple. It is likely // we could change the event struct to manage its own buffer. p, err := json.MarshalIndent(envelope, "", " ") if err != nil { for _, listener := range hs.listeners { listener.err(err, events...) } return fmt.Errorf("%v: error marshaling event envelope: %v", hs, err) } body := bytes.NewReader(p) resp, err := hs.client.Post(hs.url, EventsMediaType, body) if err != nil { for _, listener := range hs.listeners { listener.err(err, events...) } return fmt.Errorf("%v: error posting: %v", hs, err) } // The notifier will treat any 2xx or 3xx response as accepted by the // endpoint. switch { case resp.StatusCode >= 200 && resp.StatusCode < 400: for _, listener := range hs.listeners { listener.success(resp.StatusCode, events...) } // TODO(stevvooe): This is a little accepting: we may want to support // unsupported media type responses with retries using the correct // media type. There may also be cases that will never work. return nil default: for _, listener := range hs.listeners { listener.failure(resp.StatusCode, events...) } return fmt.Errorf("%v: response status %v unaccepted", hs, resp.Status) } } // Close the endpoint func (hs *httpSink) Close() error { hs.mu.Lock() defer hs.mu.Unlock() if hs.closed { return fmt.Errorf("httpsink: already closed") } hs.closed = true return nil } func (hs *httpSink) String() string { return fmt.Sprintf("httpSink{%s}", hs.url) } type headerRoundTripper struct { *http.Transport // must be transport to support CancelRequest headers http.Header } func (hrt *headerRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { var nreq http.Request nreq = *req nreq.Header = make(http.Header) merge := func(headers http.Header) { for k, v := range headers { nreq.Header[k] = append(nreq.Header[k], v...) } } merge(req.Header) merge(hrt.headers) return hrt.Transport.RoundTrip(&nreq) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/notifications/http_test.go0000644000175000017500000000757412502424227026012 0ustar tianontianonpackage notifications import ( "encoding/json" "fmt" "mime" "net/http" "net/http/httptest" "reflect" "strconv" "testing" "github.com/docker/distribution/manifest" ) // TestHTTPSink mocks out an http endpoint and notifies it under a couple of // conditions, ensuring correct behavior. func TestHTTPSink(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() if r.Method != "POST" { w.WriteHeader(http.StatusMethodNotAllowed) t.Fatalf("unexpected request method: %v", r.Method) return } // Extract the content type and make sure it matches contentType := r.Header.Get("Content-Type") mediaType, _, err := mime.ParseMediaType(contentType) if err != nil { w.WriteHeader(http.StatusBadRequest) t.Fatalf("error parsing media type: %v, contenttype=%q", err, contentType) return } if mediaType != EventsMediaType { w.WriteHeader(http.StatusUnsupportedMediaType) t.Fatalf("incorrect media type: %q != %q", mediaType, EventsMediaType) return } var envelope Envelope dec := json.NewDecoder(r.Body) if err := dec.Decode(&envelope); err != nil { w.WriteHeader(http.StatusBadRequest) t.Fatalf("error decoding request body: %v", err) return } // Let caller choose the status status, err := strconv.Atoi(r.FormValue("status")) if err != nil { t.Logf("error parsing status: %v", err) // May just be empty, set status to 200 status = http.StatusOK } w.WriteHeader(status) })) metrics := newSafeMetrics() sink := newHTTPSink(server.URL, 0, nil, &endpointMetricsHTTPStatusListener{safeMetrics: metrics}) var expectedMetrics EndpointMetrics expectedMetrics.Statuses = make(map[string]int) for _, tc := range []struct { events []Event // events to send url string failure bool // true if there should be a failure. statusCode int // if not set, no status code should be incremented. }{ { statusCode: http.StatusOK, events: []Event{ createTestEvent("push", "library/test", manifest.ManifestMediaType)}, }, { statusCode: http.StatusOK, events: []Event{ createTestEvent("push", "library/test", manifest.ManifestMediaType), createTestEvent("push", "library/test", layerMediaType), createTestEvent("push", "library/test", layerMediaType), }, }, { statusCode: http.StatusTemporaryRedirect, }, { statusCode: http.StatusBadRequest, failure: true, }, { // Case where connection never goes through. url: "http://shoudlntresolve/", failure: true, }, } { if tc.failure { expectedMetrics.Failures += len(tc.events) } else { expectedMetrics.Successes += len(tc.events) } if tc.statusCode > 0 { expectedMetrics.Statuses[fmt.Sprintf("%d %s", tc.statusCode, http.StatusText(tc.statusCode))] += len(tc.events) } url := tc.url if url == "" { url = server.URL + "/" } // setup endpoint to respond with expected status code. url += fmt.Sprintf("?status=%v", tc.statusCode) sink.url = url t.Logf("testcase: %v, fail=%v", url, tc.failure) // Try a simple event emission. err := sink.Write(tc.events...) if !tc.failure { if err != nil { t.Fatalf("unexpected error send event: %v", err) } } else { if err == nil { t.Fatalf("the endpoint should have rejected the request") } } if !reflect.DeepEqual(metrics.EndpointMetrics, expectedMetrics) { t.Fatalf("metrics not as expected: %#v != %#v", metrics.EndpointMetrics, expectedMetrics) } } if err := sink.Close(); err != nil { t.Fatalf("unexpected error closing http sink: %v", err) } // double close returns error if err := sink.Close(); err == nil { t.Fatalf("second close should have returned error: %v", err) } } func createTestEvent(action, repo, typ string) Event { event := createEvent(action) event.Target.MediaType = typ event.Target.Repository = repo return *event } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/notifications/listener.go0000644000175000017500000001051012502424227025601 0ustar tianontianonpackage notifications import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution" "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest" ) // ManifestListener describes a set of methods for listening to events related to manifests. type ManifestListener interface { ManifestPushed(repo distribution.Repository, sm *manifest.SignedManifest) error ManifestPulled(repo distribution.Repository, sm *manifest.SignedManifest) error // TODO(stevvooe): Please note that delete support is still a little shaky // and we'll need to propagate these in the future. ManifestDeleted(repo distribution.Repository, sm *manifest.SignedManifest) error } // LayerListener describes a listener that can respond to layer related events. type LayerListener interface { LayerPushed(repo distribution.Repository, layer distribution.Layer) error LayerPulled(repo distribution.Repository, layer distribution.Layer) error // TODO(stevvooe): Please note that delete support is still a little shaky // and we'll need to propagate these in the future. LayerDeleted(repo distribution.Repository, layer distribution.Layer) error } // Listener combines all repository events into a single interface. type Listener interface { ManifestListener LayerListener } type repositoryListener struct { distribution.Repository listener Listener } // Listen dispatches events on the repository to the listener. func Listen(repo distribution.Repository, listener Listener) distribution.Repository { return &repositoryListener{ Repository: repo, listener: listener, } } func (rl *repositoryListener) Manifests() distribution.ManifestService { return &manifestServiceListener{ ManifestService: rl.Repository.Manifests(), parent: rl, } } func (rl *repositoryListener) Layers() distribution.LayerService { return &layerServiceListener{ LayerService: rl.Repository.Layers(), parent: rl, } } type manifestServiceListener struct { distribution.ManifestService parent *repositoryListener } func (msl *manifestServiceListener) Get(dgst digest.Digest) (*manifest.SignedManifest, error) { sm, err := msl.ManifestService.Get(dgst) if err == nil { if err := msl.parent.listener.ManifestPulled(msl.parent.Repository, sm); err != nil { logrus.Errorf("error dispatching manifest pull to listener: %v", err) } } return sm, err } func (msl *manifestServiceListener) Put(sm *manifest.SignedManifest) error { err := msl.ManifestService.Put(sm) if err == nil { if err := msl.parent.listener.ManifestPushed(msl.parent.Repository, sm); err != nil { logrus.Errorf("error dispatching manifest push to listener: %v", err) } } return err } func (msl *manifestServiceListener) GetByTag(tag string) (*manifest.SignedManifest, error) { sm, err := msl.ManifestService.GetByTag(tag) if err == nil { if err := msl.parent.listener.ManifestPulled(msl.parent.Repository, sm); err != nil { logrus.Errorf("error dispatching manifest pull to listener: %v", err) } } return sm, err } type layerServiceListener struct { distribution.LayerService parent *repositoryListener } func (lsl *layerServiceListener) Fetch(dgst digest.Digest) (distribution.Layer, error) { layer, err := lsl.LayerService.Fetch(dgst) if err == nil { if err := lsl.parent.listener.LayerPulled(lsl.parent.Repository, layer); err != nil { logrus.Errorf("error dispatching layer pull to listener: %v", err) } } return layer, err } func (lsl *layerServiceListener) Upload() (distribution.LayerUpload, error) { lu, err := lsl.LayerService.Upload() return lsl.decorateUpload(lu), err } func (lsl *layerServiceListener) Resume(uuid string) (distribution.LayerUpload, error) { lu, err := lsl.LayerService.Resume(uuid) return lsl.decorateUpload(lu), err } func (lsl *layerServiceListener) decorateUpload(lu distribution.LayerUpload) distribution.LayerUpload { return &layerUploadListener{ LayerUpload: lu, parent: lsl, } } type layerUploadListener struct { distribution.LayerUpload parent *layerServiceListener } func (lul *layerUploadListener) Finish(dgst digest.Digest) (distribution.Layer, error) { layer, err := lul.LayerUpload.Finish(dgst) if err == nil { if err := lul.parent.parent.listener.LayerPushed(lul.parent.parent.Repository, layer); err != nil { logrus.Errorf("error dispatching layer push to listener: %v", err) } } return layer, err } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/notifications/sinks_test.go0000644000175000017500000001063112502424227026146 0ustar tianontianonpackage notifications import ( "fmt" "math/rand" "sync" "time" "github.com/Sirupsen/logrus" "testing" ) func TestBroadcaster(t *testing.T) { const nEvents = 1000 var sinks []Sink for i := 0; i < 10; i++ { sinks = append(sinks, &testSink{}) } b := NewBroadcaster(sinks...) var block []Event var wg sync.WaitGroup for i := 1; i <= nEvents; i++ { block = append(block, createTestEvent("push", "library/test", "blob")) if i%10 == 0 && i > 0 { wg.Add(1) go func(block ...Event) { if err := b.Write(block...); err != nil { t.Fatalf("error writing block of length %d: %v", len(block), err) } wg.Done() }(block...) block = nil } } wg.Wait() // Wait until writes complete checkClose(t, b) // Iterate through the sinks and check that they all have the expected length. for _, sink := range sinks { ts := sink.(*testSink) ts.mu.Lock() defer ts.mu.Unlock() if len(ts.events) != nEvents { t.Fatalf("not all events ended up in testsink: len(testSink) == %d, not %d", len(ts.events), nEvents) } if !ts.closed { t.Fatalf("sink should have been closed") } } } func TestEventQueue(t *testing.T) { const nevents = 1000 var ts testSink metrics := newSafeMetrics() eq := newEventQueue( // delayed sync simulates destination slower than channel comms &delayedSink{ Sink: &ts, delay: time.Millisecond * 1, }, metrics.eventQueueListener()) var wg sync.WaitGroup var block []Event for i := 1; i <= nevents; i++ { block = append(block, createTestEvent("push", "library/test", "blob")) if i%10 == 0 && i > 0 { wg.Add(1) go func(block ...Event) { if err := eq.Write(block...); err != nil { t.Fatalf("error writing event block: %v", err) } wg.Done() }(block...) block = nil } } wg.Wait() checkClose(t, eq) ts.mu.Lock() defer ts.mu.Unlock() metrics.Lock() defer metrics.Unlock() if len(ts.events) != nevents { t.Fatalf("events did not make it to the sink: %d != %d", len(ts.events), 1000) } if !ts.closed { t.Fatalf("sink should have been closed") } if metrics.Events != nevents { t.Fatalf("unexpected ingress count: %d != %d", metrics.Events, nevents) } if metrics.Pending != 0 { t.Fatalf("unexpected egress count: %d != %d", metrics.Pending, 0) } } func TestRetryingSink(t *testing.T) { // Make a sync that fails most of the time, ensuring that all the events // make it through. var ts testSink flaky := &flakySink{ rate: 1.0, // start out always failing. Sink: &ts, } s := newRetryingSink(flaky, 3, 10*time.Millisecond) var wg sync.WaitGroup var block []Event for i := 1; i <= 100; i++ { block = append(block, createTestEvent("push", "library/test", "blob")) // Above 50, set the failure rate lower if i > 50 { s.mu.Lock() flaky.rate = 0.90 s.mu.Unlock() } if i%10 == 0 && i > 0 { wg.Add(1) go func(block ...Event) { defer wg.Done() if err := s.Write(block...); err != nil { t.Fatalf("error writing event block: %v", err) } }(block...) block = nil } } wg.Wait() checkClose(t, s) ts.mu.Lock() defer ts.mu.Unlock() if len(ts.events) != 100 { t.Fatalf("events not propagated: %d != %d", len(ts.events), 100) } } type testSink struct { events []Event mu sync.Mutex closed bool } func (ts *testSink) Write(events ...Event) error { ts.mu.Lock() defer ts.mu.Unlock() ts.events = append(ts.events, events...) return nil } func (ts *testSink) Close() error { ts.mu.Lock() defer ts.mu.Unlock() ts.closed = true logrus.Infof("closing testSink") return nil } type delayedSink struct { Sink delay time.Duration } func (ds *delayedSink) Write(events ...Event) error { time.Sleep(ds.delay) return ds.Sink.Write(events...) } type flakySink struct { Sink rate float64 } func (fs *flakySink) Write(events ...Event) error { if rand.Float64() < fs.rate { return fmt.Errorf("error writing %d events", len(events)) } return fs.Sink.Write(events...) } func checkClose(t *testing.T, sink Sink) { if err := sink.Close(); err != nil { t.Fatalf("unexpected error closing: %v", err) } // second close should not crash but should return an error. if err := sink.Close(); err == nil { t.Fatalf("no error on double close") } // Write after closed should be an error if err := sink.Write([]Event{}...); err == nil { t.Fatalf("write after closed did not have an error") } else if err != ErrSinkClosed { t.Fatalf("error should be ErrSinkClosed") } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/notifications/event.go0000644000175000017500000001221112502424227025075 0ustar tianontianonpackage notifications import ( "fmt" "time" "github.com/docker/distribution" ) // EventAction constants used in action field of Event. const ( EventActionPull = "pull" EventActionPush = "push" EventActionDelete = "delete" ) const ( // EventsMediaType is the mediatype for the json event envelope. If the // Event, ActorRecord, SourceRecord or Envelope structs change, the version // number should be incremented. EventsMediaType = "application/vnd.docker.distribution.events.v1+json" // LayerMediaType is the media type for image rootfs diffs (aka "layers") // used by Docker. We don't expect this to change for quite a while. layerMediaType = "application/vnd.docker.container.image.rootfs.diff+x-gtar" ) // Envelope defines the fields of a json event envelope message that can hold // one or more events. type Envelope struct { // Events make up the contents of the envelope. Events present in a single // envelope are not necessarily related. Events []Event `json:"events,omitempty"` } // TODO(stevvooe): The event type should be separate from the json format. It // should be defined as an interface. Leaving as is for now since we don't // need that at this time. If we make this change, the struct below would be // called "EventRecord". // Event provides the fields required to describe a registry event. type Event struct { // ID provides a unique identifier for the event. ID string `json:"id,omitempty"` // Timestamp is the time at which the event occurred. Timestamp time.Time `json:"timestamp,omitempty"` // Action indicates what action encompasses the provided event. Action string `json:"action,omitempty"` // Target uniquely describes the target of the event. Target struct { // TODO(stevvooe): Use http.DetectContentType for layers, maybe. distribution.Descriptor // Repository identifies the named repository. Repository string `json:"repository,omitempty"` // URL provides a direct link to the content. URL string `json:"url,omitempty"` } `json:"target,omitempty"` // Request covers the request that generated the event. Request RequestRecord `json:"request,omitempty"` // Actor specifies the agent that initiated the event. For most // situations, this could be from the authorizaton context of the request. Actor ActorRecord `json:"actor,omitempty"` // Source identifies the registry node that generated the event. Put // differently, while the actor "initiates" the event, the source // "generates" it. Source SourceRecord `json:"source,omitempty"` } // ActorRecord specifies the agent that initiated the event. For most // situations, this could be from the authorizaton context of the request. // Data in this record can refer to both the initiating client and the // generating request. type ActorRecord struct { // Name corresponds to the subject or username associated with the // request context that generated the event. Name string `json:"name,omitempty"` // TODO(stevvooe): Look into setting a session cookie to get this // without docker daemon. // SessionID // TODO(stevvooe): Push the "Docker-Command" header to replace cookie and // get the actual command. // Command } // RequestRecord covers the request that generated the event. type RequestRecord struct { // ID uniquely identifies the request that initiated the event. ID string `json:"id"` // Addr contains the ip or hostname and possibly port of the client // connection that initiated the event. This is the RemoteAddr from // the standard http request. Addr string `json:"addr,omitempty"` // Host is the externally accessible host name of the registry instance, // as specified by the http host header on incoming requests. Host string `json:"host,omitempty"` // Method has the request method that generated the event. Method string `json:"method"` // UserAgent contains the user agent header of the request. UserAgent string `json:"useragent"` } // SourceRecord identifies the registry node that generated the event. Put // differently, while the actor "initiates" the event, the source "generates" // it. type SourceRecord struct { // Addr contains the ip or hostname and the port of the registry node // that generated the event. Generally, this will be resolved by // os.Hostname() along with the running port. Addr string `json:"addr,omitempty"` // InstanceID identifies a running instance of an application. Changes // after each restart. InstanceID string `json:"instanceID,omitempty"` } var ( // ErrSinkClosed is returned if a write is issued to a sink that has been // closed. If encountered, the error should be considered terminal and // retries will not be successful. ErrSinkClosed = fmt.Errorf("sink: closed") ) // Sink accepts and sends events. type Sink interface { // Write writes one or more events to the sink. If no error is returned, // the caller will assume that all events have been committed and will not // try to send them again. If an error is received, the caller may retry // sending the event. The caller should cede the slice of memory to the // sink and not modify it after calling this method. Write(events ...Event) error // Close the sink, possibly waiting for pending events to flush. Close() error } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/notifications/event_test.go0000644000175000017500000001117512502424227026144 0ustar tianontianonpackage notifications import ( "encoding/json" "strings" "testing" "time" "github.com/docker/distribution/manifest" ) // TestEventJSONFormat provides silly test to detect if the event format or // envelope has changed. If this code fails, the revision of the protocol may // need to be incremented. func TestEventEnvelopeJSONFormat(t *testing.T) { var expected = strings.TrimSpace(` { "events": [ { "id": "asdf-asdf-asdf-asdf-0", "timestamp": "2006-01-02T15:04:05Z", "action": "push", "target": { "mediaType": "application/vnd.docker.distribution.manifest.v1+json", "length": 1, "digest": "sha256:0123456789abcdef0", "repository": "library/test", "url": "http://example.com/v2/library/test/manifests/latest" }, "request": { "id": "asdfasdf", "addr": "client.local", "host": "registrycluster.local", "method": "PUT", "useragent": "test/0.1" }, "actor": { "name": "test-actor" }, "source": { "addr": "hostname.local:port" } }, { "id": "asdf-asdf-asdf-asdf-1", "timestamp": "2006-01-02T15:04:05Z", "action": "push", "target": { "mediaType": "application/vnd.docker.container.image.rootfs.diff+x-gtar", "length": 2, "digest": "tarsum.v2+sha256:0123456789abcdef1", "repository": "library/test", "url": "http://example.com/v2/library/test/manifests/latest" }, "request": { "id": "asdfasdf", "addr": "client.local", "host": "registrycluster.local", "method": "PUT", "useragent": "test/0.1" }, "actor": { "name": "test-actor" }, "source": { "addr": "hostname.local:port" } }, { "id": "asdf-asdf-asdf-asdf-2", "timestamp": "2006-01-02T15:04:05Z", "action": "push", "target": { "mediaType": "application/vnd.docker.container.image.rootfs.diff+x-gtar", "length": 3, "digest": "tarsum.v2+sha256:0123456789abcdef2", "repository": "library/test", "url": "http://example.com/v2/library/test/manifests/latest" }, "request": { "id": "asdfasdf", "addr": "client.local", "host": "registrycluster.local", "method": "PUT", "useragent": "test/0.1" }, "actor": { "name": "test-actor" }, "source": { "addr": "hostname.local:port" } } ] } `) tm, err := time.Parse(time.RFC3339, time.RFC3339[:len(time.RFC3339)-5]) if err != nil { t.Fatalf("error creating time: %v", err) } var prototype Event prototype.Action = EventActionPush prototype.Timestamp = tm prototype.Actor.Name = "test-actor" prototype.Request.ID = "asdfasdf" prototype.Request.Addr = "client.local" prototype.Request.Host = "registrycluster.local" prototype.Request.Method = "PUT" prototype.Request.UserAgent = "test/0.1" prototype.Source.Addr = "hostname.local:port" var manifestPush Event manifestPush = prototype manifestPush.ID = "asdf-asdf-asdf-asdf-0" manifestPush.Target.Digest = "sha256:0123456789abcdef0" manifestPush.Target.Length = int64(1) manifestPush.Target.MediaType = manifest.ManifestMediaType manifestPush.Target.Repository = "library/test" manifestPush.Target.URL = "http://example.com/v2/library/test/manifests/latest" var layerPush0 Event layerPush0 = prototype layerPush0.ID = "asdf-asdf-asdf-asdf-1" layerPush0.Target.Digest = "tarsum.v2+sha256:0123456789abcdef1" layerPush0.Target.Length = 2 layerPush0.Target.MediaType = layerMediaType layerPush0.Target.Repository = "library/test" layerPush0.Target.URL = "http://example.com/v2/library/test/manifests/latest" var layerPush1 Event layerPush1 = prototype layerPush1.ID = "asdf-asdf-asdf-asdf-2" layerPush1.Target.Digest = "tarsum.v2+sha256:0123456789abcdef2" layerPush1.Target.Length = 3 layerPush1.Target.MediaType = layerMediaType layerPush1.Target.Repository = "library/test" layerPush1.Target.URL = "http://example.com/v2/library/test/manifests/latest" var envelope Envelope envelope.Events = append(envelope.Events, manifestPush, layerPush0, layerPush1) p, err := json.MarshalIndent(envelope, "", " ") if err != nil { t.Fatalf("unexpected error marshaling envelope: %v", err) } if string(p) != expected { t.Fatalf("format has changed\n%s\n != \n%s", string(p), expected) } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/notifications/sinks.go0000644000175000017500000002056212502424227025113 0ustar tianontianonpackage notifications import ( "container/list" "fmt" "sync" "time" "github.com/Sirupsen/logrus" ) // NOTE(stevvooe): This file contains definitions for several utility sinks. // Typically, the broadcaster is the only sink that should be required // externally, but others are suitable for export if the need arises. Albeit, // the tight integration with endpoint metrics should be removed. // Broadcaster sends events to multiple, reliable Sinks. The goal of this // component is to dispatch events to configured endpoints. Reliability can be // provided by wrapping incoming sinks. type Broadcaster struct { sinks []Sink events chan []Event closed chan chan struct{} } // NewBroadcaster ... // Add appends one or more sinks to the list of sinks. The broadcaster // behavior will be affected by the properties of the sink. Generally, the // sink should accept all messages and deal with reliability on its own. Use // of EventQueue and RetryingSink should be used here. func NewBroadcaster(sinks ...Sink) *Broadcaster { b := Broadcaster{ sinks: sinks, events: make(chan []Event), closed: make(chan chan struct{}), } // Start the broadcaster go b.run() return &b } // Write accepts a block of events to be dispatched to all sinks. This method // will never fail and should never block (hopefully!). The caller cedes the // slice memory to the broadcaster and should not modify it after calling // write. func (b *Broadcaster) Write(events ...Event) error { select { case b.events <- events: case <-b.closed: return ErrSinkClosed } return nil } // Close the broadcaster, ensuring that all messages are flushed to the // underlying sink before returning. func (b *Broadcaster) Close() error { logrus.Infof("broadcaster: closing") select { case <-b.closed: // already closed return fmt.Errorf("broadcaster: already closed") default: // do a little chan handoff dance to synchronize closing closed := make(chan struct{}) b.closed <- closed close(b.closed) <-closed return nil } } // run is the main broadcast loop, started when the broadcaster is created. // Under normal conditions, it waits for events on the event channel. After // Close is called, this goroutine will exit. func (b *Broadcaster) run() { for { select { case block := <-b.events: for _, sink := range b.sinks { if err := sink.Write(block...); err != nil { logrus.Errorf("broadcaster: error writing events to %v, these events will be lost: %v", sink, err) } } case closing := <-b.closed: // close all the underlying sinks for _, sink := range b.sinks { if err := sink.Close(); err != nil { logrus.Errorf("broadcaster: error closing sink %v: %v", sink, err) } } closing <- struct{}{} logrus.Debugf("broadcaster: closed") return } } } // eventQueue accepts all messages into a queue for asynchronous consumption // by a sink. It is unbounded and thread safe but the sink must be reliable or // events will be dropped. type eventQueue struct { sink Sink events *list.List listeners []eventQueueListener cond *sync.Cond mu sync.Mutex closed bool } // eventQueueListener is called when various events happen on the queue. type eventQueueListener interface { ingress(events ...Event) egress(events ...Event) } // newEventQueue returns a queue to the provided sink. If the updater is non- // nil, it will be called to update pending metrics on ingress and egress. func newEventQueue(sink Sink, listeners ...eventQueueListener) *eventQueue { eq := eventQueue{ sink: sink, events: list.New(), listeners: listeners, } eq.cond = sync.NewCond(&eq.mu) go eq.run() return &eq } // Write accepts the events into the queue, only failing if the queue has // beend closed. func (eq *eventQueue) Write(events ...Event) error { eq.mu.Lock() defer eq.mu.Unlock() if eq.closed { return ErrSinkClosed } for _, listener := range eq.listeners { listener.ingress(events...) } eq.events.PushBack(events) eq.cond.Signal() // signal waiters return nil } // Close shutsdown the event queue, flushing func (eq *eventQueue) Close() error { eq.mu.Lock() defer eq.mu.Unlock() if eq.closed { return fmt.Errorf("eventqueue: already closed") } // set closed flag eq.closed = true eq.cond.Signal() // signal flushes queue eq.cond.Wait() // wait for signal from last flush return eq.sink.Close() } // run is the main goroutine to flush events to the target sink. func (eq *eventQueue) run() { for { block := eq.next() if block == nil { return // nil block means event queue is closed. } if err := eq.sink.Write(block...); err != nil { logrus.Warnf("eventqueue: error writing events to %v, these events will be lost: %v", eq.sink, err) } for _, listener := range eq.listeners { listener.egress(block...) } } } // next encompasses the critical section of the run loop. When the queue is // empty, it will block on the condition. If new data arrives, it will wake // and return a block. When closed, a nil slice will be returned. func (eq *eventQueue) next() []Event { eq.mu.Lock() defer eq.mu.Unlock() for eq.events.Len() < 1 { if eq.closed { eq.cond.Broadcast() return nil } eq.cond.Wait() } front := eq.events.Front() block := front.Value.([]Event) eq.events.Remove(front) return block } // retryingSink retries the write until success or an ErrSinkClosed is // returned. Underlying sink must have p > 0 of succeeding or the sink will // block. Internally, it is a circuit breaker retries to manage reset. // Concurrent calls to a retrying sink are serialized through the sink, // meaning that if one is in-flight, another will not proceed. type retryingSink struct { mu sync.Mutex sink Sink closed bool // circuit breaker hueristics failures struct { threshold int recent int last time.Time backoff time.Duration // time after which we retry after failure. } } type retryingSinkListener interface { active(events ...Event) retry(events ...Event) } // TODO(stevvooe): We are using circuit break here, which actually doesn't // make a whole lot of sense for this use case, since we always retry. Move // this to use bounded exponential backoff. // newRetryingSink returns a sink that will retry writes to a sink, backing // off on failure. Parameters threshold and backoff adjust the behavior of the // circuit breaker. func newRetryingSink(sink Sink, threshold int, backoff time.Duration) *retryingSink { rs := &retryingSink{ sink: sink, } rs.failures.threshold = threshold rs.failures.backoff = backoff return rs } // Write attempts to flush the events to the downstream sink until it succeeds // or the sink is closed. func (rs *retryingSink) Write(events ...Event) error { rs.mu.Lock() defer rs.mu.Unlock() retry: if rs.closed { return ErrSinkClosed } if !rs.proceed() { logrus.Warnf("%v encountered too many errors, backing off", rs.sink) rs.wait(rs.failures.backoff) goto retry } if err := rs.write(events...); err != nil { if err == ErrSinkClosed { // terminal! return err } logrus.Errorf("retryingsink: error writing events: %v, retrying", err) goto retry } return nil } // Close closes the sink and the underlying sink. func (rs *retryingSink) Close() error { rs.mu.Lock() defer rs.mu.Unlock() if rs.closed { return fmt.Errorf("retryingsink: already closed") } rs.closed = true return rs.sink.Close() } // write provides a helper that dispatches failure and success properly. Used // by write as the single-flight write call. func (rs *retryingSink) write(events ...Event) error { if err := rs.sink.Write(events...); err != nil { rs.failure() return err } rs.reset() return nil } // wait backoff time against the sink, unlocking so others can proceed. Should // only be called by methods that currently have the mutex. func (rs *retryingSink) wait(backoff time.Duration) { rs.mu.Unlock() defer rs.mu.Lock() // backoff here time.Sleep(backoff) } // reset marks a succesful call. func (rs *retryingSink) reset() { rs.failures.recent = 0 rs.failures.last = time.Time{} } // failure records a failure. func (rs *retryingSink) failure() { rs.failures.recent++ rs.failures.last = time.Now().UTC() } // proceed returns true if the call should proceed based on circuit breaker // hueristics. func (rs *retryingSink) proceed() bool { return rs.failures.recent < rs.failures.threshold || time.Now().UTC().After(rs.failures.last.Add(rs.failures.backoff)) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/project/0000755000175000017500000000000012502424227022225 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/project/hooks/0000755000175000017500000000000012502424227023350 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/project/hooks/configure-hooks.sh0000755000175000017500000000063512502424227027015 0ustar tianontianon#!/bin/sh cd $(dirname $0) REPO_ROOT=$(git rev-parse --show-toplevel) RESOLVE_REPO_ROOT_STATUS=$? if [ "$RESOLVE_REPO_ROOT_STATUS" -ne "0" ]; then echo -e "Unable to resolve repository root. Error:\n$REPO_ROOT" > /dev/stderr exit $RESOLVE_REPO_ROOT_STATUS fi set -e set -x # Just in case the directory doesn't exist mkdir -p $REPO_ROOT/.git/hooks ln -f -s $(pwd)/pre-commit $REPO_ROOT/.git/hooks/pre-commitdistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/project/hooks/README.md0000644000175000017500000000117512502424227024633 0ustar tianontianonGit Hooks ========= To enforce valid and properly-formatted code, there is CI in place which runs `gofmt`, `golint`, and `go vet` against code in the repository. As an aid to prevent committing invalid code in the first place, a git pre-commit hook has been added to the repository, found in [pre-commit](./pre-commit). As it is impossible to automatically add linked hooks to a git repository, this hook should be linked into your `.git/hooks/pre-commit`, which can be done by running the `configure-hooks.sh` script in this directory. This script is the preferred method of configuring hooks, as it will be updated as more are added.distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/project/hooks/pre-commit0000755000175000017500000000157012502424227025355 0ustar tianontianon#!/bin/sh REPO_ROOT=$(git rev-parse --show-toplevel) RESOLVE_REPO_ROOT_STATUS=$? if [ "$RESOLVE_REPO_ROOT_STATUS" -ne "0" ]; then printf "Unable to resolve repository root. Error:\n%s\n" "$RESOLVE_REPO_ROOT_STATUS" > /dev/stderr exit $RESOLVE_REPO_ROOT_STATUS fi cd $REPO_ROOT GOFMT_ERRORS=$(gofmt -s -l . 2>&1) if [ -n "$GOFMT_ERRORS" ]; then printf 'gofmt failed for the following files:\n%s\n\nPlease run "gofmt -s -l ." in the root of your repository before committing\n' "$GOFMT_ERRORS" > /dev/stderr exit 1 fi GOLINT_ERRORS=$(golint ./... 2>&1) if [ -n "$GOLINT_ERRORS" ]; then printf "golint failed with the following errors:\n%s\n" "$GOLINT_ERRORS" > /dev/stderr exit 1 fi GOVET_ERRORS=$(go vet ./... 2>&1) GOVET_STATUS=$? if [ "$GOVET_STATUS" -ne "0" ]; then printf "govet failed with the following errors:\n%s\n" "$GOVET_ERRORS" > /dev/stderr exit $GOVET_STATUS fi distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/project/dev-image/0000755000175000017500000000000012502424227024063 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/project/dev-image/Dockerfile0000644000175000017500000000112712502424227026056 0ustar tianontianonFROM ubuntu:14.04 ENV GOLANG_VERSION 1.4rc1 ENV GOPATH /var/cache/drone ENV GOROOT /usr/local/go ENV PATH $PATH:$GOROOT/bin:$GOPATH/bin ENV LANG C ENV LC_ALL C RUN apt-get update && apt-get install -y \ wget ca-certificates git mercurial bzr \ --no-install-recommends \ && rm -rf /var/lib/apt/lists/* RUN wget https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz --quiet && \ tar -C /usr/local -xzf go$GOLANG_VERSION.linux-amd64.tar.gz && \ rm go${GOLANG_VERSION}.linux-amd64.tar.gz RUN go get github.com/axw/gocov/gocov github.com/mattn/goveralls github.com/golang/lint/golint distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry.go0000644000175000017500000001272412502424227022764 0ustar tianontianonpackage distribution import ( "io" "net/http" "time" "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest" "golang.org/x/net/context" ) // Registry represents a collection of repositories, addressable by name. type Registry interface { // Repository should return a reference to the named repository. The // registry may or may not have the repository but should always return a // reference. Repository(ctx context.Context, name string) (Repository, error) } // Repository is a named collection of manifests and layers. type Repository interface { // Name returns the name of the repository. Name() string // Manifests returns a reference to this repository's manifest service. Manifests() ManifestService // Layers returns a reference to this repository's layers service. Layers() LayerService // Signatures returns a reference to this repository's signatures service. Signatures() SignatureService } // ManifestService provides operations on image manifests. type ManifestService interface { // Exists returns true if the manifest exists. Exists(dgst digest.Digest) (bool, error) // Get retrieves the identified by the digest, if it exists. Get(dgst digest.Digest) (*manifest.SignedManifest, error) // Delete removes the manifest, if it exists. Delete(dgst digest.Digest) error // Put creates or updates the manifest. Put(manifest *manifest.SignedManifest) error // TODO(stevvooe): The methods after this message should be moved to a // discrete TagService, per active proposals. // Tags lists the tags under the named repository. Tags() ([]string, error) // ExistsByTag returns true if the manifest exists. ExistsByTag(tag string) (bool, error) // GetByTag retrieves the named manifest, if it exists. GetByTag(tag string) (*manifest.SignedManifest, error) // TODO(stevvooe): There are several changes that need to be done to this // interface: // // 1. Allow explicit tagging with Tag(digest digest.Digest, tag string) // 2. Support reading tags with a re-entrant reader to avoid large // allocations in the registry. // 3. Long-term: Provide All() method that lets one scroll through all of // the manifest entries. // 4. Long-term: break out concept of signing from manifests. This is // really a part of the distribution sprint. // 5. Long-term: Manifest should be an interface. This code shouldn't // really be concerned with the storage format. } // LayerService provides operations on layer files in a backend storage. type LayerService interface { // Exists returns true if the layer exists. Exists(digest digest.Digest) (bool, error) // Fetch the layer identifed by TarSum. Fetch(digest digest.Digest) (Layer, error) // Upload begins a layer upload to repository identified by name, // returning a handle. Upload() (LayerUpload, error) // Resume continues an in progress layer upload, returning a handle to the // upload. The caller should seek to the latest desired upload location // before proceeding. Resume(uuid string) (LayerUpload, error) } // Layer provides a readable and seekable layer object. Typically, // implementations are *not* goroutine safe. type Layer interface { // http.ServeContent requires an efficient implementation of // ReadSeeker.Seek(0, os.SEEK_END). io.ReadSeeker io.Closer // Digest returns the unique digest of the blob. Digest() digest.Digest // Length returns the length in bytes of the blob. Length() int64 // CreatedAt returns the time this layer was created. CreatedAt() time.Time // Handler returns an HTTP handler which serves the layer content, whether // by providing a redirect directly to the content, or by serving the // content itself. Handler(r *http.Request) (http.Handler, error) } // LayerUpload provides a handle for working with in-progress uploads. // Instances can be obtained from the LayerService.Upload and // LayerService.Resume. type LayerUpload interface { io.WriteSeeker io.ReaderFrom io.Closer // UUID returns the identifier for this upload. UUID() string // StartedAt returns the time this layer upload was started. StartedAt() time.Time // Finish marks the upload as completed, returning a valid handle to the // uploaded layer. The digest is validated against the contents of the // uploaded layer. Finish(digest digest.Digest) (Layer, error) // Cancel the layer upload process. Cancel() error } // SignatureService provides operations on signatures. type SignatureService interface { // Get retrieves all of the signature blobs for the specified digest. Get(dgst digest.Digest) ([][]byte, error) // Put stores the signature for the provided digest. Put(dgst digest.Digest, signatures ...[]byte) error } // Descriptor describes targeted content. Used in conjunction with a blob // store, a descriptor can be used to fetch, store and target any kind of // blob. The struct also describes the wire protocol format. Fields should // only be added but never changed. type Descriptor struct { // MediaType describe the type of the content. All text based formats are // encoded as utf-8. MediaType string `json:"mediaType,omitempty"` // Length in bytes of content. Length int64 `json:"length,omitempty"` // Digest uniquely identifies the content. A byte stream can be verified // against against this digest. Digest digest.Digest `json:"digest,omitempty"` // NOTE: Before adding a field here, please ensure that all // other options have been exhausted. Much of the type relationships // depend on the simplicity of this type. } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/Godeps/0000755000175000017500000000000012527300127021777 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/Godeps/Readme0000644000175000017500000000021012502424227023111 0ustar tianontianonThis directory tree is generated automatically by godep. Please do not edit. See https://github.com/tools/godep for more information. distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/Godeps/Godeps.json0000644000175000017500000000546412502424227024125 0ustar tianontianon{ "ImportPath": "github.com/docker/distribution", "GoVersion": "go1.4.1", "Packages": [ "./..." ], "Deps": [ { "ImportPath": "code.google.com/p/go-uuid/uuid", "Comment": "null-15", "Rev": "35bc42037350f0078e3c974c6ea690f1926603ab" }, { "ImportPath": "github.com/AdRoll/goamz/aws", "Rev": "d3664b76d90508cdda5a6c92042f26eab5db3103" }, { "ImportPath": "github.com/AdRoll/goamz/cloudfront", "Rev": "d3664b76d90508cdda5a6c92042f26eab5db3103" }, { "ImportPath": "github.com/AdRoll/goamz/s3", "Rev": "d3664b76d90508cdda5a6c92042f26eab5db3103" }, { "ImportPath": "github.com/MSOpenTech/azure-sdk-for-go", "Comment": "v1.2", "Rev": "0fbd37144de3adc2aef74db867c0e15e41c7f74a" }, { "ImportPath": "github.com/Sirupsen/logrus", "Comment": "v0.6.4-12-g467d9d5", "Rev": "467d9d55c2d2c17248441a8fc661561161f40d5e" }, { "ImportPath": "github.com/bugsnag/bugsnag-go", "Comment": "v1.0.2-5-gb1d1530", "Rev": "b1d153021fcd90ca3f080db36bec96dc690fb274" }, { "ImportPath": "github.com/bugsnag/osext", "Rev": "0dd3f918b21bec95ace9dc86c7e70266cfc5c702" }, { "ImportPath": "github.com/bugsnag/panicwrap", "Rev": "e5f9854865b9778a45169fc249e99e338d4d6f27" }, { "ImportPath": "github.com/codegangsta/cli", "Comment": "1.2.0-66-g6086d79", "Rev": "6086d7927ec35315964d9fea46df6c04e6d697c1" }, { "ImportPath": "github.com/docker/docker/pkg/tarsum", "Comment": "v1.4.1-863-g165ea5c", "Rev": "165ea5c158cff3fc40d476ffe233a5ccc03e7d61" }, { "ImportPath": "github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar", "Comment": "v1.4.1-863-g165ea5c", "Rev": "165ea5c158cff3fc40d476ffe233a5ccc03e7d61" }, { "ImportPath": "github.com/docker/libtrust", "Rev": "fa567046d9b14f6aa788882a950d69651d230b21" }, { "ImportPath": "github.com/gorilla/context", "Rev": "14f550f51af52180c2eefed15e5fd18d63c0a64a" }, { "ImportPath": "github.com/gorilla/handlers", "Rev": "0e84b7d810c16aed432217e330206be156bafae0" }, { "ImportPath": "github.com/gorilla/mux", "Rev": "e444e69cbd2e2e3e0749a2f3c717cec491552bbf" }, { "ImportPath": "github.com/yvasiyarov/go-metrics", "Rev": "57bccd1ccd43f94bb17fdd8bf3007059b802f85e" }, { "ImportPath": "github.com/yvasiyarov/gorelic", "Comment": "v0.0.6-8-ga9bba5b", "Rev": "a9bba5b9ab508a086f9a12b8c51fab68478e2128" }, { "ImportPath": "github.com/yvasiyarov/newrelic_platform_go", "Rev": "b21fdbd4370f3717f3bbd2bf41c223bc273068e6" }, { "ImportPath": "golang.org/x/net/context", "Rev": "1dfe7915deaf3f80b962c163b918868d8a6d8974" }, { "ImportPath": "gopkg.in/check.v1", "Rev": "64131543e7896d5bcc6bd5a76287eb75ea96c673" }, { "ImportPath": "gopkg.in/yaml.v2", "Rev": "bef53efd0c76e49e6de55ead051f886bea7e9420" } ] } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/errors.go0000644000175000017500000000577112502424227022434 0ustar tianontianonpackage distribution import ( "fmt" "strings" "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest" ) var ( // ErrLayerExists returned when layer already exists ErrLayerExists = fmt.Errorf("layer exists") // ErrLayerTarSumVersionUnsupported when tarsum is unsupported version. ErrLayerTarSumVersionUnsupported = fmt.Errorf("unsupported tarsum version") // ErrLayerUploadUnknown returned when upload is not found. ErrLayerUploadUnknown = fmt.Errorf("layer upload unknown") // ErrLayerClosed returned when an operation is attempted on a closed // Layer or LayerUpload. ErrLayerClosed = fmt.Errorf("layer closed") ) // ErrRepositoryUnknown is returned if the named repository is not known by // the registry. type ErrRepositoryUnknown struct { Name string } func (err ErrRepositoryUnknown) Error() string { return fmt.Sprintf("unknown respository name=%s", err.Name) } // ErrRepositoryNameInvalid should be used to denote an invalid repository // name. Reason may set, indicating the cause of invalidity. type ErrRepositoryNameInvalid struct { Name string Reason error } func (err ErrRepositoryNameInvalid) Error() string { return fmt.Sprintf("repository name %q invalid: %v", err.Name, err.Reason) } // ErrManifestUnknown is returned if the manifest is not known by the // registry. type ErrManifestUnknown struct { Name string Tag string } func (err ErrManifestUnknown) Error() string { return fmt.Sprintf("unknown manifest name=%s tag=%s", err.Name, err.Tag) } // ErrUnknownManifestRevision is returned when a manifest cannot be found by // revision within a repository. type ErrUnknownManifestRevision struct { Name string Revision digest.Digest } func (err ErrUnknownManifestRevision) Error() string { return fmt.Sprintf("unknown manifest name=%s revision=%s", err.Name, err.Revision) } // ErrManifestUnverified is returned when the registry is unable to verify // the manifest. type ErrManifestUnverified struct{} func (ErrManifestUnverified) Error() string { return fmt.Sprintf("unverified manifest") } // ErrManifestVerification provides a type to collect errors encountered // during manifest verification. Currently, it accepts errors of all types, // but it may be narrowed to those involving manifest verification. type ErrManifestVerification []error func (errs ErrManifestVerification) Error() string { var parts []string for _, err := range errs { parts = append(parts, err.Error()) } return fmt.Sprintf("errors verifying manifest: %v", strings.Join(parts, ",")) } // ErrUnknownLayer returned when layer cannot be found. type ErrUnknownLayer struct { FSLayer manifest.FSLayer } func (err ErrUnknownLayer) Error() string { return fmt.Sprintf("unknown layer %v", err.FSLayer.BlobSum) } // ErrLayerInvalidDigest returned when tarsum check fails. type ErrLayerInvalidDigest struct { Digest digest.Digest Reason error } func (err ErrLayerInvalidDigest) Error() string { return fmt.Sprintf("invalid digest for referenced layer: %v, %v", err.Digest, err.Reason) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/README.md0000644000175000017500000002052512502424227022042 0ustar tianontianon> **Notice:** *This repository hosts experimental components that are > currently under heavy and fast-paced development, not-ready for public > consumption. If you are looking for the stable registry, please head over to > [docker/docker-registry](https://github.com/docker/docker-registry) > instead.* Distribution ============ The Docker toolset to pack, ship, store, and deliver content. The main product of this repository is the new registry implementation for storing and distributing docker images. It supersedes the [docker/docker- registry](https://github.com/docker/docker-registry) project with a new API design, focused around security and performance. The _Distribution_ project has the further long term goal of providing a secure tool chain for distributing content. The specifications, APIs and tools should be as useful with docker as they are without. This repository contains the following components: - **registry (beta):** An implementation of the [Docker Registry HTTP API V2](doc/spec/api.md) for use with docker 1.5+. - **libraries (unstable):** A rich set of libraries for interacting with distribution components. Please see [godoc](http://godoc.org/github.com/docker/distribution) for details. Note that the libraries *are not* considered stable. - **dist (experimental):** An experimental tool to provide distribution oriented functionality without the docker daemon. - **specifications**: _Distribution_ related specifications are available in [doc/spec](doc/spec). - **documentation:** Documentation is available in [doc](doc/overview.md). ### How will this integrate with Docker engine? This project should provide an implementation to a V2 API for use in the Docker core project. The API should be embeddable and simplify the process of securely pulling and pushing content from docker daemons. ### What are the long term goals of the Distribution project? Design a professional grade and extensible content distribution system, that allow users to: * Enjoy an efficient, secured and reliable way to store, manage, package and exchange content * Hack/roll their own on top of healthy open-source components * Implement their own home made solution through good specs, and solid extensions mechanism. Features -------- The new registry implementation provides the following benefits: - faster push and pull - new, more efficient implementation - simplified deployment - pluggable storage backend - webhook notifications Installation ------------ **TODO(stevvooe):** Add the following here: - docker file - binary builds for non-docker environment (test installations, etc.) Configuration ------------- The registry server can be configured with a yaml file. The following is a simple example that can used for local development: ```yaml version: 0.1 loglevel: debug storage: filesystem: rootdirectory: /tmp/registry-dev http: addr: localhost:5000 secret: asecretforlocaldevelopment debug: addr: localhost:5001 ``` The above configures the registry instance to run on port 5000, binding to "localhost", with the debug server enabled. Registry data will be stored in "/tmp/registry-dev". Logging will be in "debug" mode, which is the most verbose. A similar simple configuration is available at [cmd/registry/config.yml], which is generally useful for local development. **TODO(stevvooe): Need a "best practice" configuration overview. Perhaps, we can point to a documentation section. For full details about configuring a registry server, please see [the documentation](doc/configuration.md). ### Upgrading **TODO:** Add a section about upgrading from V1 registry along with link to migrating in documentation. Build ----- If a go development environment is setup, one can use `go get` to install the `registry` command from the current latest: ```sh go get github.com/docker/distribution/cmd/registry ``` The above will install the source repository into the `GOPATH`. The `registry` binary can then be run with the following: ``` $ $GOPATH/bin/registry -version $GOPATH/bin/registry github.com/docker/distribution v2.0.0-alpha.1+unknown ``` The registry can be run with the default config using the following incantantation: ``` $ $GOPATH/bin/registry $GOPATH/src/github.com/docker/distribution/cmd/registry/config.yml INFO[0000] endpoint local-8082 disabled, skipping app.id=34bbec38-a91a-494a-9a3f-b72f9010081f version=v2.0.0-alpha.1+unknown INFO[0000] endpoint local-8083 disabled, skipping app.id=34bbec38-a91a-494a-9a3f-b72f9010081f version=v2.0.0-alpha.1+unknown INFO[0000] listening on :5000 app.id=34bbec38-a91a-494a-9a3f-b72f9010081f version=v2.0.0-alpha.1+unknown INFO[0000] debug server listening localhost:5001 ``` If it is working, one should see the above log messages. ### Repeatable Builds For the full development experience, one should `cd` into `$GOPATH/src/github.com/docker/distribution`. From there, the regular `go` commands, such as `go test`, should work per package (please see [Developing](#developing) if they don't work). A `Makefile` has been provided as a convenience to support repeatable builds. Please install the following into `GOPATH` for it to work: ``` go get github.com/tools/godep github.com/golang/lint/golint ``` **TODO(stevvooe):** Add a `make setup` command to Makefile to run this. Have to think about how to interact with Godeps properly. Once these commands are available in the `GOPATH`, run `make` to get a full build: ``` $ GOPATH=`godep path`:$GOPATH make + clean + fmt + vet + lint + build github.com/docker/docker/vendor/src/code.google.com/p/go/src/pkg/archive/tar github.com/Sirupsen/logrus github.com/docker/libtrust ... github.com/yvasiyarov/gorelic github.com/docker/distribution/registry/handlers github.com/docker/distribution/cmd/registry + test ... ok github.com/docker/distribution/digest 7.875s ok github.com/docker/distribution/manifest 0.028s ok github.com/docker/distribution/notifications 17.322s ? github.com/docker/distribution/registry [no test files] ok github.com/docker/distribution/registry/api/v2 0.101s ? github.com/docker/distribution/registry/auth [no test files] ok github.com/docker/distribution/registry/auth/silly 0.011s ... + /Users/sday/go/src/github.com/docker/distribution/bin/registry + /Users/sday/go/src/github.com/docker/distribution/bin/registry-api-descriptor-template + /Users/sday/go/src/github.com/docker/distribution/bin/dist + binaries ``` The above provides a repeatable build using the contents of the vendored Godeps directory. This includes formatting, vetting, linting, building, testing and generating tagged binaries. We can verify this worked by running the registry binary generated in the "./bin" directory: ```sh $ ./bin/registry -version ./bin/registry github.com/docker/distribution v2.0.0-alpha.2-80-g16d8b2c.m ``` ### Developing The above approaches are helpful for small experimentation. If more complex tasks are at hand, it is recommended to employ the full power of `godep`. The Makefile is designed to have its `GOPATH` defined externally. This allows one to experiment with various development environment setups. This is primarily useful when testing upstream bugfixes, by modifying local code. This can be demonstrated using `godep` to migrate the `GOPATH` to use the specified dependencies. The `GOPATH` can be migrated to the current package versions declared in `Godeps` with the following command: ```sh godep restore ``` > **WARNING:** This command will checkout versions of the code specified in > Godeps/Godeps.json, modifying the contents of `GOPATH`. If this is > undesired, it is recommended to create a workspace devoted to work on the > _Distribution_ project. With a successful run of the above command, one can now use `make` without specifying the `GOPATH`: ```sh $ make ``` If that is successful, standard `go` commands, such as `go test` should work, per package, without issue. Support ------- If any issues are encountered while using the _Distribution_ project, several avenues are available for support: IRC: #docker-distribution on FreeNode Issue Tracker: github.com/docker/distribution/issues Google Groups: https://groups.google.com/a/dockerproject.org/forum/#!forum/distribution Mailing List: docker@dockerproject.org Contribute ---------- Please see [CONTRIBUTING.md](CONTRIBUTING.md). License ------- This project is distributed under [Apache License, Version 2.0](LICENSE.md). distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/CONTRIBUTING.md0000644000175000017500000001161612502424227023015 0ustar tianontianon# Contributing to the registry ## Are you having issues? Please first try any of these support forums before opening an issue: * irc #docker on freenode (archives: [https://botbot.me/freenode/docker/]) * https://forums.docker.com/ * if your problem is with the "hub" (the website and other user-facing components), or about automated builds, then please direct your issues to https://support.docker.com ## So, you found a bug? First check if your problem was already reported in the issue tracker. If it's already there, please refrain from adding "same here" comments - these don't add any value and are only adding useless noise. **Said comments will quite often be deleted at sight**. On the other hand, if you have any technical, relevant information to add, by all means do! Your issue is not there? Then please, create a ticket. If possible the following guidelines should be followed: * try to come up with a minimal, simple to reproduce test-case * try to add a title that describe succinctly the issue * if you are running your own registry, please provide: * registry version * registry launch command used * registry configuration * registry logs * in all cases: * `docker version` and `docker info` * run your docker daemon in debug mode (-D), and provide docker daemon logs ## You have a patch for a known bug, or a small correction? Basic github workflow (fork, patch, make sure the tests pass, PR). ... and some simple rules to ensure quick merge: * clearly point to the issue(s) you want to fix * when possible, prefer multiple (smaller) PRs addressing individual issues over a big one trying to address multiple issues at once * if you need to amend your PR following comments, squash instead of adding more commits ## You want some shiny new feature to be added? Fork the project. Create a new proposal in the folder `open-design/specs`, named `DEP_MY_AWESOME_PROPOSAL.md`, using `open-design/specs/TEMPLATE.md` as a starting point. Then immediately submit this new file as a pull-request, in order to get early feedback. Eventually, you will have to update your proposal to accommodate the feedback you received. Usually, it's not advisable to start working too much on the implementation itself before the proposal receives sufficient feedback, since it can significantly altered (or rejected). Your implementation should then be submitted as a separate PR, that will be reviewed as well. ## Issue and PR labels To keep track of the state of issues and PRs, we've adopted a set of simple labels. The following are currently in use:
Backlog
Issues marked with this label are considered not yet ready for implementation. Either they are untriaged or require futher detail to proceed.
Blocked
If an issue requires further clarification or is blocked on an unresolved dependency, this label should be used.
Sprint
Issues marked with this label are being worked in the current sprint. All required information should be available and design details have been worked out.
In Progress
The issue or PR is being actively worked on by the assignee.
Done
Issues marked with this label are complete. This can be considered a psuedo-label, in that if it is closed, it is considered "Done".
These integrate with waffle.io to show the current status of the project. The project board is available at the following url: https://waffle.io/docker/distribution If an issue or PR is not labeled correctly or you believe it is not in the right state, please contact a maintainer to fix the problem. ## Milestones Issues and PRs should be assigned to relevant milestones. If an issue or PR is assigned a milestone, it should be available by that date. Depending on level of effort, items may be shuffled in or out of milestones. Issues or PRs that don't have a milestone are considered unscheduled. Typically, "In Progress" issues should have a milestone. ## PR Titles PR titles should be lowercased, except for proper noun references (such a method name or type). PR titles should be prefixed with affected directories, comma separated. For example, if a specification is modified, the prefix would be "doc/spec". If the modifications are only in the root, do not include it. If multiple directories are modified, include each, separated by a comma and space. Here are some examples: - doc/spec: move API specification into correct position - context, registry, auth, auth/token, cmd/registry: context aware logging distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/manifest/0000755000175000017500000000000012502424227022365 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/manifest/sign.go0000644000175000017500000000260212502424227023654 0ustar tianontianonpackage manifest import ( "crypto/x509" "encoding/json" "github.com/docker/libtrust" ) // Sign signs the manifest with the provided private key, returning a // SignedManifest. This typically won't be used within the registry, except // for testing. func Sign(m *Manifest, pk libtrust.PrivateKey) (*SignedManifest, error) { p, err := json.MarshalIndent(m, "", " ") if err != nil { return nil, err } js, err := libtrust.NewJSONSignature(p) if err != nil { return nil, err } if err := js.Sign(pk); err != nil { return nil, err } pretty, err := js.PrettySignature("signatures") if err != nil { return nil, err } return &SignedManifest{ Manifest: *m, Raw: pretty, }, nil } // SignWithChain signs the manifest with the given private key and x509 chain. // The public key of the first element in the chain must be the public key // corresponding with the sign key. func SignWithChain(m *Manifest, key libtrust.PrivateKey, chain []*x509.Certificate) (*SignedManifest, error) { p, err := json.MarshalIndent(m, "", " ") if err != nil { return nil, err } js, err := libtrust.NewJSONSignature(p) if err != nil { return nil, err } if err := js.SignWithChain(key, chain); err != nil { return nil, err } pretty, err := js.PrettySignature("signatures") if err != nil { return nil, err } return &SignedManifest{ Manifest: *m, Raw: pretty, }, nil } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/manifest/manifest_test.go0000644000175000017500000000417612502424227025571 0ustar tianontianonpackage manifest import ( "bytes" "encoding/json" "reflect" "testing" "github.com/docker/libtrust" ) type testEnv struct { name, tag string manifest *Manifest signed *SignedManifest pk libtrust.PrivateKey } func TestManifestMarshaling(t *testing.T) { env := genEnv(t) // Check that the Raw field is the same as json.MarshalIndent with these // parameters. p, err := json.MarshalIndent(env.signed, "", " ") if err != nil { t.Fatalf("error marshaling manifest: %v", err) } if !bytes.Equal(p, env.signed.Raw) { t.Fatalf("manifest bytes not equal: %q != %q", string(env.signed.Raw), string(p)) } } func TestManifestUnmarshaling(t *testing.T) { env := genEnv(t) var signed SignedManifest if err := json.Unmarshal(env.signed.Raw, &signed); err != nil { t.Fatalf("error unmarshaling signed manifest: %v", err) } if !reflect.DeepEqual(&signed, env.signed) { t.Fatalf("manifests are different after unmarshaling: %v != %v", signed, env.signed) } } func TestManifestVerification(t *testing.T) { env := genEnv(t) publicKeys, err := Verify(env.signed) if err != nil { t.Fatalf("error verifying manifest: %v", err) } if len(publicKeys) == 0 { t.Fatalf("no public keys found in signature") } var found bool publicKey := env.pk.PublicKey() // ensure that one of the extracted public keys matches the private key. for _, candidate := range publicKeys { if candidate.KeyID() == publicKey.KeyID() { found = true break } } if !found { t.Fatalf("expected public key, %v, not found in verified keys: %v", publicKey, publicKeys) } } func genEnv(t *testing.T) *testEnv { pk, err := libtrust.GenerateECP256PrivateKey() if err != nil { t.Fatalf("error generating test key: %v", err) } name, tag := "foo/bar", "test" m := Manifest{ Versioned: Versioned{ SchemaVersion: 1, }, Name: name, Tag: tag, FSLayers: []FSLayer{ { BlobSum: "asdf", }, { BlobSum: "qwer", }, }, } sm, err := Sign(&m, pk) if err != nil { t.Fatalf("error signing manifest: %v", err) } return &testEnv{ name: name, tag: tag, manifest: &m, signed: sm, pk: pk, } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/manifest/verify.go0000644000175000017500000000155512502424227024226 0ustar tianontianonpackage manifest import ( "crypto/x509" "github.com/Sirupsen/logrus" "github.com/docker/libtrust" ) // Verify verifies the signature of the signed manifest returning the public // keys used during signing. func Verify(sm *SignedManifest) ([]libtrust.PublicKey, error) { js, err := libtrust.ParsePrettySignature(sm.Raw, "signatures") if err != nil { logrus.WithField("err", err).Debugf("(*SignedManifest).Verify") return nil, err } return js.Verify() } // VerifyChains verifies the signature of the signed manifest against the // certificate pool returning the list of verified chains. Signatures without // an x509 chain are not checked. func VerifyChains(sm *SignedManifest, ca *x509.CertPool) ([][]*x509.Certificate, error) { js, err := libtrust.ParsePrettySignature(sm.Raw, "signatures") if err != nil { return nil, err } return js.VerifyChains(ca) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/manifest/manifest.go0000644000175000017500000000743312502424227024531 0ustar tianontianonpackage manifest import ( "encoding/json" "github.com/docker/distribution/digest" "github.com/docker/libtrust" ) // TODO(stevvooe): When we rev the manifest format, the contents of this // package should me moved to manifest/v1. const ( // ManifestMediaType specifies the mediaType for the current version. Note // that for schema version 1, the the media is optionally // "application/json". ManifestMediaType = "application/vnd.docker.distribution.manifest.v1+json" ) // Versioned provides a struct with just the manifest schemaVersion. Incoming // content with unknown schema version can be decoded against this struct to // check the version. type Versioned struct { // SchemaVersion is the image manifest schema that this image follows SchemaVersion int `json:"schemaVersion"` } // Manifest provides the base accessible fields for working with V2 image // format in the registry. type Manifest struct { Versioned // Name is the name of the image's repository Name string `json:"name"` // Tag is the tag of the image specified by this manifest Tag string `json:"tag"` // Architecture is the host architecture on which this image is intended to // run Architecture string `json:"architecture"` // FSLayers is a list of filesystem layer blobSums contained in this image FSLayers []FSLayer `json:"fsLayers"` // History is a list of unstructured historical data for v1 compatibility History []History `json:"history"` } // SignedManifest provides an envelope for a signed image manifest, including // the format sensitive raw bytes. It contains fields to type SignedManifest struct { Manifest // Raw is the byte representation of the ImageManifest, used for signature // verification. The value of Raw must be used directly during // serialization, or the signature check will fail. The manifest byte // representation cannot change or it will have to be re-signed. Raw []byte `json:"-"` } // UnmarshalJSON populates a new ImageManifest struct from JSON data. func (sm *SignedManifest) UnmarshalJSON(b []byte) error { var manifest Manifest if err := json.Unmarshal(b, &manifest); err != nil { return err } sm.Manifest = manifest sm.Raw = make([]byte, len(b), len(b)) copy(sm.Raw, b) return nil } // Payload returns the raw, signed content of the signed manifest. The // contents can be used to calculate the content identifier. func (sm *SignedManifest) Payload() ([]byte, error) { jsig, err := libtrust.ParsePrettySignature(sm.Raw, "signatures") if err != nil { return nil, err } // Resolve the payload in the manifest. return jsig.Payload() } // Signatures returns the signatures as provided by // (*libtrust.JSONSignature).Signatures. The byte slices are opaque jws // signatures. func (sm *SignedManifest) Signatures() ([][]byte, error) { jsig, err := libtrust.ParsePrettySignature(sm.Raw, "signatures") if err != nil { return nil, err } // Resolve the payload in the manifest. return jsig.Signatures() } // MarshalJSON returns the contents of raw. If Raw is nil, marshals the inner // contents. Applications requiring a marshaled signed manifest should simply // use Raw directly, since the the content produced by json.Marshal will be // compacted and will fail signature checks. func (sm *SignedManifest) MarshalJSON() ([]byte, error) { if len(sm.Raw) > 0 { return sm.Raw, nil } // If the raw data is not available, just dump the inner content. return json.Marshal(&sm.Manifest) } // FSLayer is a container struct for BlobSums defined in an image manifest type FSLayer struct { // BlobSum is the tarsum of the referenced filesystem image layer BlobSum digest.Digest `json:"blobSum"` } // History stores unstructured v1 compatibility information type History struct { // V1Compatibility is the raw v1 compatibility information V1Compatibility string `json:"v1Compatibility"` } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/doc.go0000644000175000017500000000046612502424227021661 0ustar tianontianon// Package distribution will define the interfaces for the components of // docker distribution. The goal is to allow users to reliably package, ship // and store content related to docker images. // // This is currently a work in progress. More details are availalbe in the // README.md. package distribution distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/Dockerfile0000644000175000017500000000061412502424227022552 0ustar tianontianonFROM golang:1.4 ENV CONFIG_PATH /etc/docker/registry/config.yml ENV DISTRIBUTION_DIR /go/src/github.com/docker/distribution ENV GOPATH $DISTRIBUTION_DIR/Godeps/_workspace:$GOPATH WORKDIR $DISTRIBUTION_DIR COPY . $DISTRIBUTION_DIR RUN make PREFIX=/go clean binaries RUN mkdir -pv "$(dirname $CONFIG_PATH)" RUN cp -lv ./cmd/registry/config.yml $CONFIG_PATH EXPOSE 5000 CMD registry $CONFIG_PATH distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/MAINTAINERS0000644000175000017500000000026112502424227022253 0ustar tianontianonSolomon Hykes (@shykes) Olivier Gambier (@dmp42) Sam Alba (@samalba) Stephen Day (@stevvooe) distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/context/0000755000175000017500000000000012502424227022243 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/context/http.go0000644000175000017500000001544412502424227023561 0ustar tianontianonpackage context import ( "errors" "net/http" "strings" "sync" "time" "code.google.com/p/go-uuid/uuid" "github.com/gorilla/mux" "golang.org/x/net/context" ) // Common errors used with this package. var ( ErrNoRequestContext = errors.New("no http request in context") ) // WithRequest places the request on the context. The context of the request // is assigned a unique id, available at "http.request.id". The request itself // is available at "http.request". Other common attributes are available under // the prefix "http.request.". If a request is already present on the context, // this method will panic. func WithRequest(ctx context.Context, r *http.Request) context.Context { if ctx.Value("http.request") != nil { // NOTE(stevvooe): This needs to be considered a programming error. It // is unlikely that we'd want to have more than one request in // context. panic("only one request per context") } return &httpRequestContext{ Context: ctx, startedAt: time.Now(), id: uuid.New(), // assign the request a unique. r: r, } } // GetRequest returns the http request in the given context. Returns // ErrNoRequestContext if the context does not have an http request associated // with it. func GetRequest(ctx context.Context) (*http.Request, error) { if r, ok := ctx.Value("http.request").(*http.Request); r != nil && ok { return r, nil } return nil, ErrNoRequestContext } // GetRequestID attempts to resolve the current request id, if possible. An // error is return if it is not available on the context. func GetRequestID(ctx context.Context) string { return GetStringValue(ctx, "http.request.id") } // WithResponseWriter returns a new context and response writer that makes // interesting response statistics available within the context. func WithResponseWriter(ctx context.Context, w http.ResponseWriter) (context.Context, http.ResponseWriter) { irw := &instrumentedResponseWriter{ ResponseWriter: w, Context: ctx, } return irw, irw } // getVarsFromRequest let's us change request vars implementation for testing // and maybe future changes. var getVarsFromRequest = mux.Vars // WithVars extracts gorilla/mux vars and makes them available on the returned // context. Variables are available at keys with the prefix "vars.". For // example, if looking for the variable "name", it can be accessed as // "vars.name". Implementations that are accessing values need not know that // the underlying context is implemented with gorilla/mux vars. func WithVars(ctx context.Context, r *http.Request) context.Context { return &muxVarsContext{ Context: ctx, vars: getVarsFromRequest(r), } } // GetRequestLogger returns a logger that contains fields from the request in // the current context. If the request is not available in the context, no // fields will display. Request loggers can safely be pushed onto the context. func GetRequestLogger(ctx context.Context) Logger { return GetLogger(ctx, "http.request.id", "http.request.method", "http.request.host", "http.request.uri", "http.request.referer", "http.request.useragent", "http.request.remoteaddr", "http.request.contenttype") } // GetResponseLogger reads the current response stats and builds a logger. // Because the values are read at call time, pushing a logger returned from // this function on the context will lead to missing or invalid data. Only // call this at the end of a request, after the response has been written. func GetResponseLogger(ctx context.Context) Logger { l := getLogrusLogger(ctx, "http.response.written", "http.response.status", "http.response.contenttype") duration := Since(ctx, "http.request.startedat") if duration > 0 { l = l.WithField("http.response.duration", duration) } return l } // httpRequestContext makes information about a request available to context. type httpRequestContext struct { context.Context startedAt time.Time id string r *http.Request } // Value returns a keyed element of the request for use in the context. To get // the request itself, query "request". For other components, access them as // "request.". For example, r.RequestURI func (ctx *httpRequestContext) Value(key interface{}) interface{} { if keyStr, ok := key.(string); ok { if keyStr == "http.request" { return ctx.r } if !strings.HasPrefix(keyStr, "http.request.") { goto fallback } parts := strings.Split(keyStr, ".") if len(parts) != 3 { goto fallback } switch parts[2] { case "uri": return ctx.r.RequestURI case "remoteaddr": return ctx.r.RemoteAddr case "method": return ctx.r.Method case "host": return ctx.r.Host case "referer": referer := ctx.r.Referer() if referer != "" { return referer } case "useragent": return ctx.r.UserAgent() case "id": return ctx.id case "startedat": return ctx.startedAt case "contenttype": ct := ctx.r.Header.Get("Content-Type") if ct != "" { return ct } } } fallback: return ctx.Context.Value(key) } type muxVarsContext struct { context.Context vars map[string]string } func (ctx *muxVarsContext) Value(key interface{}) interface{} { if keyStr, ok := key.(string); ok { if keyStr == "vars" { return ctx.vars } if strings.HasPrefix(keyStr, "vars.") { keyStr = strings.TrimPrefix(keyStr, "vars.") } if v, ok := ctx.vars[keyStr]; ok { return v } } return ctx.Context.Value(key) } // instrumentedResponseWriter provides response writer information in a // context. type instrumentedResponseWriter struct { http.ResponseWriter context.Context mu sync.Mutex status int written int64 } func (irw *instrumentedResponseWriter) Write(p []byte) (n int, err error) { n, err = irw.ResponseWriter.Write(p) irw.mu.Lock() irw.written += int64(n) // Guess the likely status if not set. if irw.status == 0 { irw.status = http.StatusOK } irw.mu.Unlock() return } func (irw *instrumentedResponseWriter) WriteHeader(status int) { irw.ResponseWriter.WriteHeader(status) irw.mu.Lock() irw.status = status irw.mu.Unlock() } func (irw *instrumentedResponseWriter) Flush() { if flusher, ok := irw.ResponseWriter.(http.Flusher); ok { flusher.Flush() } } func (irw *instrumentedResponseWriter) Value(key interface{}) interface{} { if keyStr, ok := key.(string); ok { if keyStr == "http.response" { return irw.ResponseWriter } if !strings.HasPrefix(keyStr, "http.response.") { goto fallback } parts := strings.Split(keyStr, ".") if len(parts) != 3 { goto fallback } irw.mu.Lock() defer irw.mu.Unlock() switch parts[2] { case "written": return irw.written case "status": if irw.status != 0 { return irw.status } case "contenttype": contentType := irw.Header().Get("Content-Type") if contentType != "" { return contentType } } } fallback: return irw.Context.Value(key) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/context/util.go0000644000175000017500000000150212502424227023545 0ustar tianontianonpackage context import ( "time" "golang.org/x/net/context" ) // Since looks up key, which should be a time.Time, and returns the duration // since that time. If the key is not found, the value returned will be zero. // This is helpful when inferring metrics related to context execution times. func Since(ctx context.Context, key interface{}) time.Duration { startedAtI := ctx.Value(key) if startedAtI != nil { if startedAt, ok := startedAtI.(time.Time); ok { return time.Since(startedAt) } } return 0 } // GetStringValue returns a string value from the context. The empty string // will be returned if not found. func GetStringValue(ctx context.Context, key string) (value string) { stringi := ctx.Value(key) if stringi != nil { if valuev, ok := stringi.(string); ok { value = valuev } } return value } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/context/doc.go0000644000175000017500000000663312502424227023347 0ustar tianontianon// Package context provides several utilities for working with // golang.org/x/net/context in http requests. Primarily, the focus is on // logging relevent request information but this package is not limited to // that purpose. // // Logging // // The most useful aspect of this package is GetLogger. This function takes // any context.Context interface and returns the current logger from the // context. Canonical usage looks like this: // // GetLogger(ctx).Infof("something interesting happened") // // GetLogger also takes optional key arguments. The keys will be looked up in // the context and reported with the logger. The following example would // return a logger that prints the version with each log message: // // ctx := context.Context(context.Background(), "version", version) // GetLogger(ctx, "version").Infof("this log message has a version field") // // The above would print out a log message like this: // // INFO[0000] this log message has a version field version=v2.0.0-alpha.2.m // // When used with WithLogger, we gain the ability to decorate the context with // loggers that have information from disparate parts of the call stack. // Following from the version example, we can build a new context with the // configured logger such that we always print the version field: // // ctx = WithLogger(ctx, GetLogger(ctx, "version")) // // Since the logger has been pushed to the context, we can now get the version // field for free with our log messages. Future calls to GetLogger on the new // context will have the version field: // // GetLogger(ctx).Infof("this log message has a version field") // // This becomes more powerful when we start stacking loggers. Let's say we // have the version logger from above but also want a request id. Using the // context above, in our request scoped function, we place another logger in // the context: // // ctx = context.WithValue(ctx, "http.request.id", "unique id") // called when building request context // ctx = WithLogger(ctx, GetLogger(ctx, "http.request.id")) // // When GetLogger is called on the new context, "http.request.id" will be // included as a logger field, along with the original "version" field: // // INFO[0000] this log message has a version field http.request.id=unique id version=v2.0.0-alpha.2.m // // Note that this only affects the new context, the previous context, with the // version field, can be used independently. Put another way, the new logger, // added to the request context, is unique to that context and can have // request scoped varaibles. // // HTTP Requests // // This package also contains several methods for working with http requests. // The concepts are very similar to those described above. We simply place the // request in the context using WithRequest. This makes the request variables // available. GetRequestLogger can then be called to get request specific // variables in a log line: // // ctx = WithRequest(ctx, req) // GetRequestLogger(ctx).Infof("request variables") // // Like above, if we want to include the request data in all log messages in // the context, we push the logger to a new context and use that one: // // ctx = WithLogger(ctx, GetRequestLogger(ctx)) // // The concept is fairly powerful and ensures that calls throughout the stack // can be traced in log messages. Using the fields like "http.request.id", one // can analyze call flow for a particular request with a simple grep of the // logs. package context distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/context/http_test.go0000644000175000017500000001042712502424227024614 0ustar tianontianonpackage context import ( "net/http" "reflect" "testing" "time" "golang.org/x/net/context" ) func TestWithRequest(t *testing.T) { var req http.Request start := time.Now() req.Method = "GET" req.Host = "example.com" req.RequestURI = "/test-test" req.Header = make(http.Header) req.Header.Set("Referer", "foo.com/referer") req.Header.Set("User-Agent", "test/0.1") ctx := WithRequest(context.Background(), &req) for _, testcase := range []struct { key string expected interface{} }{ { key: "http.request", expected: &req, }, { key: "http.request.id", }, { key: "http.request.method", expected: req.Method, }, { key: "http.request.host", expected: req.Host, }, { key: "http.request.uri", expected: req.RequestURI, }, { key: "http.request.referer", expected: req.Referer(), }, { key: "http.request.useragent", expected: req.UserAgent(), }, { key: "http.request.remoteaddr", expected: req.RemoteAddr, }, { key: "http.request.startedat", }, } { v := ctx.Value(testcase.key) if v == nil { t.Fatalf("value not found for %q", testcase.key) } if testcase.expected != nil && v != testcase.expected { t.Fatalf("%s: %v != %v", testcase.key, v, testcase.expected) } // Key specific checks! switch testcase.key { case "http.request.id": if _, ok := v.(string); !ok { t.Fatalf("request id not a string: %v", v) } case "http.request.startedat": vt, ok := v.(time.Time) if !ok { t.Fatalf("value not a time: %v", v) } now := time.Now() if vt.After(now) { t.Fatalf("time generated too late: %v > %v", vt, now) } if vt.Before(start) { t.Fatalf("time generated too early: %v < %v", vt, start) } } } } type testResponseWriter struct { flushed bool status int written int64 header http.Header } func (trw *testResponseWriter) Header() http.Header { if trw.header == nil { trw.header = make(http.Header) } return trw.header } func (trw *testResponseWriter) Write(p []byte) (n int, err error) { if trw.status == 0 { trw.status = http.StatusOK } n = len(p) trw.written += int64(n) return } func (trw *testResponseWriter) WriteHeader(status int) { trw.status = status } func (trw *testResponseWriter) Flush() { trw.flushed = true } func TestWithResponseWriter(t *testing.T) { trw := testResponseWriter{} ctx, rw := WithResponseWriter(context.Background(), &trw) if ctx.Value("http.response") != &trw { t.Fatalf("response not available in context: %v != %v", ctx.Value("http.response"), &trw) } if n, err := rw.Write(make([]byte, 1024)); err != nil { t.Fatalf("unexpected error writing: %v", err) } else if n != 1024 { t.Fatalf("unexpected number of bytes written: %v != %v", n, 1024) } if ctx.Value("http.response.status") != http.StatusOK { t.Fatalf("unexpected response status in context: %v != %v", ctx.Value("http.response.status"), http.StatusOK) } if ctx.Value("http.response.written") != int64(1024) { t.Fatalf("unexpected number reported bytes written: %v != %v", ctx.Value("http.response.written"), 1024) } // Make sure flush propagates rw.(http.Flusher).Flush() if !trw.flushed { t.Fatalf("response writer not flushed") } // Write another status and make sure context is correct. This normally // wouldn't work except for in this contrived testcase. rw.WriteHeader(http.StatusBadRequest) if ctx.Value("http.response.status") != http.StatusBadRequest { t.Fatalf("unexpected response status in context: %v != %v", ctx.Value("http.response.status"), http.StatusBadRequest) } } func TestWithVars(t *testing.T) { var req http.Request vars := map[string]string{ "foo": "asdf", "bar": "qwer", } getVarsFromRequest = func(r *http.Request) map[string]string { if r != &req { t.Fatalf("unexpected request: %v != %v", r, req) } return vars } ctx := WithVars(context.Background(), &req) for _, testcase := range []struct { key string expected interface{} }{ { key: "vars", expected: vars, }, { key: "vars.foo", expected: "asdf", }, { key: "vars.bar", expected: "qwer", }, } { v := ctx.Value(testcase.key) if !reflect.DeepEqual(v, testcase.expected) { t.Fatalf("%q: %v != %v", testcase.key, v, testcase.expected) } } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/context/logger.go0000644000175000017500000000470012502424227024052 0ustar tianontianonpackage context import ( "fmt" "github.com/Sirupsen/logrus" "golang.org/x/net/context" ) // Logger provides a leveled-logging interface. type Logger interface { // standard logger methods Print(args ...interface{}) Printf(format string, args ...interface{}) Println(args ...interface{}) Fatal(args ...interface{}) Fatalf(format string, args ...interface{}) Fatalln(args ...interface{}) Panic(args ...interface{}) Panicf(format string, args ...interface{}) Panicln(args ...interface{}) // Leveled methods, from logrus Debug(args ...interface{}) Debugf(format string, args ...interface{}) Debugln(args ...interface{}) Error(args ...interface{}) Errorf(format string, args ...interface{}) Errorln(args ...interface{}) Info(args ...interface{}) Infof(format string, args ...interface{}) Infoln(args ...interface{}) Warn(args ...interface{}) Warnf(format string, args ...interface{}) Warnln(args ...interface{}) } // WithLogger creates a new context with provided logger. func WithLogger(ctx context.Context, logger Logger) context.Context { return context.WithValue(ctx, "logger", logger) } // GetLogger returns the logger from the current context, if present. If one // or more keys are provided, they will be resolved on the context and // included in the logger. While context.Value takes an interface, any key // argument passed to GetLogger will be passed to fmt.Sprint when expanded as // a logging key field. If context keys are integer constants, for example, // its recommended that a String method is implemented. func GetLogger(ctx context.Context, keys ...interface{}) Logger { return getLogrusLogger(ctx, keys...) } // GetLogrusLogger returns the logrus logger for the context. If one more keys // are provided, they will be resolved on the context and included in the // logger. Only use this function if specific logrus functionality is // required. func getLogrusLogger(ctx context.Context, keys ...interface{}) *logrus.Entry { var logger *logrus.Entry // Get a logger, if it is present. loggerInterface := ctx.Value("logger") if loggerInterface != nil { if lgr, ok := loggerInterface.(*logrus.Entry); ok { logger = lgr } } if logger == nil { // If no logger is found, just return the standard logger. logger = logrus.NewEntry(logrus.StandardLogger()) } fields := logrus.Fields{} for _, key := range keys { v := ctx.Value(key) if v != nil { fields[fmt.Sprint(key)] = v } } return logger.WithFields(fields) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/LICENSE0000644000175000017500000002607512502424227021576 0ustar tianontianonApache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/open-design/0000755000175000017500000000000012502424227022767 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/open-design/specs/0000755000175000017500000000000012502424227024104 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/open-design/specs/TEMPLATE.md0000644000175000017500000000213212502424227025637 0ustar tianontianon# DEP #X: Awesome proposal ## Scope This is related to "Foo" (eg: authentication/storage/extension/...). ## Abstract This proposal suggests to add support for "bar". ## User stories "I'm a Hub user, and 'bar' allows me to do baz1" "I'm a FOSS user running my private registry and 'bar' allows me to do baz2" "I'm a company running the registry and 'bar' allows me to do baz3" ## Technology pre-requisites 'bar' can be implemented using: * foobar approach * barfoo concurrent approach ## Dependencies Project depends on baz to be completed (eg: docker engine support, or another registry proposal). ## Technical proposal We are going to do foofoo alongside with some chunks of barbaz. ## Roadmap * YYYY-MM-DD: proposal submitted * YYYY-MM-DD: proposal reviewed and updated * YYYY-MM-DD: implementation started (WIP PR) * YYYY-MM-DD: implementation complete ready for thorough review * YYYY-MM-DD: final PR version * YYYY-MM-DD: implementation merged ## Editors Editors: * my Company, or maybe just me Implementors: * me and my buddies * another team working on a different approach distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/open-design/MANIFESTO.md0000644000175000017500000000132212502424227024634 0ustar tianontianon# The "Distribution" project ## What is this This is a part of the Docker project, or "primitive" that handles the "distribution" of images. ### Punchline Pack. Sign. Ship. Store. Deliver. Verify. ### Technical scope Distribution has tight relations with: * libtrust, providing cryptographical primitives to handle image signing and verification * image format, as transferred over the wire * docker-registry, the server side component that allows storage and retrieval of packed images * authentication and key management APIs, that are used to verify images and access storage services * PKI infrastructure * docker "pull/push client" code gluing all this together - network communication code, tarsum, etc distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/open-design/ROADMAP.md0000644000175000017500000000236712502424227024404 0ustar tianontianon# Roadmap ## 11/24/2014: alpha Design and code: - implements a basic configuration loading mechanism: https://github.com/docker/docker-registry/issues/646 - storage API is frozen, implemented and used: https://github.com/docker/docker-registry/issues/616 - REST API defined and partly implemented: https://github.com/docker/docker-registry/issues/634 - basic logging: https://github.com/docker/docker-registry/issues/635 - auth design is frozen: https://github.com/docker/docker-registry/issues/623 Environment: - some good practice are in place and documented: https://github.com/docker/docker-registry/issues/657 ## 12/22/2014: beta Design and code: - feature freeze - mirroring defined: https://github.com/docker/docker-registry/issues/658 - extension model defined: https://github.com/docker/docker-registry/issues/613 Environment: - doc-driven approach: https://github.com/docker/docker-registry/issues/627 ## 01/12/2015: RC Design and code: - third party drivers and extensions - basic search extension - third-party layers garbage collection scripts - healthcheck endpoints: https://github.com/docker/docker-registry/issues/656 - bugnsnag/new-relic support: https://github.com/docker/docker-registry/issues/680 Environment: - exhaustive test-cases distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/.gitignore0000644000175000017500000000055512502424227022554 0ustar tianontianon# Compiled Object files, Static and Dynamic libs (Shared Objects) *.o *.a *.so # Folders _obj _test # Architecture specific extensions/prefixes *.[568vq] [568vq].out *.cgo1.go *.cgo2.c _cgo_defun.c _cgo_gotypes.go _cgo_export.* _testmain.go *.exe *.test *.prof # never checkin from the bin file (for now) bin/* # Test key files *.pem # Cover profiles *.out distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/AUTHORS0000644000175000017500000000102512502424227021625 0ustar tianontianonAhmet Alp Balkan Andrey Kostov Anton Tiurin Arnaud Porterie Brian Bland David Lawrence Derek McGowan Donald Huang Frederick F. Kautz IV Josh Hawn Olivier Gambier Stephen J Day Tianon Gravi xiekeyang distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/cmd/0000755000175000017500000000000012502424227021322 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/cmd/registry-storagedriver-s3/0000755000175000017500000000000012502424227026373 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/cmd/registry-storagedriver-s3/main.go0000644000175000017500000000117212502424227027647 0ustar tianontianon// +build ignore package main import ( "encoding/json" "os" "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/storage/driver/ipc" "github.com/docker/distribution/registry/storage/driver/s3" ) // An out-of-process S3 driver, intended to be run by ipc.NewDriverClient func main() { parametersBytes := []byte(os.Args[1]) var parameters map[string]string err := json.Unmarshal(parametersBytes, ¶meters) if err != nil { panic(err) } driver, err := s3.FromParameters(parameters) if err != nil { panic(err) } if err := ipc.StorageDriverServer(driver); err != nil { logrus.Fatalln(err) } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/cmd/registry-storagedriver-inmemory/0000755000175000017500000000000012502424227027705 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/cmd/registry-storagedriver-inmemory/main.go0000644000175000017500000000067712502424227031172 0ustar tianontianon// +build ignore package main import ( "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/storage/driver/inmemory" "github.com/docker/distribution/registry/storage/driver/ipc" ) // An out-of-process inmemory driver, intended to be run by ipc.NewDriverClient // This exists primarily for example and testing purposes func main() { if err := ipc.StorageDriverServer(inmemory.New()); err != nil { logrus.Fatalln(err) } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/cmd/dist/0000755000175000017500000000000012502424227022265 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/cmd/dist/main.go0000644000175000017500000000045112502424227023540 0ustar tianontianonpackage main import ( "os" "github.com/codegangsta/cli" ) func main() { app := cli.NewApp() app.Name = "dist" app.Usage = "Package and ship Docker content" app.Action = commandList.Action app.Commands = []cli.Command{ commandList, commandPull, commandPush, } app.Run(os.Args) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/cmd/dist/list.go0000644000175000017500000000031212502424227023563 0ustar tianontianonpackage main import "github.com/codegangsta/cli" var ( commandList = cli.Command{ Name: "images", Usage: "List available images", Action: imageList, } ) func imageList(c *cli.Context) { } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/cmd/dist/push.go0000644000175000017500000000055112502424227023574 0ustar tianontianonpackage main import "github.com/codegangsta/cli" var ( commandPush = cli.Command{ Name: "push", Usage: "Push an image to a registry", Action: imagePush, Flags: []cli.Flag{ cli.StringFlag{ Name: "r,registry", Value: "hub.docker.io", Usage: "Registry to use (e.g.: localhost:5000)", }, }, } ) func imagePush(*cli.Context) { } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/cmd/dist/pull.go0000644000175000017500000000057012502424227023572 0ustar tianontianonpackage main import "github.com/codegangsta/cli" var ( commandPull = cli.Command{ Name: "pull", Usage: "Pull and verify an image from a registry", Action: imagePull, Flags: []cli.Flag{ cli.StringFlag{ Name: "r,registry", Value: "hub.docker.io", Usage: "Registry to use (e.g.: localhost:5000)", }, }, } ) func imagePull(c *cli.Context) { } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/cmd/registry-storagedriver-azure/0000755000175000017500000000000012502424227027174 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/cmd/registry-storagedriver-azure/main.go0000644000175000017500000000124112502424227030445 0ustar tianontianon// +build ignore package main import ( "encoding/json" "os" log "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/storage/driver/azure" "github.com/docker/distribution/registry/storage/driver/ipc" ) // An out-of-process Azure Storage driver, intended to be run by ipc.NewDriverClient func main() { parametersBytes := []byte(os.Args[1]) var parameters map[string]interface{} err := json.Unmarshal(parametersBytes, ¶meters) if err != nil { panic(err) } driver, err := azure.FromParameters(parameters) if err != nil { panic(err) } if err := ipc.StorageDriverServer(driver); err != nil { log.Fatalln("driver error:", err) } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/cmd/registry-api-descriptor-template/0000755000175000017500000000000012502424227027726 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/cmd/registry-api-descriptor-template/main.go0000644000175000017500000000436512502424227031211 0ustar tianontianon// registry-api-descriptor-template uses the APIDescriptor defined in the // api/v2 package to execute templates passed to the command line. // // For example, to generate a new API specification, one would execute the // following command from the repo root: // // $ registry-api-descriptor-template doc/spec/api.md.tmpl > doc/spec/api.md // // The templates are passed in the api/v2.APIDescriptor object. Please see the // package documentation for fields available on that object. The template // syntax is from Go's standard library text/template package. For information // on Go's template syntax, please see golang.org/pkg/text/template. package main import ( "log" "net/http" "os" "path/filepath" "regexp" "text/template" "github.com/docker/distribution/registry/api/v2" ) var spaceRegex = regexp.MustCompile(`\n\s*`) func main() { if len(os.Args) != 2 { log.Fatalln("please specify a template to execute.") } path := os.Args[1] filename := filepath.Base(path) funcMap := template.FuncMap{ "removenewlines": func(s string) string { return spaceRegex.ReplaceAllString(s, " ") }, "statustext": http.StatusText, "prettygorilla": prettyGorillaMuxPath, } tmpl := template.Must(template.New(filename).Funcs(funcMap).ParseFiles(path)) if err := tmpl.Execute(os.Stdout, v2.APIDescriptor); err != nil { log.Fatalln(err) } } // prettyGorillaMuxPath removes the regular expressions from a gorilla/mux // route string, making it suitable for documentation. func prettyGorillaMuxPath(s string) string { // Stateful parser that removes regular expressions from gorilla // routes. It correctly handles balanced bracket pairs. var output string var label string var level int start: if s[0] == '{' { s = s[1:] level++ goto capture } output += string(s[0]) s = s[1:] goto end capture: switch s[0] { case '{': level++ case '}': level-- if level == 0 { s = s[1:] goto label } case ':': s = s[1:] goto skip default: label += string(s[0]) } s = s[1:] goto capture skip: switch s[0] { case '{': level++ case '}': level-- } s = s[1:] if level == 0 { goto label } goto skip label: if label != "" { output += "<" + label + ">" label = "" } end: if s != "" { goto start } return output } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/cmd/registry-storagedriver-filesystem/0000755000175000017500000000000012502424227030232 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/cmd/registry-storagedriver-filesystem/main.go0000644000175000017500000000113112502424227031501 0ustar tianontianon// +build ignore package main import ( "encoding/json" "os" "github.com/Sirupsen/logrus" "github.com/docker/distribution/registry/storage/driver/filesystem" "github.com/docker/distribution/registry/storage/driver/ipc" ) // An out-of-process filesystem driver, intended to be run by ipc.NewDriverClient func main() { parametersBytes := []byte(os.Args[1]) var parameters map[string]string err := json.Unmarshal(parametersBytes, ¶meters) if err != nil { panic(err) } if err := ipc.StorageDriverServer(filesystem.FromParameters(parameters)); err != nil { logrus.Fatalln(err) } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/cmd/registry/0000755000175000017500000000000012502424227023172 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/cmd/registry/main.go0000644000175000017500000001070712502424227024452 0ustar tianontianonpackage main import ( _ "expvar" "flag" "fmt" "net/http" _ "net/http/pprof" "os" log "github.com/Sirupsen/logrus" "github.com/bugsnag/bugsnag-go" "github.com/docker/distribution/configuration" ctxu "github.com/docker/distribution/context" _ "github.com/docker/distribution/registry/auth/silly" _ "github.com/docker/distribution/registry/auth/token" "github.com/docker/distribution/registry/handlers" _ "github.com/docker/distribution/registry/storage/driver/filesystem" _ "github.com/docker/distribution/registry/storage/driver/inmemory" _ "github.com/docker/distribution/registry/storage/driver/middleware/cloudfront" _ "github.com/docker/distribution/registry/storage/driver/s3" "github.com/docker/distribution/version" gorhandlers "github.com/gorilla/handlers" "github.com/yvasiyarov/gorelic" "golang.org/x/net/context" ) var showVersion bool func init() { flag.BoolVar(&showVersion, "version", false, "show the version and exit") } func main() { flag.Usage = usage flag.Parse() if showVersion { version.PrintVersion() return } ctx := context.Background() config, err := resolveConfiguration() if err != nil { fatalf("configuration error: %v", err) } log.SetLevel(logLevel(config.Loglevel)) ctx = context.WithValue(ctx, "version", version.Version) ctx = ctxu.WithLogger(ctx, ctxu.GetLogger(ctx, "version")) app := handlers.NewApp(ctx, *config) handler := configureReporting(app) handler = gorhandlers.CombinedLoggingHandler(os.Stdout, handler) if config.HTTP.Debug.Addr != "" { go debugServer(config.HTTP.Debug.Addr) } if config.HTTP.TLS.Certificate == "" { ctxu.GetLogger(app).Infof("listening on %v", config.HTTP.Addr) if err := http.ListenAndServe(config.HTTP.Addr, handler); err != nil { ctxu.GetLogger(app).Fatalln(err) } } else { ctxu.GetLogger(app).Infof("listening on %v, tls", config.HTTP.Addr) if err := http.ListenAndServeTLS(config.HTTP.Addr, config.HTTP.TLS.Certificate, config.HTTP.TLS.Key, handler); err != nil { ctxu.GetLogger(app).Fatalln(err) } } } func usage() { fmt.Fprintln(os.Stderr, "usage:", os.Args[0], "") flag.PrintDefaults() } func fatalf(format string, args ...interface{}) { fmt.Fprintf(os.Stderr, format+"\n", args...) usage() os.Exit(1) } func resolveConfiguration() (*configuration.Configuration, error) { var configurationPath string if flag.NArg() > 0 { configurationPath = flag.Arg(0) } else if os.Getenv("REGISTRY_CONFIGURATION_PATH") != "" { configurationPath = os.Getenv("REGISTRY_CONFIGURATION_PATH") } if configurationPath == "" { return nil, fmt.Errorf("configuration path unspecified") } fp, err := os.Open(configurationPath) if err != nil { return nil, err } config, err := configuration.Parse(fp) if err != nil { return nil, fmt.Errorf("error parsing %s: %v", configurationPath, err) } return config, nil } func logLevel(level configuration.Loglevel) log.Level { l, err := log.ParseLevel(string(level)) if err != nil { log.Warnf("error parsing level %q: %v", level, err) l = log.InfoLevel } return l } func configureReporting(app *handlers.App) http.Handler { var handler http.Handler = app if app.Config.Reporting.Bugsnag.APIKey != "" { bugsnagConfig := bugsnag.Configuration{ APIKey: app.Config.Reporting.Bugsnag.APIKey, // TODO(brianbland): provide the registry version here // AppVersion: "2.0", } if app.Config.Reporting.Bugsnag.ReleaseStage != "" { bugsnagConfig.ReleaseStage = app.Config.Reporting.Bugsnag.ReleaseStage } if app.Config.Reporting.Bugsnag.Endpoint != "" { bugsnagConfig.Endpoint = app.Config.Reporting.Bugsnag.Endpoint } bugsnag.Configure(bugsnagConfig) handler = bugsnag.Handler(handler) } if app.Config.Reporting.NewRelic.LicenseKey != "" { agent := gorelic.NewAgent() agent.NewrelicLicense = app.Config.Reporting.NewRelic.LicenseKey if app.Config.Reporting.NewRelic.Name != "" { agent.NewrelicName = app.Config.Reporting.NewRelic.Name } agent.CollectHTTPStat = true agent.Verbose = true agent.Run() handler = agent.WrapHTTPHandler(handler) } return handler } // debugServer starts the debug server with pprof, expvar among other // endpoints. The addr should not be exposed externally. For most of these to // work, tls cannot be enabled on the endpoint, so it is generally separate. func debugServer(addr string) { log.Infof("debug server listening %v", addr) if err := http.ListenAndServe(addr, nil); err != nil { log.Fatalf("error listening on debug interface: %v", err) } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/cmd/registry/config.yml0000644000175000017500000000117112502424227025162 0ustar tianontianonversion: 0.1 loglevel: debug storage: filesystem: rootdirectory: /tmp/registry-dev http: addr: :5000 secret: asecretforlocaldevelopment debug: addr: localhost:5001 notifications: endpoints: - name: local-8082 url: http://localhost:5003/callback headers: Authorization: [Bearer ] timeout: 1s threshold: 10 backoff: 1s disabled: true - name: local-8083 url: http://localhost:8083/callback timeout: 1s threshold: 10 backoff: 1s disabled: true distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/testutil/0000755000175000017500000000000012502424227022434 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/testutil/handler.go0000644000175000017500000000574512502424227024413 0ustar tianontianonpackage testutil import ( "bytes" "fmt" "io" "io/ioutil" "net/http" "sort" "strings" ) // RequestResponseMap is an ordered mapping from Requests to Responses type RequestResponseMap []RequestResponseMapping // RequestResponseMapping defines a Response to be sent in response to a given // Request type RequestResponseMapping struct { Request Request Response Response } // TODO(bbland): add support for request headers // Request is a simplified http.Request object type Request struct { // Method is the http method of the request, for example GET Method string // Route is the http route of this request Route string // QueryParams are the query parameters of this request QueryParams map[string][]string // Body is the byte contents of the http request Body []byte } func (r Request) String() string { queryString := "" if len(r.QueryParams) > 0 { queryString = "?" keys := make([]string, 0, len(r.QueryParams)) for k := range r.QueryParams { keys = append(keys, k) } sort.Strings(keys) for _, k := range keys { queryString += strings.Join(r.QueryParams[k], "&") + "&" } queryString = queryString[:len(queryString)-1] } return fmt.Sprintf("%s %s%s\n%s", r.Method, r.Route, queryString, r.Body) } // Response is a simplified http.Response object type Response struct { // Statuscode is the http status code of the Response StatusCode int // Headers are the http headers of this Response Headers http.Header // Body is the response body Body []byte } // testHandler is an http.Handler with a defined mapping from Request to an // ordered list of Response objects type testHandler struct { responseMap map[string][]Response } // NewHandler returns a new test handler that responds to defined requests // with specified responses // Each time a Request is received, the next Response is returned in the // mapping, until no Responses are defined, at which point a 404 is sent back func NewHandler(requestResponseMap RequestResponseMap) http.Handler { responseMap := make(map[string][]Response) for _, mapping := range requestResponseMap { responses, ok := responseMap[mapping.Request.String()] if ok { responseMap[mapping.Request.String()] = append(responses, mapping.Response) } else { responseMap[mapping.Request.String()] = []Response{mapping.Response} } } return &testHandler{responseMap: responseMap} } func (app *testHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() requestBody, _ := ioutil.ReadAll(r.Body) request := Request{ Method: r.Method, Route: r.URL.Path, QueryParams: r.URL.Query(), Body: requestBody, } responses, ok := app.responseMap[request.String()] if !ok || len(responses) == 0 { http.NotFound(w, r) return } response := responses[0] app.responseMap[request.String()] = responses[1:] responseHeader := w.Header() for k, v := range response.Headers { responseHeader[k] = v } w.WriteHeader(response.StatusCode) io.Copy(w, bytes.NewReader(response.Body)) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/testutil/tarfile.go0000644000175000017500000000421012502424227024406 0ustar tianontianonpackage testutil import ( "archive/tar" "bytes" "crypto/rand" "fmt" "io" "io/ioutil" mrand "math/rand" "time" "github.com/docker/docker/pkg/tarsum" ) // CreateRandomTarFile creates a random tarfile, returning it as an // io.ReadSeeker along with its tarsum. An error is returned if there is a // problem generating valid content. func CreateRandomTarFile() (rs io.ReadSeeker, tarSum string, err error) { nFiles := mrand.Intn(10) + 10 target := &bytes.Buffer{} wr := tar.NewWriter(target) // Perturb this on each iteration of the loop below. header := &tar.Header{ Mode: 0644, ModTime: time.Now(), Typeflag: tar.TypeReg, Uname: "randocalrissian", Gname: "cloudcity", AccessTime: time.Now(), ChangeTime: time.Now(), } for fileNumber := 0; fileNumber < nFiles; fileNumber++ { fileSize := mrand.Int63n(1<<20) + 1<<20 header.Name = fmt.Sprint(fileNumber) header.Size = fileSize if err := wr.WriteHeader(header); err != nil { return nil, "", err } randomData := make([]byte, fileSize) // Fill up the buffer with some random data. n, err := rand.Read(randomData) if n != len(randomData) { return nil, "", fmt.Errorf("short read creating random reader: %v bytes != %v bytes", n, len(randomData)) } if err != nil { return nil, "", err } nn, err := io.Copy(wr, bytes.NewReader(randomData)) if nn != fileSize { return nil, "", fmt.Errorf("short copy writing random file to tar") } if err != nil { return nil, "", err } if err := wr.Flush(); err != nil { return nil, "", err } } if err := wr.Close(); err != nil { return nil, "", err } reader := bytes.NewReader(target.Bytes()) // A tar builder that supports tarsum inline calculation would be awesome // here. ts, err := tarsum.NewTarSum(reader, true, tarsum.Version1) if err != nil { return nil, "", err } nn, err := io.Copy(ioutil.Discard, ts) if nn != int64(len(target.Bytes())) { return nil, "", fmt.Errorf("short copy when getting tarsum of random layer: %v != %v", nn, len(target.Bytes())) } if err != nil { return nil, "", err } return bytes.NewReader(target.Bytes()), ts.Sum(nil), nil } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/digest/0000755000175000017500000000000012502424227022036 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/digest/verifiers_test.go0000644000175000017500000001265012502424227025426 0ustar tianontianonpackage digest import ( "bytes" "crypto/rand" "encoding/base64" "io" "os" "strings" "testing" "github.com/docker/distribution/testutil" ) func TestDigestVerifier(t *testing.T) { p := make([]byte, 1<<20) rand.Read(p) digest, err := FromBytes(p) if err != nil { t.Fatalf("unexpected error digesting bytes: %#v", err) } verifier, err := NewDigestVerifier(digest) if err != nil { t.Fatalf("unexpected error getting digest verifier: %s", err) } io.Copy(verifier, bytes.NewReader(p)) if !verifier.Verified() { t.Fatalf("bytes not verified") } tf, tarSum, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating tarfile: %v", err) } digest, err = FromTarArchive(tf) if err != nil { t.Fatalf("error digesting tarsum: %v", err) } if digest.String() != tarSum { t.Fatalf("unexpected digest: %q != %q", digest.String(), tarSum) } expectedSize, _ := tf.Seek(0, os.SEEK_END) // Get tar file size tf.Seek(0, os.SEEK_SET) // seek back // This is the most relevant example for the registry application. It's // effectively a read through pipeline, where the final sink is the digest // verifier. verifier, err = NewDigestVerifier(digest) if err != nil { t.Fatalf("unexpected error getting digest verifier: %s", err) } lengthVerifier := NewLengthVerifier(expectedSize) rd := io.TeeReader(tf, lengthVerifier) io.Copy(verifier, rd) if !lengthVerifier.Verified() { t.Fatalf("verifier detected incorrect length") } if !verifier.Verified() { t.Fatalf("bytes not verified") } } // TestVerifierUnsupportedDigest ensures that unsupported digest validation is // flowing through verifier creation. func TestVerifierUnsupportedDigest(t *testing.T) { unsupported := Digest("bean:0123456789abcdef") _, err := NewDigestVerifier(unsupported) if err == nil { t.Fatalf("expected error when creating verifier") } if err != ErrDigestUnsupported { t.Fatalf("incorrect error for unsupported digest: %v %p %p", err, ErrDigestUnsupported, err) } } // TestJunkNoDeadlock ensures that junk input into a digest verifier properly // returns errors from the tarsum library. Specifically, we pass in a file // with a "bad header" and should see the error from the io.Copy to verifier. // This has been seen with gzipped tarfiles, mishandled by the tarsum package, // but also on junk input, such as html. func TestJunkNoDeadlock(t *testing.T) { expected := Digest("tarsum.dev+sha256:62e15750aae345f6303469a94892e66365cc5e3abdf8d7cb8b329f8fb912e473") junk := bytes.Repeat([]byte{'a'}, 1024) verifier, err := NewDigestVerifier(expected) if err != nil { t.Fatalf("unexpected error creating verifier: %v", err) } rd := bytes.NewReader(junk) if _, err := io.Copy(verifier, rd); err == nil { t.Fatalf("unexpected error verifying input data: %v", err) } } // TestBadTarNoDeadlock runs a tar with a "bad" tar header through digest // verifier, ensuring that the verifier returns an error properly. func TestBadTarNoDeadlock(t *testing.T) { // TODO(stevvooe): This test is exposing a bug in tarsum where if we pass // a gzipped tar file into tarsum, the library returns an error. This // should actually work. When the tarsum package is fixed, this test will // fail and we can remove this test or invert it. // This tarfile was causing deadlocks in verifiers due mishandled copy error. // This is a gzipped tar, which we typically don't see but should handle. // // From https://registry-1.docker.io/v2/library/ubuntu/blobs/tarsum.dev+sha256:62e15750aae345f6303469a94892e66365cc5e3abdf8d7cb8b329f8fb912e473 const badTar = ` H4sIAAAJbogA/0otSdZnoDEwMDAxMDc1BdJggE6D2YZGJobGBmbGRsZAdYYGBkZGDAqmtHYYCJQW lyQWAZ1CqTnonhsiAAAAAP//AsV/YkEJTdMAGfFvZmA2Gv/0AAAAAAD//4LFf3F+aVFyarFeTmZx CbXtAOVnMxMTXPFvbGpmjhb/xobmwPinSyCO8PgHAAAA///EVU9v2z4MvedTEMihl9a5/26/YTkU yNKiTTDsKMt0rE0WDYmK628/ym7+bFmH2DksQACbIB/5+J7kObwiQsXc/LdYVGibLObRccw01Qv5 19EZ7hbbZudVgWtiDFCSh4paYII4xOVxNgeHLXrYow+GXAAqgSuEQhzlTR5ZgtlsVmB+aKe8rswe zzsOjwtoPGoTEGplHHhMCJqxSNUPwesbEGbzOXxR34VCHndQmjfhUKhEq/FURI0FqJKFR5q9NE5Z qbaoBGoglAB+5TSK0sOh3c3UPkRKE25dEg8dDzzIWmqN2wG3BNY4qRL1VFFAoJJb5SXHU90n34nk SUS8S0AeGwqGyXdZel1nn7KLGhPO0kDeluvN48ty9Q2269ft8/PTy2b5GfKuh9/2LBIWo6oz+N8G uodmWLETg0mW4lMP4XYYCL4+rlawftpIO40SA+W6Yci9wRZE1MNOjmyGdhBQRy9OHpqOdOGh/wT7 nZdOkHZ650uIK+WrVZdkgErJfnNEJysLnI5FSAj4xuiCQNpOIoNWmhyLByVHxEpLf3dkr+k9KMsV xV0FhiVB21hgD3V5XwSqRdOmsUYr7oNtZXTVzyTHc2/kqokBy2ihRMVRTN+78goP5Ur/aMhz+KOJ 3h2UsK43kdwDo0Q9jfD7ie2RRur7MdpIrx1Z3X4j/Q1qCswN9r/EGCvXiUy0fI4xeSknnH/92T/+ fgIAAP//GkWjYBSMXAAIAAD//2zZtzAAEgAA` expected := Digest("tarsum.dev+sha256:62e15750aae345f6303469a94892e66365cc5e3abdf8d7cb8b329f8fb912e473") verifier, err := NewDigestVerifier(expected) if err != nil { t.Fatalf("unexpected error creating verifier: %v", err) } rd := base64.NewDecoder(base64.StdEncoding, strings.NewReader(badTar)) if _, err := io.Copy(verifier, rd); err == nil { t.Fatalf("unexpected error verifying input data: %v", err) } if verifier.Verified() { // For now, we expect an error, since tarsum library cannot handle // compressed tars (!!!). t.Fatalf("no error received after invalid tar") } } // TODO(stevvooe): Add benchmarks to measure bytes/second throughput for // DigestVerifier. We should be tarsum/gzip limited for common cases but we // want to verify this. // // The relevant benchmarks for comparison can be run with the following // commands: // // go test -bench . crypto/sha1 // go test -bench . github.com/docker/docker/pkg/tarsum // distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/digest/doc.go0000644000175000017500000000420012502424227023126 0ustar tianontianon// Package digest provides a generalized type to opaquely represent message // digests and their operations within the registry. The Digest type is // designed to serve as a flexible identifier in a content-addressable system. // More importantly, it provides tools and wrappers to work with tarsums and // hash.Hash-based digests with little effort. // // Basics // // The format of a digest is simply a string with two parts, dubbed the // "algorithm" and the "digest", separated by a colon: // // : // // An example of a sha256 digest representation follows: // // sha256:7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc // // In this case, the string "sha256" is the algorithm and the hex bytes are // the "digest". A tarsum example will be more illustrative of the use case // involved in the registry: // // tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b // // For this, we consider the algorithm to be "tarsum+sha256". Prudent // applications will favor the ParseDigest function to verify the format over // using simple type casts. However, a normal string can be cast as a digest // with a simple type conversion: // // Digest("tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b") // // Because the Digest type is simply a string, once a valid Digest is // obtained, comparisons are cheap, quick and simple to express with the // standard equality operator. // // Verification // // The main benefit of using the Digest type is simple verification against a // given digest. The Verifier interface, modeled after the stdlib hash.Hash // interface, provides a common write sink for digest verification. After // writing is complete, calling the Verifier.Verified method will indicate // whether or not the stream of bytes matches the target digest. // // Missing Features // // In addition to the above, we intend to add the following features to this // package: // // 1. A Digester type that supports write sink digest calculation. // // 2. Suspend and resume of ongoing digest calculations to support efficient digest verification in the registry. // package digest distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/digest/verifiers.go0000644000175000017500000000562612502424227024374 0ustar tianontianonpackage digest import ( "crypto/sha256" "crypto/sha512" "hash" "io" "io/ioutil" "github.com/docker/docker/pkg/tarsum" ) // Verifier presents a general verification interface to be used with message // digests and other byte stream verifications. Users instantiate a Verifier // from one of the various methods, write the data under test to it then check // the result with the Verified method. type Verifier interface { io.Writer // Verified will return true if the content written to Verifier matches // the digest. Verified() bool } // NewDigestVerifier returns a verifier that compares the written bytes // against a passed in digest. func NewDigestVerifier(d Digest) (Verifier, error) { if err := d.Validate(); err != nil { return nil, err } alg := d.Algorithm() switch alg { case "sha256", "sha384", "sha512": return hashVerifier{ hash: newHash(alg), digest: d, }, nil default: // Assume we have a tarsum. version, err := tarsum.GetVersionFromTarsum(string(d)) if err != nil { return nil, err } pr, pw := io.Pipe() // TODO(stevvooe): We may actually want to ban the earlier versions of // tarsum. That decision may not be the place of the verifier. ts, err := tarsum.NewTarSum(pr, true, version) if err != nil { return nil, err } // TODO(sday): Ick! A goroutine per digest verification? We'll have to // get the tarsum library to export an io.Writer variant. go func() { if _, err := io.Copy(ioutil.Discard, ts); err != nil { pr.CloseWithError(err) } else { pr.Close() } }() return &tarsumVerifier{ digest: d, ts: ts, pr: pr, pw: pw, }, nil } } // NewLengthVerifier returns a verifier that returns true when the number of // read bytes equals the expected parameter. func NewLengthVerifier(expected int64) Verifier { return &lengthVerifier{ expected: expected, } } type lengthVerifier struct { expected int64 // expected bytes read len int64 // bytes read } func (lv *lengthVerifier) Write(p []byte) (n int, err error) { n = len(p) lv.len += int64(n) return n, err } func (lv *lengthVerifier) Verified() bool { return lv.expected == lv.len } func newHash(name string) hash.Hash { switch name { case "sha256": return sha256.New() case "sha384": return sha512.New384() case "sha512": return sha512.New() default: panic("unsupport algorithm: " + name) } } type hashVerifier struct { digest Digest hash hash.Hash } func (hv hashVerifier) Write(p []byte) (n int, err error) { return hv.hash.Write(p) } func (hv hashVerifier) Verified() bool { return hv.digest == NewDigest(hv.digest.Algorithm(), hv.hash) } type tarsumVerifier struct { digest Digest ts tarsum.TarSum pr *io.PipeReader pw *io.PipeWriter } func (tv *tarsumVerifier) Write(p []byte) (n int, err error) { return tv.pw.Write(p) } func (tv *tarsumVerifier) Verified() bool { return tv.digest == Digest(tv.ts.Sum(nil)) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/digest/tarsum_test.go0000644000175000017500000000365312502424227024746 0ustar tianontianonpackage digest import ( "reflect" "testing" ) func TestParseTarSumComponents(t *testing.T) { for _, testcase := range []struct { input string expected TarSumInfo err error }{ { input: "tarsum.v1+sha256:220a60ecd4a3c32c282622a625a54db9ba0ff55b5ba9c29c7064a2bc358b6a3e", expected: TarSumInfo{ Version: "v1", Algorithm: "sha256", Digest: "220a60ecd4a3c32c282622a625a54db9ba0ff55b5ba9c29c7064a2bc358b6a3e", }, }, { input: "", err: InvalidTarSumError(""), }, { input: "purejunk", err: InvalidTarSumError("purejunk"), }, { input: "tarsum.v23+test:12341234123412341effefefe", expected: TarSumInfo{ Version: "v23", Algorithm: "test", Digest: "12341234123412341effefefe", }, }, // The following test cases are ported from docker core { // Version 0 tarsum input: "tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b", expected: TarSumInfo{ Algorithm: "sha256", Digest: "e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b", }, }, { // Dev version tarsum input: "tarsum.dev+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b", expected: TarSumInfo{ Version: "dev", Algorithm: "sha256", Digest: "e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b", }, }, } { tsi, err := ParseTarSum(testcase.input) if err != nil { if testcase.err != nil && err == testcase.err { continue // passes } t.Fatalf("unexpected error parsing tarsum: %v", err) } if testcase.err != nil { t.Fatalf("expected error not encountered on %q: %v", testcase.input, testcase.err) } if !reflect.DeepEqual(tsi, testcase.expected) { t.Fatalf("expected tarsum info: %v != %v", tsi, testcase.expected) } if testcase.input != tsi.String() { t.Fatalf("input should equal output: %q != %q", tsi.String(), testcase.input) } } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/digest/tarsum.go0000644000175000017500000000400712502424227023701 0ustar tianontianonpackage digest import ( "fmt" "regexp" ) // TarSumRegexp defines a reguler expression to match tarsum identifiers. var TarsumRegexp = regexp.MustCompile("tarsum(?:.[a-z0-9]+)?\\+[a-zA-Z0-9]+:[A-Fa-f0-9]+") // TarsumRegexpCapturing defines a reguler expression to match tarsum identifiers with // capture groups corresponding to each component. var TarsumRegexpCapturing = regexp.MustCompile("(tarsum)(.([a-z0-9]+))?\\+([a-zA-Z0-9]+):([A-Fa-f0-9]+)") // TarSumInfo contains information about a parsed tarsum. type TarSumInfo struct { // Version contains the version of the tarsum. Version string // Algorithm contains the algorithm for the final digest Algorithm string // Digest contains the hex-encoded digest. Digest string } // InvalidTarSumError provides informations about a TarSum that cannot be parsed // by ParseTarSum. type InvalidTarSumError string func (e InvalidTarSumError) Error() string { return fmt.Sprintf("invalid tarsum: %q", string(e)) } // ParseTarSum parses a tarsum string into its components of interest. For // example, this method may receive the tarsum in the following format: // // tarsum.v1+sha256:220a60ecd4a3c32c282622a625a54db9ba0ff55b5ba9c29c7064a2bc358b6a3e // // The function will return the following: // // TarSumInfo{ // Version: "v1", // Algorithm: "sha256", // Digest: "220a60ecd4a3c32c282622a625a54db9ba0ff55b5ba9c29c7064a2bc358b6a3e", // } // func ParseTarSum(tarSum string) (tsi TarSumInfo, err error) { components := TarsumRegexpCapturing.FindStringSubmatch(tarSum) if len(components) != 1+TarsumRegexpCapturing.NumSubexp() { return TarSumInfo{}, InvalidTarSumError(tarSum) } return TarSumInfo{ Version: components[3], Algorithm: components[4], Digest: components[5], }, nil } // String returns the valid, string representation of the tarsum info. func (tsi TarSumInfo) String() string { if tsi.Version == "" { return fmt.Sprintf("tarsum+%s:%s", tsi.Algorithm, tsi.Digest) } return fmt.Sprintf("tarsum.%s+%s:%s", tsi.Version, tsi.Algorithm, tsi.Digest) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/digest/digest.go0000644000175000017500000001037312502424227023650 0ustar tianontianonpackage digest import ( "bytes" "crypto/sha256" "fmt" "hash" "io" "io/ioutil" "regexp" "strings" "github.com/docker/docker/pkg/tarsum" ) const ( // DigestTarSumV1EmptyTar is the digest for the empty tar file. DigestTarSumV1EmptyTar = "tarsum.v1+sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" // DigestSha256EmptyTar is the canonical sha256 digest of empty data DigestSha256EmptyTar = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" ) // Digest allows simple protection of hex formatted digest strings, prefixed // by their algorithm. Strings of type Digest have some guarantee of being in // the correct format and it provides quick access to the components of a // digest string. // // The following is an example of the contents of Digest types: // // sha256:7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc // // More important for this code base, this type is compatible with tarsum // digests. For example, the following would be a valid Digest: // // tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b // // This allows to abstract the digest behind this type and work only in those // terms. type Digest string // NewDigest returns a Digest from alg and a hash.Hash object. func NewDigest(alg string, h hash.Hash) Digest { return Digest(fmt.Sprintf("%s:%x", alg, h.Sum(nil))) } // NewDigestFromHex returns a Digest from alg and a the hex encoded digest. func NewDigestFromHex(alg, hex string) Digest { return Digest(fmt.Sprintf("%s:%s", alg, hex)) } // DigestRegexp matches valid digest types. var DigestRegexp = regexp.MustCompile(`[a-zA-Z0-9-_+.]+:[a-fA-F0-9]+`) // DigestRegexpAnchored matches valid digest types, anchored to the start and end of the match. var DigestRegexpAnchored = regexp.MustCompile(`^` + DigestRegexp.String() + `$`) var ( // ErrDigestInvalidFormat returned when digest format invalid. ErrDigestInvalidFormat = fmt.Errorf("invalid checksum digest format") // ErrDigestUnsupported returned when the digest algorithm is unsupported. ErrDigestUnsupported = fmt.Errorf("unsupported digest algorithm") ) // ParseDigest parses s and returns the validated digest object. An error will // be returned if the format is invalid. func ParseDigest(s string) (Digest, error) { d := Digest(s) return d, d.Validate() } // FromReader returns the most valid digest for the underlying content. func FromReader(rd io.Reader) (Digest, error) { h := sha256.New() if _, err := io.Copy(h, rd); err != nil { return "", err } return NewDigest("sha256", h), nil } // FromTarArchive produces a tarsum digest from reader rd. func FromTarArchive(rd io.Reader) (Digest, error) { ts, err := tarsum.NewTarSum(rd, true, tarsum.Version1) if err != nil { return "", err } if _, err := io.Copy(ioutil.Discard, ts); err != nil { return "", err } d, err := ParseDigest(ts.Sum(nil)) if err != nil { return "", err } return d, nil } // FromBytes digests the input and returns a Digest. func FromBytes(p []byte) (Digest, error) { return FromReader(bytes.NewReader(p)) } // Validate checks that the contents of d is a valid digest, returning an // error if not. func (d Digest) Validate() error { s := string(d) // Common case will be tarsum _, err := ParseTarSum(s) if err == nil { return nil } // Continue on for general parser if !DigestRegexpAnchored.MatchString(s) { return ErrDigestInvalidFormat } i := strings.Index(s, ":") if i < 0 { return ErrDigestInvalidFormat } // case: "sha256:" with no hex. if i+1 == len(s) { return ErrDigestInvalidFormat } switch s[:i] { case "sha256", "sha384", "sha512": break default: return ErrDigestUnsupported } return nil } // Algorithm returns the algorithm portion of the digest. This will panic if // the underlying digest is not in a valid format. func (d Digest) Algorithm() string { return string(d[:d.sepIndex()]) } // Hex returns the hex digest portion of the digest. This will panic if the // underlying digest is not in a valid format. func (d Digest) Hex() string { return string(d[d.sepIndex()+1:]) } func (d Digest) String() string { return string(d) } func (d Digest) sepIndex() int { i := strings.Index(string(d), ":") if i < 0 { panic("could not find ':' in digest: " + d) } return i } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/digest/digest_test.go0000644000175000017500000000631212502424227024705 0ustar tianontianonpackage digest import ( "bytes" "io" "testing" ) func TestParseDigest(t *testing.T) { for _, testcase := range []struct { input string err error algorithm string hex string }{ { input: "tarsum+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b", algorithm: "tarsum+sha256", hex: "e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b", }, { input: "tarsum.dev+sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b", algorithm: "tarsum.dev+sha256", hex: "e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b", }, { input: "tarsum.v1+sha256:220a60ecd4a3c32c282622a625a54db9ba0ff55b5ba9c29c7064a2bc358b6a3e", algorithm: "tarsum.v1+sha256", hex: "220a60ecd4a3c32c282622a625a54db9ba0ff55b5ba9c29c7064a2bc358b6a3e", }, { input: "sha256:e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b", algorithm: "sha256", hex: "e58fcf7418d4390dec8e8fb69d88c06ec07039d651fedd3aa72af9972e7d046b", }, { input: "sha384:d3fc7881460b7e22e3d172954463dddd7866d17597e7248453c48b3e9d26d9596bf9c4a9cf8072c9d5bad76e19af801d", algorithm: "sha384", hex: "d3fc7881460b7e22e3d172954463dddd7866d17597e7248453c48b3e9d26d9596bf9c4a9cf8072c9d5bad76e19af801d", }, { // empty hex input: "sha256:", err: ErrDigestInvalidFormat, }, { // just hex input: "d41d8cd98f00b204e9800998ecf8427e", err: ErrDigestInvalidFormat, }, { // not hex input: "sha256:d41d8cd98f00b204e9800m98ecf8427e", err: ErrDigestInvalidFormat, }, { input: "foo:d41d8cd98f00b204e9800998ecf8427e", err: ErrDigestUnsupported, }, } { digest, err := ParseDigest(testcase.input) if err != testcase.err { t.Fatalf("error differed from expected while parsing %q: %v != %v", testcase.input, err, testcase.err) } if testcase.err != nil { continue } if digest.Algorithm() != testcase.algorithm { t.Fatalf("incorrect algorithm for parsed digest: %q != %q", digest.Algorithm(), testcase.algorithm) } if digest.Hex() != testcase.hex { t.Fatalf("incorrect hex for parsed digest: %q != %q", digest.Hex(), testcase.hex) } // Parse string return value and check equality newParsed, err := ParseDigest(digest.String()) if err != nil { t.Fatalf("unexpected error parsing input %q: %v", testcase.input, err) } if newParsed != digest { t.Fatalf("expected equal: %q != %q", newParsed, digest) } } } // A few test cases used to fix behavior we expect in storage backend. func TestFromTarArchiveZeroLength(t *testing.T) { checkTarsumDigest(t, "zero-length archive", bytes.NewReader([]byte{}), DigestTarSumV1EmptyTar) } func TestFromTarArchiveEmptyTar(t *testing.T) { // String of 1024 zeros is a valid, empty tar file. checkTarsumDigest(t, "1024 zero bytes", bytes.NewReader(bytes.Repeat([]byte("\x00"), 1024)), DigestTarSumV1EmptyTar) } func checkTarsumDigest(t *testing.T, msg string, rd io.Reader, expected Digest) { dgst, err := FromTarArchive(rd) if err != nil { t.Fatalf("unexpected error digesting %s: %v", msg, err) } if dgst != expected { t.Fatalf("unexpected digest for %s: %q != %q", msg, dgst, expected) } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/digest/digester.go0000644000175000017500000000201312502424227024167 0ustar tianontianonpackage digest import ( "crypto/sha256" "hash" ) // Digester calculates the digest of written data. It is functionally // equivalent to hash.Hash but provides methods for returning the Digest type // rather than raw bytes. type Digester struct { alg string hash hash.Hash } // NewDigester create a new Digester with the given hashing algorithm and instance // of that algo's hasher. func NewDigester(alg string, h hash.Hash) Digester { return Digester{ alg: alg, hash: h, } } // NewCanonicalDigester is a convenience function to create a new Digester with // out default settings. func NewCanonicalDigester() Digester { return NewDigester("sha256", sha256.New()) } // Write data to the digester. These writes cannot fail. func (d *Digester) Write(p []byte) (n int, err error) { return d.hash.Write(p) } // Digest returns the current digest for this digester. func (d *Digester) Digest() Digest { return NewDigest(d.alg, d.hash) } // Reset the state of the digester. func (d *Digester) Reset() { d.hash.Reset() } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/.drone.yml0000644000175000017500000000162612502424227022474 0ustar tianontianonimage: dmp42/go:stable script: # To be spoofed back into the test image - go get github.com/modocache/gover - go get -t ./... # Go fmt - test -z "$(gofmt -s -l -w . | tee /dev/stderr)" # Go lint - test -z "$(golint ./... | tee /dev/stderr)" # Go vet - go vet ./... # Go test - go test -v -race -cover ./... # Helper to concatenate reports - gover # Send to coverall - goveralls -service drone.io -coverprofile=gover.coverprofile -repotoken {{COVERALLS_TOKEN}} # Do we want these as well? # - go get code.google.com/p/go.tools/cmd/goimports # - test -z "$(goimports -l -w ./... | tee /dev/stderr)" # http://labix.org/gocheck notify: email: recipients: - distribution@docker.com slack: team: docker channel: "#dt" username: mom token: {{SLACK_TOKEN}} on_success: true on_failure: true distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/version/0000755000175000017500000000000012502424227022244 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/version/version.go0000644000175000017500000000074512502424227024266 0ustar tianontianonpackage version // Package is the overall, canonical project import path under which the // package was built. var Package = "github.com/docker/distribution" // Version indicates which version of the binary is running. This is set to // the latest release tag by hand, always suffixed by "+unknown". During // build, it will be replaced by the actual version. The value here will be // used if the registry is run after a go get based install. var Version = "v2.0.0-alpha.2+unknown" distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/version/version.sh0000755000175000017500000000145712502424227024277 0ustar tianontianon#!/bin/sh # This bash script outputs the current, desired content of version.go, using # git describe. For best effect, pipe this to the target file. Generally, this # only needs to updated for releases. The actual value of will be replaced # during build time if the makefile is used. set -e cat < // // For example, a binary "registry" built from github.com/docker/distribution // with version "v2.0" would print the following: // // registry github.com/docker/distribution v2.0 // func FprintVersion(w io.Writer) { fmt.Fprintln(w, os.Args[0], Package, Version) } // PrintVersion outputs the version information, from Fprint, to stdout. func PrintVersion() { FprintVersion(os.Stdout) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/configuration/0000755000175000017500000000000012502424227023426 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/configuration/configuration_test.go0000644000175000017500000002554112502424227027672 0ustar tianontianonpackage configuration import ( "bytes" "net/http" "os" "testing" . "gopkg.in/check.v1" "gopkg.in/yaml.v2" ) // Hook up gocheck into the "go test" runner func Test(t *testing.T) { TestingT(t) } // configStruct is a canonical example configuration, which should map to configYamlV0_1 var configStruct = Configuration{ Version: "0.1", Loglevel: "info", Storage: Storage{ "s3": Parameters{ "region": "us-east-1", "bucket": "my-bucket", "rootpath": "/registry", "encrypt": true, "secure": false, "accesskey": "SAMPLEACCESSKEY", "secretkey": "SUPERSECRET", "host": nil, "port": 42, }, }, Auth: Auth{ "silly": Parameters{ "realm": "silly", "service": "silly", }, }, Reporting: Reporting{ Bugsnag: BugsnagReporting{ APIKey: "BugsnagApiKey", }, }, Notifications: Notifications{ Endpoints: []Endpoint{ { Name: "endpoint-1", URL: "http://example.com", Headers: http.Header{ "Authorization": []string{"Bearer "}, }, }, }, }, } // configYamlV0_1 is a Version 0.1 yaml document representing configStruct var configYamlV0_1 = ` version: 0.1 loglevel: info storage: s3: region: us-east-1 bucket: my-bucket rootpath: /registry encrypt: true secure: false accesskey: SAMPLEACCESSKEY secretkey: SUPERSECRET host: ~ port: 42 auth: silly: realm: silly service: silly notifications: endpoints: - name: endpoint-1 url: http://example.com headers: Authorization: [Bearer ] reporting: bugsnag: apikey: BugsnagApiKey ` // inmemoryConfigYamlV0_1 is a Version 0.1 yaml document specifying an inmemory // storage driver with no parameters var inmemoryConfigYamlV0_1 = ` version: 0.1 loglevel: info storage: inmemory auth: silly: realm: silly service: silly notifications: endpoints: - name: endpoint-1 url: http://example.com headers: Authorization: [Bearer ] ` type ConfigSuite struct { expectedConfig *Configuration } var _ = Suite(new(ConfigSuite)) func (suite *ConfigSuite) SetUpTest(c *C) { os.Clearenv() suite.expectedConfig = copyConfig(configStruct) } // TestMarshalRoundtrip validates that configStruct can be marshaled and // unmarshaled without changing any parameters func (suite *ConfigSuite) TestMarshalRoundtrip(c *C) { configBytes, err := yaml.Marshal(suite.expectedConfig) c.Assert(err, IsNil) config, err := Parse(bytes.NewReader(configBytes)) c.Assert(err, IsNil) c.Assert(config, DeepEquals, suite.expectedConfig) } // TestParseSimple validates that configYamlV0_1 can be parsed into a struct // matching configStruct func (suite *ConfigSuite) TestParseSimple(c *C) { config, err := Parse(bytes.NewReader([]byte(configYamlV0_1))) c.Assert(err, IsNil) c.Assert(config, DeepEquals, suite.expectedConfig) } // TestParseInmemory validates that configuration yaml with storage provided as // a string can be parsed into a Configuration struct with no storage parameters func (suite *ConfigSuite) TestParseInmemory(c *C) { suite.expectedConfig.Storage = Storage{"inmemory": Parameters{}} suite.expectedConfig.Reporting = Reporting{} config, err := Parse(bytes.NewReader([]byte(inmemoryConfigYamlV0_1))) c.Assert(err, IsNil) c.Assert(config, DeepEquals, suite.expectedConfig) } // TestParseIncomplete validates that an incomplete yaml configuration cannot // be parsed without providing environment variables to fill in the missing // components. func (suite *ConfigSuite) TestParseIncomplete(c *C) { incompleteConfigYaml := "version: 0.1" _, err := Parse(bytes.NewReader([]byte(incompleteConfigYaml))) c.Assert(err, NotNil) suite.expectedConfig.Storage = Storage{"filesystem": Parameters{"rootdirectory": "/tmp/testroot"}} suite.expectedConfig.Auth = Auth{"silly": Parameters{"realm": "silly"}} suite.expectedConfig.Reporting = Reporting{} suite.expectedConfig.Notifications = Notifications{} os.Setenv("REGISTRY_STORAGE", "filesystem") os.Setenv("REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY", "/tmp/testroot") os.Setenv("REGISTRY_AUTH", "silly") os.Setenv("REGISTRY_AUTH_SILLY_REALM", "silly") config, err := Parse(bytes.NewReader([]byte(incompleteConfigYaml))) c.Assert(err, IsNil) c.Assert(config, DeepEquals, suite.expectedConfig) } // TestParseWithSameEnvStorage validates that providing environment variables // that match the given storage type will only include environment-defined // parameters and remove yaml-defined parameters func (suite *ConfigSuite) TestParseWithSameEnvStorage(c *C) { suite.expectedConfig.Storage = Storage{"s3": Parameters{"region": "us-east-1"}} os.Setenv("REGISTRY_STORAGE", "s3") os.Setenv("REGISTRY_STORAGE_S3_REGION", "us-east-1") config, err := Parse(bytes.NewReader([]byte(configYamlV0_1))) c.Assert(err, IsNil) c.Assert(config, DeepEquals, suite.expectedConfig) } // TestParseWithDifferentEnvStorageParams validates that providing environment variables that change // and add to the given storage parameters will change and add parameters to the parsed // Configuration struct func (suite *ConfigSuite) TestParseWithDifferentEnvStorageParams(c *C) { suite.expectedConfig.Storage.setParameter("region", "us-west-1") suite.expectedConfig.Storage.setParameter("secure", true) suite.expectedConfig.Storage.setParameter("newparam", "some Value") os.Setenv("REGISTRY_STORAGE_S3_REGION", "us-west-1") os.Setenv("REGISTRY_STORAGE_S3_SECURE", "true") os.Setenv("REGISTRY_STORAGE_S3_NEWPARAM", "some Value") config, err := Parse(bytes.NewReader([]byte(configYamlV0_1))) c.Assert(err, IsNil) c.Assert(config, DeepEquals, suite.expectedConfig) } // TestParseWithDifferentEnvStorageType validates that providing an environment variable that // changes the storage type will be reflected in the parsed Configuration struct func (suite *ConfigSuite) TestParseWithDifferentEnvStorageType(c *C) { suite.expectedConfig.Storage = Storage{"inmemory": Parameters{}} os.Setenv("REGISTRY_STORAGE", "inmemory") config, err := Parse(bytes.NewReader([]byte(configYamlV0_1))) c.Assert(err, IsNil) c.Assert(config, DeepEquals, suite.expectedConfig) } // TestParseWithExtraneousEnvStorageParams validates that environment variables // that change parameters out of the scope of the specified storage type are // ignored. func (suite *ConfigSuite) TestParseWithExtraneousEnvStorageParams(c *C) { os.Setenv("REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY", "/tmp/testroot") config, err := Parse(bytes.NewReader([]byte(configYamlV0_1))) c.Assert(err, IsNil) c.Assert(config, DeepEquals, suite.expectedConfig) } // TestParseWithDifferentEnvStorageTypeAndParams validates that providing an environment variable // that changes the storage type will be reflected in the parsed Configuration struct and that // environment storage parameters will also be included func (suite *ConfigSuite) TestParseWithDifferentEnvStorageTypeAndParams(c *C) { suite.expectedConfig.Storage = Storage{"filesystem": Parameters{}} suite.expectedConfig.Storage.setParameter("rootdirectory", "/tmp/testroot") os.Setenv("REGISTRY_STORAGE", "filesystem") os.Setenv("REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY", "/tmp/testroot") config, err := Parse(bytes.NewReader([]byte(configYamlV0_1))) c.Assert(err, IsNil) c.Assert(config, DeepEquals, suite.expectedConfig) } // TestParseWithSameEnvLoglevel validates that providing an environment variable defining the log // level to the same as the one provided in the yaml will not change the parsed Configuration struct func (suite *ConfigSuite) TestParseWithSameEnvLoglevel(c *C) { os.Setenv("REGISTRY_LOGLEVEL", "info") config, err := Parse(bytes.NewReader([]byte(configYamlV0_1))) c.Assert(err, IsNil) c.Assert(config, DeepEquals, suite.expectedConfig) } // TestParseWithDifferentEnvLoglevel validates that providing an environment variable defining the // log level will override the value provided in the yaml document func (suite *ConfigSuite) TestParseWithDifferentEnvLoglevel(c *C) { suite.expectedConfig.Loglevel = "error" os.Setenv("REGISTRY_LOGLEVEL", "error") config, err := Parse(bytes.NewReader([]byte(configYamlV0_1))) c.Assert(err, IsNil) c.Assert(config, DeepEquals, suite.expectedConfig) } // TestParseInvalidLoglevel validates that the parser will fail to parse a // configuration if the loglevel is malformed func (suite *ConfigSuite) TestParseInvalidLoglevel(c *C) { invalidConfigYaml := "version: 0.1\nloglevel: derp\nstorage: inmemory" _, err := Parse(bytes.NewReader([]byte(invalidConfigYaml))) c.Assert(err, NotNil) os.Setenv("REGISTRY_LOGLEVEL", "derp") _, err = Parse(bytes.NewReader([]byte(configYamlV0_1))) c.Assert(err, NotNil) } // TestParseWithDifferentEnvReporting validates that environment variables // properly override reporting parameters func (suite *ConfigSuite) TestParseWithDifferentEnvReporting(c *C) { suite.expectedConfig.Reporting.Bugsnag.APIKey = "anotherBugsnagApiKey" suite.expectedConfig.Reporting.Bugsnag.Endpoint = "localhost:8080" suite.expectedConfig.Reporting.NewRelic.LicenseKey = "NewRelicLicenseKey" suite.expectedConfig.Reporting.NewRelic.Name = "some NewRelic NAME" os.Setenv("REGISTRY_REPORTING_BUGSNAG_APIKEY", "anotherBugsnagApiKey") os.Setenv("REGISTRY_REPORTING_BUGSNAG_ENDPOINT", "localhost:8080") os.Setenv("REGISTRY_REPORTING_NEWRELIC_LICENSEKEY", "NewRelicLicenseKey") os.Setenv("REGISTRY_REPORTING_NEWRELIC_NAME", "some NewRelic NAME") config, err := Parse(bytes.NewReader([]byte(configYamlV0_1))) c.Assert(err, IsNil) c.Assert(config, DeepEquals, suite.expectedConfig) } // TestParseInvalidVersion validates that the parser will fail to parse a newer configuration // version than the CurrentVersion func (suite *ConfigSuite) TestParseInvalidVersion(c *C) { suite.expectedConfig.Version = MajorMinorVersion(CurrentVersion.Major(), CurrentVersion.Minor()+1) configBytes, err := yaml.Marshal(suite.expectedConfig) c.Assert(err, IsNil) _, err = Parse(bytes.NewReader(configBytes)) c.Assert(err, NotNil) } func copyConfig(config Configuration) *Configuration { configCopy := new(Configuration) configCopy.Version = MajorMinorVersion(config.Version.Major(), config.Version.Minor()) configCopy.Loglevel = config.Loglevel configCopy.Storage = Storage{config.Storage.Type(): Parameters{}} for k, v := range config.Storage.Parameters() { configCopy.Storage.setParameter(k, v) } configCopy.Reporting = Reporting{ Bugsnag: BugsnagReporting{config.Reporting.Bugsnag.APIKey, config.Reporting.Bugsnag.ReleaseStage, config.Reporting.Bugsnag.Endpoint}, NewRelic: NewRelicReporting{config.Reporting.NewRelic.LicenseKey, config.Reporting.NewRelic.Name}, } configCopy.Auth = Auth{config.Auth.Type(): Parameters{}} for k, v := range config.Auth.Parameters() { configCopy.Auth.setParameter(k, v) } configCopy.Notifications = Notifications{Endpoints: []Endpoint{}} for _, v := range config.Notifications.Endpoints { configCopy.Notifications.Endpoints = append(configCopy.Notifications.Endpoints, v) } return configCopy } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/configuration/configuration.go0000644000175000017500000002562412502424227026635 0ustar tianontianonpackage configuration import ( "fmt" "io" "io/ioutil" "net/http" "reflect" "strings" "time" ) // Configuration is a versioned registry configuration, intended to be provided by a yaml file, and // optionally modified by environment variables type Configuration struct { // Version is the version which defines the format of the rest of the configuration Version Version `yaml:"version"` // Loglevel is the level at which registry operations are logged Loglevel Loglevel `yaml:"loglevel"` // Storage is the configuration for the registry's storage driver Storage Storage `yaml:"storage"` // Auth allows configuration of various authorization methods that may be // used to gate requests. Auth Auth `yaml:"auth,omitempty"` // Middleware lists all middlewares to be used by the registry. Middleware map[string][]Middleware `yaml:"middleware,omitempty"` // Reporting is the configuration for error reporting Reporting Reporting `yaml:"reporting,omitempty"` // HTTP contains configuration parameters for the registry's http // interface. HTTP struct { // Addr specifies the bind address for the registry instance. Addr string `yaml:"addr,omitempty"` Prefix string `yaml:"prefix,omitempty"` // Secret specifies the secret key which HMAC tokens are created with. Secret string `yaml:"secret,omitempty"` // TLS instructs the http server to listen with a TLS configuration. // This only support simple tls configuration with a cert and key. // Mostly, this is useful for testing situations or simple deployments // that require tls. If more complex configurations are required, use // a proxy or make a proposal to add support here. TLS struct { // Certificate specifies the path to an x509 certificate file to // be used for TLS. Certificate string `yaml:"certificate,omitempty"` // Key specifies the path to the x509 key file, which should // contain the private portion for the file specified in // Certificate. Key string `yaml:"key,omitempty"` } `yaml:"tls,omitempty"` // Debug configures the http debug interface, if specified. This can // include services such as pprof, expvar and other data that should // not be exposed externally. Left disabled by default. Debug struct { // Addr specifies the bind address for the debug server. Addr string `yaml:"addr,omitempty"` } `yaml:"debug,omitempty"` } `yaml:"http,omitempty"` // Notifications specifies configuration about various endpoint to which // registry events are dispatched. Notifications Notifications `yaml:"notifications,omitempty"` } // v0_1Configuration is a Version 0.1 Configuration struct // This is currently aliased to Configuration, as it is the current version type v0_1Configuration Configuration // UnmarshalYAML implements the yaml.Unmarshaler interface // Unmarshals a string of the form X.Y into a Version, validating that X and Y can represent uints func (version *Version) UnmarshalYAML(unmarshal func(interface{}) error) error { var versionString string err := unmarshal(&versionString) if err != nil { return err } newVersion := Version(versionString) if _, err := newVersion.major(); err != nil { return err } if _, err := newVersion.minor(); err != nil { return err } *version = newVersion return nil } // CurrentVersion is the most recent Version that can be parsed var CurrentVersion = MajorMinorVersion(0, 1) // Loglevel is the level at which operations are logged // This can be error, warn, info, or debug type Loglevel string // UnmarshalYAML implements the yaml.Umarshaler interface // Unmarshals a string into a Loglevel, lowercasing the string and validating that it represents a // valid loglevel func (loglevel *Loglevel) UnmarshalYAML(unmarshal func(interface{}) error) error { var loglevelString string err := unmarshal(&loglevelString) if err != nil { return err } loglevelString = strings.ToLower(loglevelString) switch loglevelString { case "error", "warn", "info", "debug": default: return fmt.Errorf("Invalid loglevel %s Must be one of [error, warn, info, debug]", loglevelString) } *loglevel = Loglevel(loglevelString) return nil } // Parameters defines a key-value parameters mapping type Parameters map[string]interface{} // Storage defines the configuration for registry object storage type Storage map[string]Parameters // Type returns the storage driver type, such as filesystem or s3 func (storage Storage) Type() string { // Return only key in this map for k := range storage { return k } return "" } // Parameters returns the Parameters map for a Storage configuration func (storage Storage) Parameters() Parameters { return storage[storage.Type()] } // setParameter changes the parameter at the provided key to the new value func (storage Storage) setParameter(key string, value interface{}) { storage[storage.Type()][key] = value } // UnmarshalYAML implements the yaml.Unmarshaler interface // Unmarshals a single item map into a Storage or a string into a Storage type with no parameters func (storage *Storage) UnmarshalYAML(unmarshal func(interface{}) error) error { var storageMap map[string]Parameters err := unmarshal(&storageMap) if err == nil { if len(storageMap) > 1 { types := make([]string, 0, len(storageMap)) for k := range storageMap { types = append(types, k) } return fmt.Errorf("Must provide exactly one storage type. Provided: %v", types) } *storage = storageMap return nil } var storageType string err = unmarshal(&storageType) if err == nil { *storage = Storage{storageType: Parameters{}} return nil } return err } // MarshalYAML implements the yaml.Marshaler interface func (storage Storage) MarshalYAML() (interface{}, error) { if storage.Parameters() == nil { return storage.Type(), nil } return map[string]Parameters(storage), nil } // Auth defines the configuration for registry authorization. type Auth map[string]Parameters // Type returns the storage driver type, such as filesystem or s3 func (auth Auth) Type() string { // Return only key in this map for k := range auth { return k } return "" } // Parameters returns the Parameters map for an Auth configuration func (auth Auth) Parameters() Parameters { return auth[auth.Type()] } // setParameter changes the parameter at the provided key to the new value func (auth Auth) setParameter(key string, value interface{}) { auth[auth.Type()][key] = value } // UnmarshalYAML implements the yaml.Unmarshaler interface // Unmarshals a single item map into a Storage or a string into a Storage type with no parameters func (auth *Auth) UnmarshalYAML(unmarshal func(interface{}) error) error { var m map[string]Parameters err := unmarshal(&m) if err == nil { if len(m) > 1 { types := make([]string, 0, len(m)) for k := range m { types = append(types, k) } // TODO(stevvooe): May want to change this slightly for // authorization to allow multiple challenges. return fmt.Errorf("must provide exactly one type. Provided: %v", types) } *auth = m return nil } var authType string err = unmarshal(&authType) if err == nil { *auth = Auth{authType: Parameters{}} return nil } return err } // MarshalYAML implements the yaml.Marshaler interface func (auth Auth) MarshalYAML() (interface{}, error) { if auth.Parameters() == nil { return auth.Type(), nil } return map[string]Parameters(auth), nil } // Notifications configures multiple http endpoints. type Notifications struct { // Endpoints is a list of http configurations for endpoints that // respond to webhook notifications. In the future, we may allow other // kinds of endpoints, such as external queues. Endpoints []Endpoint `yaml:"endpoints,omitempty"` } // Endpoint describes the configuration of an http webhook notification // endpoint. type Endpoint struct { Name string `yaml:"name"` // identifies the endpoint in the registry instance. Disabled bool `yaml:"disabled"` // disables the endpoint URL string `yaml:"url"` // post url for the endpoint. Headers http.Header `yaml:"headers"` // static headers that should be added to all requests Timeout time.Duration `yaml:"timeout"` // HTTP timeout Threshold int `yaml:"threshold"` // circuit breaker threshold before backing off on failure Backoff time.Duration `yaml:"backoff"` // backoff duration } // Reporting defines error reporting methods. type Reporting struct { // Bugsnag configures error reporting for Bugsnag (bugsnag.com). Bugsnag BugsnagReporting `yaml:"bugsnag,omitempty"` // NewRelic configures error reporting for NewRelic (newrelic.com) NewRelic NewRelicReporting `yaml:"newrelic,omitempty"` } // BugsnagReporting configures error reporting for Bugsnag (bugsnag.com). type BugsnagReporting struct { // APIKey is the Bugsnag api key. APIKey string `yaml:"apikey,omitempty"` // ReleaseStage tracks where the registry is deployed. // Examples: production, staging, development ReleaseStage string `yaml:"releasestage,omitempty"` // Endpoint is used for specifying an enterprise Bugsnag endpoint. Endpoint string `yaml:"endpoint,omitempty"` } // NewRelicReporting configures error reporting for NewRelic (newrelic.com) type NewRelicReporting struct { // LicenseKey is the NewRelic user license key LicenseKey string `yaml:"licensekey,omitempty"` // Name is the component name of the registry in NewRelic Name string `yaml:"name,omitempty"` } // Middleware configures named middlewares to be applied at injection points. type Middleware struct { // Name the middleware registers itself as Name string `yaml:"name"` // Flag to disable middleware easily Disabled bool `yaml:"disabled,omitempty"` // Map of parameters that will be passed to the middleware's initialization function Options Parameters `yaml:"options"` } // Parse parses an input configuration yaml document into a Configuration struct // This should generally be capable of handling old configuration format versions // // Environment variables may be used to override configuration parameters other than version, // following the scheme below: // Configuration.Abc may be replaced by the value of REGISTRY_ABC, // Configuration.Abc.Xyz may be replaced by the value of REGISTRY_ABC_XYZ, and so forth func Parse(rd io.Reader) (*Configuration, error) { in, err := ioutil.ReadAll(rd) if err != nil { return nil, err } p := NewParser("registry", []VersionedParseInfo{ { Version: MajorMinorVersion(0, 1), ParseAs: reflect.TypeOf(v0_1Configuration{}), ConversionFunc: func(c interface{}) (interface{}, error) { if v0_1, ok := c.(*v0_1Configuration); ok { if v0_1.Loglevel == Loglevel("") { v0_1.Loglevel = Loglevel("info") } if v0_1.Storage.Type() == "" { return nil, fmt.Errorf("No storage configuration provided") } return (*Configuration)(v0_1), nil } return nil, fmt.Errorf("Expected *v0_1Configuration, received %#v", c) }, }, }) config := new(Configuration) err = p.Parse(in, config) if err != nil { return nil, err } return config, nil } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/configuration/README.md0000644000175000017500000001215212502424227024706 0ustar tianontianonDocker-Registry Configuration ============================= This document describes the registry configuration model and how to specify a custom configuration with a configuration file and/or environment variables. Semantic-ish Versioning ----------------------- The configuration file is designed with versioning in mind, such that most upgrades will not require a change in configuration files, and such that configuration files can be "upgraded" from one version to another. The version is specified as a string of the form `MajorVersion.MinorVersion`, where MajorVersion and MinorVersion are both non-negative integer values. Much like [semantic versioning](http://semver.org/), minor version increases denote inherently backwards-compatible changes, such as the addition of optional fields, whereas major version increases denote a restructuring, such as renaming fields or adding required fields. Because of the explicit version definition in the configuration file, it should be possible to parse old configuration files and port them to the current configuration version, although this is not guaranteed for all future versions. File Structure (as of Version 0.1) ------------------------------------ The configuration structure is defined by the `Configuration` struct in `configuration.go`, and is best described by the following two examples: ```yaml version: 0.1 loglevel: info storage: s3: region: us-east-1 bucket: my-bucket rootpath: /registry encrypt: true secure: false accesskey: SAMPLEACCESSKEY secretkey: SUPERSECRET host: ~ port: ~ auth: silly: realm: test-realm service: my-service reporting: bugsnag: apikey: mybugsnagapikey releasestage: development newrelic: licensekey: mynewreliclicensekey name: docker-distribution http: addr: 0.0.0.0:5000 secret: mytokensecret ``` ```yaml version: 0.1 loglevel: debug storage: inmemory ``` ### version The version is expected to remain a top-level field, as to allow for a consistent version check before parsing the remainder of the configuration file. ### loglevel This specifies the log level of the registry. Supported values: * `error` * `warn` * `info` * `debug` ### storage This specifies the storage driver, and may be provided either as a string (only the driver type) or as a driver name with a parameters map, as seen in the first example above. The parameters map will be passed into the factory constructor of the given storage driver type. ### auth This specifies the authorization method the registry will use, and is provided as an auth type with a parameters map. The parameters map will be passed into the factory constructor of the given auth type. ### reporting This specifies metrics/error reporting systems which the registry will forward information about stats/errors to. There are currently two supported systems, which are documented below. #### bugsnag Reports http errors and panics to [bugsnag](https://bugsnag.com). ##### apikey (Required for bugsnag use) Specifies the bugnsag API Key for authenticating to your account. ##### releasestage (Optional) Tracks the stage at which the registry is deployed. For example: "production", "staging", "development". ##### endpoint (Optional) Used for specifying an enterprise bugsnag endpoint other than https://bugsnag.com. #### newrelic Reports heap, goroutine, and http stats to [NewRelic](https://newrelic.com). ##### licensekey (Required for newrelic use) Specifies the NewRelic License Key for authenticating to your account. ##### name (Optional) Specifies the component name that is displayed in the NewRelic panel. ### http This is used for HTTP transport-specific configuration options. #### addr Specifies the bind address for the registry instance. Example: 0.0.0.0:5000 #### secret Specifies the secret key with which query-string HMAC tokens are generated. ### Notes All keys in the configuration file **must** be provided as a string of lowercase letters and numbers only, and values must be string-like (booleans and numerical values are fine to parse as strings). Environment Variables --------------------- To support the workflow of running a docker registry from a standard container without having to modify configuration files, the registry configuration also supports environment variables for overriding fields. Any configuration field other than version can be replaced by providing an environment variable of the following form: `REGISTRY_[_]...`. For example, to change the loglevel to `error`, one can provide `REGISTRY_LOGLEVEL=error`, and to change the s3 storage driver's region parameter to `us-west-1`, one can provide `REGISTRY_STORAGE_S3_LOGLEVEL=us-west-1`. ### Notes If an environment variable changes a map value into a string, such as replacing the storage driver type with `REGISTRY_STORAGE=filesystem`, then all sub-fields will be erased. As such, specifying the storage type in the environment will remove all parameters related to the old storage configuration. By restricting all keys in the configuration file to lowercase letters and numbers, we can avoid any potential environment variable mapping ambiguity. distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/configuration/parser.go0000644000175000017500000001323312502424227025253 0ustar tianontianonpackage configuration import ( "fmt" "os" "reflect" "regexp" "strconv" "strings" "gopkg.in/yaml.v2" ) // Version is a major/minor version pair of the form Major.Minor // Major version upgrades indicate structure or type changes // Minor version upgrades should be strictly additive type Version string // MajorMinorVersion constructs a Version from its Major and Minor components func MajorMinorVersion(major, minor uint) Version { return Version(fmt.Sprintf("%d.%d", major, minor)) } func (version Version) major() (uint, error) { majorPart := strings.Split(string(version), ".")[0] major, err := strconv.ParseUint(majorPart, 10, 0) return uint(major), err } // Major returns the major version portion of a Version func (version Version) Major() uint { major, _ := version.major() return major } func (version Version) minor() (uint, error) { minorPart := strings.Split(string(version), ".")[1] minor, err := strconv.ParseUint(minorPart, 10, 0) return uint(minor), err } // Minor returns the minor version portion of a Version func (version Version) Minor() uint { minor, _ := version.minor() return minor } // VersionedParseInfo defines how a specific version of a configuration should // be parsed into the current version type VersionedParseInfo struct { // Version is the version which this parsing information relates to Version Version // ParseAs defines the type which a configuration file of this version // should be parsed into ParseAs reflect.Type // ConversionFunc defines a method for converting the parsed configuration // (of type ParseAs) into the current configuration version // Note: this method signature is very unclear with the absence of generics ConversionFunc func(interface{}) (interface{}, error) } // Parser can be used to parse a configuration file and environment of a defined // version into a unified output structure type Parser struct { prefix string mapping map[Version]VersionedParseInfo env map[string]string } // NewParser returns a *Parser with the given environment prefix which handles // versioned configurations which match the given parseInfos func NewParser(prefix string, parseInfos []VersionedParseInfo) *Parser { p := Parser{prefix: prefix, mapping: make(map[Version]VersionedParseInfo), env: make(map[string]string)} for _, parseInfo := range parseInfos { p.mapping[parseInfo.Version] = parseInfo } for _, env := range os.Environ() { envParts := strings.SplitN(env, "=", 2) p.env[envParts[0]] = envParts[1] } return &p } // Parse reads in the given []byte and environment and writes the resulting // configuration into the input v // // Environment variables may be used to override configuration parameters other // than version, following the scheme below: // v.Abc may be replaced by the value of PREFIX_ABC, // v.Abc.Xyz may be replaced by the value of PREFIX_ABC_XYZ, and so forth func (p *Parser) Parse(in []byte, v interface{}) error { var versionedStruct struct { Version Version } if err := yaml.Unmarshal(in, &versionedStruct); err != nil { return err } parseInfo, ok := p.mapping[versionedStruct.Version] if !ok { return fmt.Errorf("Unsupported version: %q", versionedStruct.Version) } parseAs := reflect.New(parseInfo.ParseAs) err := yaml.Unmarshal(in, parseAs.Interface()) if err != nil { return err } err = p.overwriteFields(parseAs, p.prefix) if err != nil { return err } c, err := parseInfo.ConversionFunc(parseAs.Interface()) if err != nil { return err } reflect.ValueOf(v).Elem().Set(reflect.Indirect(reflect.ValueOf(c))) return nil } func (p *Parser) overwriteFields(v reflect.Value, prefix string) error { for v.Kind() == reflect.Ptr { v = reflect.Indirect(v) } switch v.Kind() { case reflect.Struct: for i := 0; i < v.NumField(); i++ { sf := v.Type().Field(i) fieldPrefix := strings.ToUpper(prefix + "_" + sf.Name) if e, ok := p.env[fieldPrefix]; ok { fieldVal := reflect.New(sf.Type) err := yaml.Unmarshal([]byte(e), fieldVal.Interface()) if err != nil { return err } v.Field(i).Set(reflect.Indirect(fieldVal)) } err := p.overwriteFields(v.Field(i), fieldPrefix) if err != nil { return err } } case reflect.Map: p.overwriteMap(v, prefix) } return nil } func (p *Parser) overwriteMap(m reflect.Value, prefix string) error { switch m.Type().Elem().Kind() { case reflect.Struct: for _, k := range m.MapKeys() { err := p.overwriteFields(m.MapIndex(k), strings.ToUpper(fmt.Sprintf("%s_%s", prefix, k))) if err != nil { return err } } envMapRegexp, err := regexp.Compile(fmt.Sprintf("^%s_([A-Z0-9]+)$", strings.ToUpper(prefix))) if err != nil { return err } for key, val := range p.env { if submatches := envMapRegexp.FindStringSubmatch(key); submatches != nil { mapValue := reflect.New(m.Type().Elem()) err := yaml.Unmarshal([]byte(val), mapValue.Interface()) if err != nil { return err } m.SetMapIndex(reflect.ValueOf(strings.ToLower(submatches[1])), reflect.Indirect(mapValue)) } } case reflect.Map: for _, k := range m.MapKeys() { err := p.overwriteMap(m.MapIndex(k), strings.ToUpper(fmt.Sprintf("%s_%s", prefix, k))) if err != nil { return err } } default: envMapRegexp, err := regexp.Compile(fmt.Sprintf("^%s_([A-Z0-9]+)$", strings.ToUpper(prefix))) if err != nil { return err } for key, val := range p.env { if submatches := envMapRegexp.FindStringSubmatch(key); submatches != nil { mapValue := reflect.New(m.Type().Elem()) err := yaml.Unmarshal([]byte(val), mapValue.Interface()) if err != nil { return err } m.SetMapIndex(reflect.ValueOf(strings.ToLower(submatches[1])), reflect.Indirect(mapValue)) } } } return nil } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/circle.yml0000644000175000017500000000703212502424227022545 0ustar tianontianon# Pony-up! machine: pre: # Install gvm - bash < <(curl -s -S -L https://raw.githubusercontent.com/moovweb/gvm/1.0.22/binscripts/gvm-installer) post: # Install many go versions - gvm install go1.3.3 -B --name=old - gvm install go1.4 -B --name=stable # - gvm install tip --name=bleed environment: # Convenient shortcuts to "common" locations CHECKOUT: /home/ubuntu/$CIRCLE_PROJECT_REPONAME BASE_DIR: src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME # Trick circle brainflat "no absolute path" behavior BASE_OLD: ../../../$HOME/.gvm/pkgsets/old/global/$BASE_DIR BASE_STABLE: ../../../$HOME/.gvm/pkgsets/stable/global/$BASE_DIR # BASE_BLEED: ../../../$HOME/.gvm/pkgsets/bleed/global/$BASE_DIR # Workaround Circle parsing dumb bugs and/or YAML wonkyness CIRCLE_PAIN: "mode: set" hosts: # Not used yet fancy: 127.0.0.1 dependencies: pre: # Copy the code to the gopath of all go versions - > gvm use old && mkdir -p "$(dirname $BASE_OLD)" && cp -R "$CHECKOUT" "$BASE_OLD" - > gvm use stable && mkdir -p "$(dirname $BASE_STABLE)" && cp -R "$CHECKOUT" "$BASE_STABLE" # - > # gvm use bleed && # mkdir -p "$(dirname $BASE_BLEED)" && # cp -R "$CHECKOUT" "$BASE_BLEED" override: # Install dependencies for every copied clone/go version - gvm use old && go get github.com/tools/godep: pwd: $BASE_OLD - gvm use stable && go get github.com/tools/godep: pwd: $BASE_STABLE # - gvm use bleed && go get github.com/tools/godep: # pwd: $BASE_BLEED post: # For the stable go version, additionally install linting tools - > gvm use stable && go get github.com/axw/gocov/gocov github.com/mattn/goveralls github.com/golang/lint/golint test: pre: # Output the go versions we are going to test - gvm use old && go version - gvm use stable && go version # - gvm use bleed && go version # FMT - gvm use stable && test -z "$(gofmt -s -l . | grep -v Godeps/_workspace/src/ | tee /dev/stderr)": pwd: $BASE_STABLE # VET - gvm use stable && go vet ./...: pwd: $BASE_STABLE # LINT - gvm use stable && test -z "$(golint ./... | grep -v Godeps/_workspace/src/ | tee /dev/stderr)": pwd: $BASE_STABLE override: # Test every version we have (but stable) - gvm use old; godep go test -test.v -test.short ./...: timeout: 600 pwd: $BASE_OLD # - gvm use bleed; go test -test.v -test.short ./...: # timeout: 600 # pwd: $BASE_BLEED # Test stable, and report # Preset the goverall report file - echo "$CIRCLE_PAIN" > ~/goverage.report - gvm use stable; go list ./... | xargs -L 1 -I{} rm -f $GOPATH/src/{}/coverage.out: pwd: $BASE_STABLE - gvm use stable; go list ./... | xargs -L 1 -I{} godep go test -test.short -coverprofile=$GOPATH/src/{}/coverage.out {}: timeout: 600 pwd: $BASE_STABLE post: # Aggregate and report to coveralls - gvm use stable; go list ./... | xargs -L 1 -I{} cat "$GOPATH/src/{}/coverage.out" | grep -v "$CIRCLE_PAIN" >> ~/goverage.report: pwd: $BASE_STABLE - gvm use stable; goveralls -service circleci -coverprofile=/home/ubuntu/goverage.report -repotoken $COVERALLS_TOKEN: pwd: $BASE_STABLE ## Notes # Disabled the -race detector due to massive memory usage. # Do we want these as well? # - go get code.google.com/p/go.tools/cmd/goimports # - test -z "$(goimports -l -w ./... | tee /dev/stderr)" # http://labix.org/gocheck distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/0000755000175000017500000000000012502424227022427 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/handlers/0000755000175000017500000000000012502424227024227 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/handlers/api_test.go0000644000175000017500000005067012502424227026376 0ustar tianontianonpackage handlers import ( "bytes" "encoding/json" "fmt" "io" "io/ioutil" "net/http" "net/http/httptest" "net/http/httputil" "net/url" "os" "path" "reflect" "strings" "testing" "github.com/docker/distribution/configuration" "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest" "github.com/docker/distribution/registry/api/v2" _ "github.com/docker/distribution/registry/storage/driver/inmemory" "github.com/docker/distribution/testutil" "github.com/docker/libtrust" "github.com/gorilla/handlers" "golang.org/x/net/context" ) // TestCheckAPI hits the base endpoint (/v2/) ensures we return the specified // 200 OK response. func TestCheckAPI(t *testing.T) { env := newTestEnv(t) baseURL, err := env.builder.BuildBaseURL() if err != nil { t.Fatalf("unexpected error building base url: %v", err) } resp, err := http.Get(baseURL) if err != nil { t.Fatalf("unexpected error issuing request: %v", err) } defer resp.Body.Close() checkResponse(t, "issuing api base check", resp, http.StatusOK) checkHeaders(t, resp, http.Header{ "Content-Type": []string{"application/json; charset=utf-8"}, "Content-Length": []string{"2"}, }) p, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatalf("unexpected error reading response body: %v", err) } if string(p) != "{}" { t.Fatalf("unexpected response body: %v", string(p)) } } func TestURLPrefix(t *testing.T) { config := configuration.Configuration{ Storage: configuration.Storage{ "inmemory": configuration.Parameters{}, }, } config.HTTP.Prefix = "/test/" env := newTestEnvWithConfig(t, &config) baseURL, err := env.builder.BuildBaseURL() if err != nil { t.Fatalf("unexpected error building base url: %v", err) } parsed, _ := url.Parse(baseURL) if !strings.HasPrefix(parsed.Path, config.HTTP.Prefix) { t.Fatalf("Prefix %v not included in test url %v", config.HTTP.Prefix, baseURL) } resp, err := http.Get(baseURL) if err != nil { t.Fatalf("unexpected error issuing request: %v", err) } defer resp.Body.Close() checkResponse(t, "issuing api base check", resp, http.StatusOK) checkHeaders(t, resp, http.Header{ "Content-Type": []string{"application/json; charset=utf-8"}, "Content-Length": []string{"2"}, }) } // TestLayerAPI conducts a full of the of the layer api. func TestLayerAPI(t *testing.T) { // TODO(stevvooe): This test code is complete junk but it should cover the // complete flow. This must be broken down and checked against the // specification *before* we submit the final to docker core. env := newTestEnv(t) imageName := "foo/bar" // "build" our layer file layerFile, tarSumStr, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating random layer file: %v", err) } layerDigest := digest.Digest(tarSumStr) // ----------------------------------- // Test fetch for non-existent content layerURL, err := env.builder.BuildBlobURL(imageName, layerDigest) if err != nil { t.Fatalf("error building url: %v", err) } resp, err := http.Get(layerURL) if err != nil { t.Fatalf("unexpected error fetching non-existent layer: %v", err) } checkResponse(t, "fetching non-existent content", resp, http.StatusNotFound) // ------------------------------------------ // Test head request for non-existent content resp, err = http.Head(layerURL) if err != nil { t.Fatalf("unexpected error checking head on non-existent layer: %v", err) } checkResponse(t, "checking head on non-existent layer", resp, http.StatusNotFound) // ------------------------------------------ // Start an upload, check the status then cancel uploadURLBase, uploadUUID := startPushLayer(t, env.builder, imageName) // A status check should work resp, err = http.Get(uploadURLBase) if err != nil { t.Fatalf("unexpected error getting upload status: %v", err) } checkResponse(t, "status of deleted upload", resp, http.StatusNoContent) checkHeaders(t, resp, http.Header{ "Location": []string{"*"}, "Range": []string{"0-0"}, "Docker-Upload-UUID": []string{uploadUUID}, }) req, err := http.NewRequest("DELETE", uploadURLBase, nil) if err != nil { t.Fatalf("unexpected error creating delete request: %v", err) } resp, err = http.DefaultClient.Do(req) if err != nil { t.Fatalf("unexpected error sending delete request: %v", err) } checkResponse(t, "deleting upload", resp, http.StatusNoContent) // A status check should result in 404 resp, err = http.Get(uploadURLBase) if err != nil { t.Fatalf("unexpected error getting upload status: %v", err) } checkResponse(t, "status of deleted upload", resp, http.StatusNotFound) // ----------------------------------------- // Do layer push with an empty body and different digest uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName) resp, err = doPushLayer(t, env.builder, imageName, layerDigest, uploadURLBase, bytes.NewReader([]byte{})) if err != nil { t.Fatalf("unexpected error doing bad layer push: %v", err) } checkResponse(t, "bad layer push", resp, http.StatusBadRequest) checkBodyHasErrorCodes(t, "bad layer push", resp, v2.ErrorCodeDigestInvalid) // ----------------------------------------- // Do layer push with an empty body and correct digest zeroDigest, err := digest.FromTarArchive(bytes.NewReader([]byte{})) if err != nil { t.Fatalf("unexpected error digesting empty buffer: %v", err) } uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName) pushLayer(t, env.builder, imageName, zeroDigest, uploadURLBase, bytes.NewReader([]byte{})) // ----------------------------------------- // Do layer push with an empty body and correct digest // This is a valid but empty tarfile! emptyTar := bytes.Repeat([]byte("\x00"), 1024) emptyDigest, err := digest.FromTarArchive(bytes.NewReader(emptyTar)) if err != nil { t.Fatalf("unexpected error digesting empty tar: %v", err) } uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName) pushLayer(t, env.builder, imageName, emptyDigest, uploadURLBase, bytes.NewReader(emptyTar)) // ------------------------------------------ // Now, actually do successful upload. layerLength, _ := layerFile.Seek(0, os.SEEK_END) layerFile.Seek(0, os.SEEK_SET) uploadURLBase, uploadUUID = startPushLayer(t, env.builder, imageName) pushLayer(t, env.builder, imageName, layerDigest, uploadURLBase, layerFile) // ------------------------ // Use a head request to see if the layer exists. resp, err = http.Head(layerURL) if err != nil { t.Fatalf("unexpected error checking head on existing layer: %v", err) } checkResponse(t, "checking head on existing layer", resp, http.StatusOK) checkHeaders(t, resp, http.Header{ "Content-Length": []string{fmt.Sprint(layerLength)}, "Docker-Content-Digest": []string{layerDigest.String()}, }) // ---------------- // Fetch the layer! resp, err = http.Get(layerURL) if err != nil { t.Fatalf("unexpected error fetching layer: %v", err) } checkResponse(t, "fetching layer", resp, http.StatusOK) checkHeaders(t, resp, http.Header{ "Content-Length": []string{fmt.Sprint(layerLength)}, "Docker-Content-Digest": []string{layerDigest.String()}, }) // Verify the body verifier, err := digest.NewDigestVerifier(layerDigest) if err != nil { t.Fatalf("unexpected error getting digest verifier: %s", err) } io.Copy(verifier, resp.Body) if !verifier.Verified() { t.Fatalf("response body did not pass verification") } // Missing tests: // - Upload the same tarsum file under and different repository and // ensure the content remains uncorrupted. } func TestManifestAPI(t *testing.T) { env := newTestEnv(t) imageName := "foo/bar" tag := "thetag" manifestURL, err := env.builder.BuildManifestURL(imageName, tag) if err != nil { t.Fatalf("unexpected error getting manifest url: %v", err) } // ----------------------------- // Attempt to fetch the manifest resp, err := http.Get(manifestURL) if err != nil { t.Fatalf("unexpected error getting manifest: %v", err) } defer resp.Body.Close() checkResponse(t, "getting non-existent manifest", resp, http.StatusNotFound) checkBodyHasErrorCodes(t, "getting non-existent manifest", resp, v2.ErrorCodeManifestUnknown) tagsURL, err := env.builder.BuildTagsURL(imageName) if err != nil { t.Fatalf("unexpected error building tags url: %v", err) } resp, err = http.Get(tagsURL) if err != nil { t.Fatalf("unexpected error getting unknown tags: %v", err) } defer resp.Body.Close() // Check that we get an unknown repository error when asking for tags checkResponse(t, "getting unknown manifest tags", resp, http.StatusNotFound) checkBodyHasErrorCodes(t, "getting unknown manifest tags", resp, v2.ErrorCodeNameUnknown) // -------------------------------- // Attempt to push unsigned manifest with missing layers unsignedManifest := &manifest.Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, }, Name: imageName, Tag: tag, FSLayers: []manifest.FSLayer{ { BlobSum: "asdf", }, { BlobSum: "qwer", }, }, } resp = putManifest(t, "putting unsigned manifest", manifestURL, unsignedManifest) defer resp.Body.Close() checkResponse(t, "posting unsigned manifest", resp, http.StatusBadRequest) _, p, counts := checkBodyHasErrorCodes(t, "getting unknown manifest tags", resp, v2.ErrorCodeManifestUnverified, v2.ErrorCodeBlobUnknown, v2.ErrorCodeDigestInvalid) expectedCounts := map[v2.ErrorCode]int{ v2.ErrorCodeManifestUnverified: 1, v2.ErrorCodeBlobUnknown: 2, v2.ErrorCodeDigestInvalid: 2, } if !reflect.DeepEqual(counts, expectedCounts) { t.Fatalf("unexpected number of error codes encountered: %v\n!=\n%v\n---\n%s", counts, expectedCounts, string(p)) } // TODO(stevvooe): Add a test case where we take a mostly valid registry, // tamper with the content and ensure that we get a unverified manifest // error. // Push 2 random layers expectedLayers := make(map[digest.Digest]io.ReadSeeker) for i := range unsignedManifest.FSLayers { rs, dgstStr, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating random layer %d: %v", i, err) } dgst := digest.Digest(dgstStr) expectedLayers[dgst] = rs unsignedManifest.FSLayers[i].BlobSum = dgst uploadURLBase, _ := startPushLayer(t, env.builder, imageName) pushLayer(t, env.builder, imageName, dgst, uploadURLBase, rs) } // ------------------- // Push the signed manifest with all layers pushed. signedManifest, err := manifest.Sign(unsignedManifest, env.pk) if err != nil { t.Fatalf("unexpected error signing manifest: %v", err) } payload, err := signedManifest.Payload() checkErr(t, err, "getting manifest payload") dgst, err := digest.FromBytes(payload) checkErr(t, err, "digesting manifest") manifestDigestURL, err := env.builder.BuildManifestURL(imageName, dgst.String()) checkErr(t, err, "building manifest url") resp = putManifest(t, "putting signed manifest", manifestURL, signedManifest) checkResponse(t, "putting signed manifest", resp, http.StatusAccepted) checkHeaders(t, resp, http.Header{ "Location": []string{manifestDigestURL}, "Docker-Content-Digest": []string{dgst.String()}, }) // -------------------- // Push by digest -- should get same result resp = putManifest(t, "putting signed manifest", manifestDigestURL, signedManifest) checkResponse(t, "putting signed manifest", resp, http.StatusAccepted) checkHeaders(t, resp, http.Header{ "Location": []string{manifestDigestURL}, "Docker-Content-Digest": []string{dgst.String()}, }) // ------------------ // Fetch by tag name resp, err = http.Get(manifestURL) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } defer resp.Body.Close() checkResponse(t, "fetching uploaded manifest", resp, http.StatusOK) checkHeaders(t, resp, http.Header{ "Docker-Content-Digest": []string{dgst.String()}, }) var fetchedManifest manifest.SignedManifest dec := json.NewDecoder(resp.Body) if err := dec.Decode(&fetchedManifest); err != nil { t.Fatalf("error decoding fetched manifest: %v", err) } if !bytes.Equal(fetchedManifest.Raw, signedManifest.Raw) { t.Fatalf("manifests do not match") } // --------------- // Fetch by digest resp, err = http.Get(manifestDigestURL) checkErr(t, err, "fetching manifest by digest") defer resp.Body.Close() checkResponse(t, "fetching uploaded manifest", resp, http.StatusOK) checkHeaders(t, resp, http.Header{ "Docker-Content-Digest": []string{dgst.String()}, }) var fetchedManifestByDigest manifest.SignedManifest dec = json.NewDecoder(resp.Body) if err := dec.Decode(&fetchedManifestByDigest); err != nil { t.Fatalf("error decoding fetched manifest: %v", err) } if !bytes.Equal(fetchedManifestByDigest.Raw, signedManifest.Raw) { t.Fatalf("manifests do not match") } // Ensure that the tag is listed. resp, err = http.Get(tagsURL) if err != nil { t.Fatalf("unexpected error getting unknown tags: %v", err) } defer resp.Body.Close() // Check that we get an unknown repository error when asking for tags checkResponse(t, "getting unknown manifest tags", resp, http.StatusOK) dec = json.NewDecoder(resp.Body) var tagsResponse tagsAPIResponse if err := dec.Decode(&tagsResponse); err != nil { t.Fatalf("unexpected error decoding error response: %v", err) } if tagsResponse.Name != imageName { t.Fatalf("tags name should match image name: %v != %v", tagsResponse.Name, imageName) } if len(tagsResponse.Tags) != 1 { t.Fatalf("expected some tags in response: %v", tagsResponse.Tags) } if tagsResponse.Tags[0] != tag { t.Fatalf("tag not as expected: %q != %q", tagsResponse.Tags[0], tag) } } type testEnv struct { pk libtrust.PrivateKey ctx context.Context config configuration.Configuration app *App server *httptest.Server builder *v2.URLBuilder } func newTestEnv(t *testing.T) *testEnv { config := configuration.Configuration{ Storage: configuration.Storage{ "inmemory": configuration.Parameters{}, }, } return newTestEnvWithConfig(t, &config) } func newTestEnvWithConfig(t *testing.T, config *configuration.Configuration) *testEnv { ctx := context.Background() app := NewApp(ctx, *config) server := httptest.NewServer(handlers.CombinedLoggingHandler(os.Stderr, app)) builder, err := v2.NewURLBuilderFromString(server.URL + config.HTTP.Prefix) if err != nil { t.Fatalf("error creating url builder: %v", err) } pk, err := libtrust.GenerateECP256PrivateKey() if err != nil { t.Fatalf("unexpected error generating private key: %v", err) } return &testEnv{ pk: pk, ctx: ctx, config: *config, app: app, server: server, builder: builder, } } func putManifest(t *testing.T, msg, url string, v interface{}) *http.Response { var body []byte if sm, ok := v.(*manifest.SignedManifest); ok { body = sm.Raw } else { var err error body, err = json.MarshalIndent(v, "", " ") if err != nil { t.Fatalf("unexpected error marshaling %v: %v", v, err) } } req, err := http.NewRequest("PUT", url, bytes.NewReader(body)) if err != nil { t.Fatalf("error creating request for %s: %v", msg, err) } resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatalf("error doing put request while %s: %v", msg, err) } return resp } func startPushLayer(t *testing.T, ub *v2.URLBuilder, name string) (location string, uuid string) { layerUploadURL, err := ub.BuildBlobUploadURL(name) if err != nil { t.Fatalf("unexpected error building layer upload url: %v", err) } resp, err := http.Post(layerUploadURL, "", nil) if err != nil { t.Fatalf("unexpected error starting layer push: %v", err) } defer resp.Body.Close() checkResponse(t, fmt.Sprintf("pushing starting layer push %v", name), resp, http.StatusAccepted) u, err := url.Parse(resp.Header.Get("Location")) if err != nil { t.Fatalf("error parsing location header: %v", err) } uuid = path.Base(u.Path) checkHeaders(t, resp, http.Header{ "Location": []string{"*"}, "Content-Length": []string{"0"}, "Docker-Upload-UUID": []string{uuid}, }) return resp.Header.Get("Location"), uuid } // doPushLayer pushes the layer content returning the url on success returning // the response. If you're only expecting a successful response, use pushLayer. func doPushLayer(t *testing.T, ub *v2.URLBuilder, name string, dgst digest.Digest, uploadURLBase string, body io.Reader) (*http.Response, error) { u, err := url.Parse(uploadURLBase) if err != nil { t.Fatalf("unexpected error parsing pushLayer url: %v", err) } u.RawQuery = url.Values{ "_state": u.Query()["_state"], "digest": []string{dgst.String()}, }.Encode() uploadURL := u.String() // Just do a monolithic upload req, err := http.NewRequest("PUT", uploadURL, body) if err != nil { t.Fatalf("unexpected error creating new request: %v", err) } return http.DefaultClient.Do(req) } // pushLayer pushes the layer content returning the url on success. func pushLayer(t *testing.T, ub *v2.URLBuilder, name string, dgst digest.Digest, uploadURLBase string, body io.Reader) string { digester := digest.NewCanonicalDigester() resp, err := doPushLayer(t, ub, name, dgst, uploadURLBase, io.TeeReader(body, &digester)) if err != nil { t.Fatalf("unexpected error doing push layer request: %v", err) } defer resp.Body.Close() checkResponse(t, "putting monolithic chunk", resp, http.StatusCreated) if err != nil { t.Fatalf("error generating sha256 digest of body") } sha256Dgst := digester.Digest() expectedLayerURL, err := ub.BuildBlobURL(name, sha256Dgst) if err != nil { t.Fatalf("error building expected layer url: %v", err) } checkHeaders(t, resp, http.Header{ "Location": []string{expectedLayerURL}, "Content-Length": []string{"0"}, "Docker-Content-Digest": []string{sha256Dgst.String()}, }) return resp.Header.Get("Location") } func checkResponse(t *testing.T, msg string, resp *http.Response, expectedStatus int) { if resp.StatusCode != expectedStatus { t.Logf("unexpected status %s: %v != %v", msg, resp.StatusCode, expectedStatus) maybeDumpResponse(t, resp) t.FailNow() } } // checkBodyHasErrorCodes ensures the body is an error body and has the // expected error codes, returning the error structure, the json slice and a // count of the errors by code. func checkBodyHasErrorCodes(t *testing.T, msg string, resp *http.Response, errorCodes ...v2.ErrorCode) (v2.Errors, []byte, map[v2.ErrorCode]int) { p, err := ioutil.ReadAll(resp.Body) if err != nil { t.Fatalf("unexpected error reading body %s: %v", msg, err) } var errs v2.Errors if err := json.Unmarshal(p, &errs); err != nil { t.Fatalf("unexpected error decoding error response: %v", err) } if len(errs.Errors) == 0 { t.Fatalf("expected errors in response") } // TODO(stevvooe): Shoot. The error setup is not working out. The content- // type headers are being set after writing the status code. // if resp.Header.Get("Content-Type") != "application/json; charset=utf-8" { // t.Fatalf("unexpected content type: %v != 'application/json'", // resp.Header.Get("Content-Type")) // } expected := map[v2.ErrorCode]struct{}{} counts := map[v2.ErrorCode]int{} // Initialize map with zeros for expected for _, code := range errorCodes { expected[code] = struct{}{} counts[code] = 0 } for _, err := range errs.Errors { if _, ok := expected[err.Code]; !ok { t.Fatalf("unexpected error code %v encountered during %s: %s ", err.Code, msg, string(p)) } counts[err.Code]++ } // Ensure that counts of expected errors were all non-zero for code := range expected { if counts[code] == 0 { t.Fatalf("expected error code %v not encounterd during %s: %s", code, msg, string(p)) } } return errs, p, counts } func maybeDumpResponse(t *testing.T, resp *http.Response) { if d, err := httputil.DumpResponse(resp, true); err != nil { t.Logf("error dumping response: %v", err) } else { t.Logf("response:\n%s", string(d)) } } // matchHeaders checks that the response has at least the headers. If not, the // test will fail. If a passed in header value is "*", any non-zero value will // suffice as a match. func checkHeaders(t *testing.T, resp *http.Response, headers http.Header) { for k, vs := range headers { if resp.Header.Get(k) == "" { t.Fatalf("response missing header %q", k) } for _, v := range vs { if v == "*" { // Just ensure there is some value. if len(resp.Header[k]) > 0 { continue } } for _, hv := range resp.Header[k] { if hv != v { t.Fatalf("%v header value not matched in response: %q != %q", k, hv, v) } } } } } func checkErr(t *testing.T, err error, msg string) { if err != nil { t.Fatalf("unexpected error %s: %v", msg, err) } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/handlers/hmac_test.go0000644000175000017500000000550012502424227026525 0ustar tianontianonpackage handlers import "testing" var layerUploadStates = []layerUploadState{ { Name: "hello", UUID: "abcd-1234-qwer-0987", Offset: 0, }, { Name: "hello-world", UUID: "abcd-1234-qwer-0987", Offset: 0, }, { Name: "h3ll0_w0rld", UUID: "abcd-1234-qwer-0987", Offset: 1337, }, { Name: "ABCDEFG", UUID: "ABCD-1234-QWER-0987", Offset: 1234567890, }, { Name: "this-is-A-sort-of-Long-name-for-Testing", UUID: "dead-1234-beef-0987", Offset: 8675309, }, } var secrets = []string{ "supersecret", "12345", "a", "SuperSecret", "Sup3r... S3cr3t!", "This is a reasonably long secret key that is used for the purpose of testing.", "\u2603+\u2744", // snowman+snowflake } // TestLayerUploadTokens constructs stateTokens from LayerUploadStates and // validates that the tokens can be used to reconstruct the proper upload state. func TestLayerUploadTokens(t *testing.T) { secret := hmacKey("supersecret") for _, testcase := range layerUploadStates { token, err := secret.packUploadState(testcase) if err != nil { t.Fatal(err) } lus, err := secret.unpackUploadState(token) if err != nil { t.Fatal(err) } assertLayerUploadStateEquals(t, testcase, lus) } } // TestHMACValidate ensures that any HMAC token providers are compatible if and // only if they share the same secret. func TestHMACValidation(t *testing.T) { for _, secret := range secrets { secret1 := hmacKey(secret) secret2 := hmacKey(secret) badSecret := hmacKey("DifferentSecret") for _, testcase := range layerUploadStates { token, err := secret1.packUploadState(testcase) if err != nil { t.Fatal(err) } lus, err := secret2.unpackUploadState(token) if err != nil { t.Fatal(err) } assertLayerUploadStateEquals(t, testcase, lus) _, err = badSecret.unpackUploadState(token) if err == nil { t.Fatalf("Expected token provider to fail at retrieving state from token: %s", token) } badToken, err := badSecret.packUploadState(lus) if err != nil { t.Fatal(err) } _, err = secret1.unpackUploadState(badToken) if err == nil { t.Fatalf("Expected token provider to fail at retrieving state from token: %s", badToken) } _, err = secret2.unpackUploadState(badToken) if err == nil { t.Fatalf("Expected token provider to fail at retrieving state from token: %s", badToken) } } } } func assertLayerUploadStateEquals(t *testing.T, expected layerUploadState, received layerUploadState) { if expected.Name != received.Name { t.Fatalf("Expected Name=%q, Received Name=%q", expected.Name, received.Name) } if expected.UUID != received.UUID { t.Fatalf("Expected UUID=%q, Received UUID=%q", expected.UUID, received.UUID) } if expected.Offset != received.Offset { t.Fatalf("Expected Offset=%d, Received Offset=%d", expected.Offset, received.Offset) } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/handlers/helpers.go0000644000175000017500000000144712502424227026226 0ustar tianontianonpackage handlers import ( "encoding/json" "io" "net/http" ) // serveJSON marshals v and sets the content-type header to // 'application/json'. If a different status code is required, call // ResponseWriter.WriteHeader before this function. func serveJSON(w http.ResponseWriter, v interface{}) error { w.Header().Set("Content-Type", "application/json; charset=utf-8") enc := json.NewEncoder(w) if err := enc.Encode(v); err != nil { return err } return nil } // closeResources closes all the provided resources after running the target // handler. func closeResources(handler http.Handler, closers ...io.Closer) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { for _, closer := range closers { defer closer.Close() } handler.ServeHTTP(w, r) }) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/handlers/layerupload.go0000644000175000017500000002256612502424227027112 0ustar tianontianonpackage handlers import ( "fmt" "io" "net/http" "net/url" "os" "github.com/docker/distribution" ctxu "github.com/docker/distribution/context" "github.com/docker/distribution/digest" "github.com/docker/distribution/registry/api/v2" "github.com/gorilla/handlers" ) // layerUploadDispatcher constructs and returns the layer upload handler for // the given request context. func layerUploadDispatcher(ctx *Context, r *http.Request) http.Handler { luh := &layerUploadHandler{ Context: ctx, UUID: getUploadUUID(ctx), } handler := http.Handler(handlers.MethodHandler{ "POST": http.HandlerFunc(luh.StartLayerUpload), "GET": http.HandlerFunc(luh.GetUploadStatus), "HEAD": http.HandlerFunc(luh.GetUploadStatus), // TODO(stevvooe): Must implement patch support. // "PATCH": http.HandlerFunc(luh.PutLayerChunk), "PUT": http.HandlerFunc(luh.PutLayerUploadComplete), "DELETE": http.HandlerFunc(luh.CancelLayerUpload), }) if luh.UUID != "" { state, err := hmacKey(ctx.Config.HTTP.Secret).unpackUploadState(r.FormValue("_state")) if err != nil { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctxu.GetLogger(ctx).Infof("error resolving upload: %v", err) w.WriteHeader(http.StatusBadRequest) luh.Errors.Push(v2.ErrorCodeBlobUploadInvalid, err) }) } luh.State = state if state.Name != ctx.Repository.Name() { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctxu.GetLogger(ctx).Infof("mismatched repository name in upload state: %q != %q", state.Name, luh.Repository.Name()) w.WriteHeader(http.StatusBadRequest) luh.Errors.Push(v2.ErrorCodeBlobUploadInvalid, err) }) } if state.UUID != luh.UUID { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctxu.GetLogger(ctx).Infof("mismatched uuid in upload state: %q != %q", state.UUID, luh.UUID) w.WriteHeader(http.StatusBadRequest) luh.Errors.Push(v2.ErrorCodeBlobUploadInvalid, err) }) } layers := ctx.Repository.Layers() upload, err := layers.Resume(luh.UUID) if err != nil { ctxu.GetLogger(ctx).Errorf("error resolving upload: %v", err) if err == distribution.ErrLayerUploadUnknown { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) luh.Errors.Push(v2.ErrorCodeBlobUploadUnknown, err) }) } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) luh.Errors.Push(v2.ErrorCodeUnknown, err) }) } luh.Upload = upload if state.Offset > 0 { // Seek the layer upload to the correct spot if it's non-zero. // These error conditions should be rare and demonstrate really // problems. We basically cancel the upload and tell the client to // start over. if nn, err := upload.Seek(luh.State.Offset, os.SEEK_SET); err != nil { defer upload.Close() ctxu.GetLogger(ctx).Infof("error seeking layer upload: %v", err) return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusBadRequest) luh.Errors.Push(v2.ErrorCodeBlobUploadInvalid, err) upload.Cancel() }) } else if nn != luh.State.Offset { defer upload.Close() ctxu.GetLogger(ctx).Infof("seek to wrong offest: %d != %d", nn, luh.State.Offset) return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusBadRequest) luh.Errors.Push(v2.ErrorCodeBlobUploadInvalid, err) upload.Cancel() }) } } handler = closeResources(handler, luh.Upload) } return handler } // layerUploadHandler handles the http layer upload process. type layerUploadHandler struct { *Context // UUID identifies the upload instance for the current request. UUID string Upload distribution.LayerUpload State layerUploadState } // StartLayerUpload begins the layer upload process and allocates a server- // side upload session. func (luh *layerUploadHandler) StartLayerUpload(w http.ResponseWriter, r *http.Request) { layers := luh.Repository.Layers() upload, err := layers.Upload() if err != nil { w.WriteHeader(http.StatusInternalServerError) // Error conditions here? luh.Errors.Push(v2.ErrorCodeUnknown, err) return } luh.Upload = upload defer luh.Upload.Close() if err := luh.layerUploadResponse(w, r); err != nil { w.WriteHeader(http.StatusInternalServerError) // Error conditions here? luh.Errors.Push(v2.ErrorCodeUnknown, err) return } w.Header().Set("Docker-Upload-UUID", luh.Upload.UUID()) w.WriteHeader(http.StatusAccepted) } // GetUploadStatus returns the status of a given upload, identified by uuid. func (luh *layerUploadHandler) GetUploadStatus(w http.ResponseWriter, r *http.Request) { if luh.Upload == nil { w.WriteHeader(http.StatusNotFound) luh.Errors.Push(v2.ErrorCodeBlobUploadUnknown) return } if err := luh.layerUploadResponse(w, r); err != nil { w.WriteHeader(http.StatusInternalServerError) // Error conditions here? luh.Errors.Push(v2.ErrorCodeUnknown, err) return } w.Header().Set("Docker-Upload-UUID", luh.UUID) w.WriteHeader(http.StatusNoContent) } // PutLayerUploadComplete takes the final request of a layer upload. The final // chunk may include all the layer data, the final chunk of layer data or no // layer data. Any data provided is received and verified. If successful, the // layer is linked into the blob store and 201 Created is returned with the // canonical url of the layer. func (luh *layerUploadHandler) PutLayerUploadComplete(w http.ResponseWriter, r *http.Request) { if luh.Upload == nil { w.WriteHeader(http.StatusNotFound) luh.Errors.Push(v2.ErrorCodeBlobUploadUnknown) return } dgstStr := r.FormValue("digest") // TODO(stevvooe): Support multiple digest parameters! if dgstStr == "" { // no digest? return error, but allow retry. w.WriteHeader(http.StatusBadRequest) luh.Errors.Push(v2.ErrorCodeDigestInvalid, "digest missing") return } dgst, err := digest.ParseDigest(dgstStr) if err != nil { // no digest? return error, but allow retry. w.WriteHeader(http.StatusNotFound) luh.Errors.Push(v2.ErrorCodeDigestInvalid, "digest parsing failed") return } // TODO(stevvooe): Check the incoming range header here, per the // specification. LayerUpload should be seeked (sought?) to that position. // TODO(stevvooe): Consider checking the error on this copy. // Theoretically, problems should be detected during verification but we // may miss a root cause. // Read in the final chunk, if any. io.Copy(luh.Upload, r.Body) layer, err := luh.Upload.Finish(dgst) if err != nil { switch err := err.(type) { case distribution.ErrLayerInvalidDigest: w.WriteHeader(http.StatusBadRequest) luh.Errors.Push(v2.ErrorCodeDigestInvalid, err) default: ctxu.GetLogger(luh).Errorf("unknown error completing upload: %#v", err) w.WriteHeader(http.StatusInternalServerError) luh.Errors.Push(v2.ErrorCodeUnknown, err) } // Clean up the backend layer data if there was an error. if err := luh.Upload.Cancel(); err != nil { // If the cleanup fails, all we can do is observe and report. ctxu.GetLogger(luh).Errorf("error canceling upload after error: %v", err) } return } // Build our canonical layer url layerURL, err := luh.urlBuilder.BuildBlobURL(luh.Repository.Name(), layer.Digest()) if err != nil { luh.Errors.Push(v2.ErrorCodeUnknown, err) w.WriteHeader(http.StatusInternalServerError) return } w.Header().Set("Location", layerURL) w.Header().Set("Content-Length", "0") w.Header().Set("Docker-Content-Digest", layer.Digest().String()) w.WriteHeader(http.StatusCreated) } // CancelLayerUpload cancels an in-progress upload of a layer. func (luh *layerUploadHandler) CancelLayerUpload(w http.ResponseWriter, r *http.Request) { if luh.Upload == nil { w.WriteHeader(http.StatusNotFound) luh.Errors.Push(v2.ErrorCodeBlobUploadUnknown) return } w.Header().Set("Docker-Upload-UUID", luh.UUID) if err := luh.Upload.Cancel(); err != nil { ctxu.GetLogger(luh).Errorf("error encountered canceling upload: %v", err) w.WriteHeader(http.StatusInternalServerError) luh.Errors.PushErr(err) } w.WriteHeader(http.StatusNoContent) } // layerUploadResponse provides a standard request for uploading layers and // chunk responses. This sets the correct headers but the response status is // left to the caller. func (luh *layerUploadHandler) layerUploadResponse(w http.ResponseWriter, r *http.Request) error { offset, err := luh.Upload.Seek(0, os.SEEK_CUR) if err != nil { ctxu.GetLogger(luh).Errorf("unable get current offset of layer upload: %v", err) return err } // TODO(stevvooe): Need a better way to manage the upload state automatically. luh.State.Name = luh.Repository.Name() luh.State.UUID = luh.Upload.UUID() luh.State.Offset = offset luh.State.StartedAt = luh.Upload.StartedAt() token, err := hmacKey(luh.Config.HTTP.Secret).packUploadState(luh.State) if err != nil { ctxu.GetLogger(luh).Infof("error building upload state token: %s", err) return err } uploadURL, err := luh.urlBuilder.BuildBlobUploadChunkURL( luh.Repository.Name(), luh.Upload.UUID(), url.Values{ "_state": []string{token}, }) if err != nil { ctxu.GetLogger(luh).Infof("error building upload url: %s", err) return err } w.Header().Set("Docker-Upload-UUID", luh.UUID) w.Header().Set("Location", uploadURL) w.Header().Set("Content-Length", "0") w.Header().Set("Range", fmt.Sprintf("0-%d", luh.State.Offset)) return nil } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/handlers/app_test.go0000644000175000017500000001650712502424227026406 0ustar tianontianonpackage handlers import ( "encoding/json" "net/http" "net/http/httptest" "net/url" "reflect" "testing" "github.com/docker/distribution/configuration" "github.com/docker/distribution/registry/api/v2" "github.com/docker/distribution/registry/auth" _ "github.com/docker/distribution/registry/auth/silly" "github.com/docker/distribution/registry/storage" "github.com/docker/distribution/registry/storage/driver/inmemory" "golang.org/x/net/context" ) // TestAppDispatcher builds an application with a test dispatcher and ensures // that requests are properly dispatched and the handlers are constructed. // This only tests the dispatch mechanism. The underlying dispatchers must be // tested individually. func TestAppDispatcher(t *testing.T) { driver := inmemory.New() app := &App{ Config: configuration.Configuration{}, Context: context.Background(), router: v2.Router(), driver: driver, registry: storage.NewRegistryWithDriver(driver), } server := httptest.NewServer(app) router := v2.Router() serverURL, err := url.Parse(server.URL) if err != nil { t.Fatalf("error parsing server url: %v", err) } varCheckingDispatcher := func(expectedVars map[string]string) dispatchFunc { return func(ctx *Context, r *http.Request) http.Handler { // Always checks the same name context if ctx.Repository.Name() != getName(ctx) { t.Fatalf("unexpected name: %q != %q", ctx.Repository.Name(), "foo/bar") } // Check that we have all that is expected for expectedK, expectedV := range expectedVars { if ctx.Value(expectedK) != expectedV { t.Fatalf("unexpected %s in context vars: %q != %q", expectedK, ctx.Value(expectedK), expectedV) } } // Check that we only have variables that are expected for k, v := range ctx.Value("vars").(map[string]string) { _, ok := expectedVars[k] if !ok { // name is checked on context // We have an unexpected key, fail t.Fatalf("unexpected key %q in vars with value %q", k, v) } } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) }) } } // unflatten a list of variables, suitable for gorilla/mux, to a map[string]string unflatten := func(vars []string) map[string]string { m := make(map[string]string) for i := 0; i < len(vars)-1; i = i + 2 { m[vars[i]] = vars[i+1] } return m } for _, testcase := range []struct { endpoint string vars []string }{ { endpoint: v2.RouteNameManifest, vars: []string{ "name", "foo/bar", "reference", "sometag", }, }, { endpoint: v2.RouteNameTags, vars: []string{ "name", "foo/bar", }, }, { endpoint: v2.RouteNameBlob, vars: []string{ "name", "foo/bar", "digest", "tarsum.v1+bogus:abcdef0123456789", }, }, { endpoint: v2.RouteNameBlobUpload, vars: []string{ "name", "foo/bar", }, }, { endpoint: v2.RouteNameBlobUploadChunk, vars: []string{ "name", "foo/bar", "uuid", "theuuid", }, }, } { app.register(testcase.endpoint, varCheckingDispatcher(unflatten(testcase.vars))) route := router.GetRoute(testcase.endpoint).Host(serverURL.Host) u, err := route.URL(testcase.vars...) if err != nil { t.Fatal(err) } resp, err := http.Get(u.String()) if err != nil { t.Fatal(err) } if resp.StatusCode != http.StatusOK { t.Fatalf("unexpected status code: %v != %v", resp.StatusCode, http.StatusOK) } } } // TestNewApp covers the creation of an application via NewApp with a // configuration. func TestNewApp(t *testing.T) { ctx := context.Background() config := configuration.Configuration{ Storage: configuration.Storage{ "inmemory": nil, }, Auth: configuration.Auth{ // For now, we simply test that new auth results in a viable // application. "silly": { "realm": "realm-test", "service": "service-test", }, }, } // Mostly, with this test, given a sane configuration, we are simply // ensuring that NewApp doesn't panic. We might want to tweak this // behavior. app := NewApp(ctx, config) server := httptest.NewServer(app) builder, err := v2.NewURLBuilderFromString(server.URL) if err != nil { t.Fatalf("error creating urlbuilder: %v", err) } baseURL, err := builder.BuildBaseURL() if err != nil { t.Fatalf("error creating baseURL: %v", err) } // TODO(stevvooe): The rest of this test might belong in the API tests. // Just hit the app and make sure we get a 401 Unauthorized error. req, err := http.Get(baseURL) if err != nil { t.Fatalf("unexpected error during GET: %v", err) } defer req.Body.Close() if req.StatusCode != http.StatusUnauthorized { t.Fatalf("unexpected status code during request: %v", err) } if req.Header.Get("Content-Type") != "application/json; charset=utf-8" { t.Fatalf("unexpected content-type: %v != %v", req.Header.Get("Content-Type"), "application/json; charset=utf-8") } expectedAuthHeader := "Bearer realm=\"realm-test\",service=\"service-test\"" if e, a := expectedAuthHeader, req.Header.Get("WWW-Authenticate"); e != a { t.Fatalf("unexpected WWW-Authenticate header: %q != %q", e, a) } var errs v2.Errors dec := json.NewDecoder(req.Body) if err := dec.Decode(&errs); err != nil { t.Fatalf("error decoding error response: %v", err) } if errs.Errors[0].Code != v2.ErrorCodeUnauthorized { t.Fatalf("unexpected error code: %v != %v", errs.Errors[0].Code, v2.ErrorCodeUnauthorized) } } // Test the access record accumulator func TestAppendAccessRecords(t *testing.T) { repo := "testRepo" expectedResource := auth.Resource{ Type: "repository", Name: repo, } expectedPullRecord := auth.Access{ Resource: expectedResource, Action: "pull", } expectedPushRecord := auth.Access{ Resource: expectedResource, Action: "push", } expectedAllRecord := auth.Access{ Resource: expectedResource, Action: "*", } records := []auth.Access{} result := appendAccessRecords(records, "GET", repo) expectedResult := []auth.Access{expectedPullRecord} if ok := reflect.DeepEqual(result, expectedResult); !ok { t.Fatalf("Actual access record differs from expected") } records = []auth.Access{} result = appendAccessRecords(records, "HEAD", repo) expectedResult = []auth.Access{expectedPullRecord} if ok := reflect.DeepEqual(result, expectedResult); !ok { t.Fatalf("Actual access record differs from expected") } records = []auth.Access{} result = appendAccessRecords(records, "POST", repo) expectedResult = []auth.Access{expectedPullRecord, expectedPushRecord} if ok := reflect.DeepEqual(result, expectedResult); !ok { t.Fatalf("Actual access record differs from expected") } records = []auth.Access{} result = appendAccessRecords(records, "PUT", repo) expectedResult = []auth.Access{expectedPullRecord, expectedPushRecord} if ok := reflect.DeepEqual(result, expectedResult); !ok { t.Fatalf("Actual access record differs from expected") } records = []auth.Access{} result = appendAccessRecords(records, "PATCH", repo) expectedResult = []auth.Access{expectedPullRecord, expectedPushRecord} if ok := reflect.DeepEqual(result, expectedResult); !ok { t.Fatalf("Actual access record differs from expected") } records = []auth.Access{} result = appendAccessRecords(records, "DELETE", repo) expectedResult = []auth.Access{expectedAllRecord} if ok := reflect.DeepEqual(result, expectedResult); !ok { t.Fatalf("Actual access record differs from expected") } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/handlers/hmac.go0000644000175000017500000000336612502424227025476 0ustar tianontianonpackage handlers import ( "crypto/hmac" "crypto/sha256" "encoding/base64" "encoding/json" "fmt" "time" ) // layerUploadState captures the state serializable state of the layer upload. type layerUploadState struct { // name is the primary repository under which the layer will be linked. Name string // UUID identifies the upload. UUID string // offset contains the current progress of the upload. Offset int64 // StartedAt is the original start time of the upload. StartedAt time.Time } type hmacKey string // unpackUploadState unpacks and validates the layer upload state from the // token, using the hmacKey secret. func (secret hmacKey) unpackUploadState(token string) (layerUploadState, error) { var state layerUploadState tokenBytes, err := base64.URLEncoding.DecodeString(token) if err != nil { return state, err } mac := hmac.New(sha256.New, []byte(secret)) if len(tokenBytes) < mac.Size() { return state, fmt.Errorf("Invalid token") } macBytes := tokenBytes[:mac.Size()] messageBytes := tokenBytes[mac.Size():] mac.Write(messageBytes) if !hmac.Equal(mac.Sum(nil), macBytes) { return state, fmt.Errorf("Invalid token") } if err := json.Unmarshal(messageBytes, &state); err != nil { return state, err } return state, nil } // packUploadState packs the upload state signed with and hmac digest using // the hmacKey secret, encoding to url safe base64. The resulting token can be // used to share data with minimized risk of external tampering. func (secret hmacKey) packUploadState(lus layerUploadState) (string, error) { mac := hmac.New(sha256.New, []byte(secret)) p, err := json.Marshal(lus) if err != nil { return "", err } mac.Write(p) return base64.URLEncoding.EncodeToString(append(mac.Sum(nil), p...)), nil } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/handlers/basicauth_prego14.go0000644000175000017500000000201212502424227030055 0ustar tianontianon// +build !go1.4 package handlers import ( "encoding/base64" "net/http" "strings" ) // NOTE(stevvooe): This is basic auth support from go1.4 present to ensure we // can compile on go1.3 and earlier. // BasicAuth returns the username and password provided in the request's // Authorization header, if the request uses HTTP Basic Authentication. // See RFC 2617, Section 2. func basicAuth(r *http.Request) (username, password string, ok bool) { auth := r.Header.Get("Authorization") if auth == "" { return } return parseBasicAuth(auth) } // parseBasicAuth parses an HTTP Basic Authentication string. // "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==" returns ("Aladdin", "open sesame", true). func parseBasicAuth(auth string) (username, password string, ok bool) { if !strings.HasPrefix(auth, "Basic ") { return } c, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(auth, "Basic ")) if err != nil { return } cs := string(c) s := strings.IndexByte(cs, ':') if s < 0 { return } return cs[:s], cs[s+1:], true } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/handlers/basicauth.go0000644000175000017500000000023212502424227026516 0ustar tianontianon// +build go1.4 package handlers import ( "net/http" ) func basicAuth(r *http.Request) (username, password string, ok bool) { return r.BasicAuth() } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/handlers/app.go0000644000175000017500000003636212502424227025350 0ustar tianontianonpackage handlers import ( "fmt" "net" "net/http" "os" "code.google.com/p/go-uuid/uuid" "github.com/docker/distribution" "github.com/docker/distribution/configuration" ctxu "github.com/docker/distribution/context" "github.com/docker/distribution/notifications" "github.com/docker/distribution/registry/api/v2" "github.com/docker/distribution/registry/auth" registrymiddleware "github.com/docker/distribution/registry/middleware/registry" repositorymiddleware "github.com/docker/distribution/registry/middleware/repository" "github.com/docker/distribution/registry/storage" storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/distribution/registry/storage/driver/factory" storagemiddleware "github.com/docker/distribution/registry/storage/driver/middleware" "github.com/gorilla/mux" "golang.org/x/net/context" ) // App is a global registry application object. Shared resources can be placed // on this object that will be accessible from all requests. Any writable // fields should be protected. type App struct { context.Context Config configuration.Configuration // InstanceID is a unique id assigned to the application on each creation. // Provides information in the logs and context to identify restarts. InstanceID string router *mux.Router // main application router, configured with dispatchers driver storagedriver.StorageDriver // driver maintains the app global storage driver instance. registry distribution.Registry // registry is the primary registry backend for the app instance. accessController auth.AccessController // main access controller for application // events contains notification related configuration. events struct { sink notifications.Sink source notifications.SourceRecord } } // Value intercepts calls context.Context.Value, returning the current app id, // if requested. func (app *App) Value(key interface{}) interface{} { switch key { case "app.id": return app.InstanceID } return app.Context.Value(key) } // NewApp takes a configuration and returns a configured app, ready to serve // requests. The app only implements ServeHTTP and can be wrapped in other // handlers accordingly. func NewApp(ctx context.Context, configuration configuration.Configuration) *App { app := &App{ Config: configuration, Context: ctx, InstanceID: uuid.New(), router: v2.RouterWithPrefix(configuration.HTTP.Prefix), } app.Context = ctxu.WithLogger(app.Context, ctxu.GetLogger(app, "app.id")) // Register the handler dispatchers. app.register(v2.RouteNameBase, func(ctx *Context, r *http.Request) http.Handler { return http.HandlerFunc(apiBase) }) app.register(v2.RouteNameManifest, imageManifestDispatcher) app.register(v2.RouteNameTags, tagsDispatcher) app.register(v2.RouteNameBlob, layerDispatcher) app.register(v2.RouteNameBlobUpload, layerUploadDispatcher) app.register(v2.RouteNameBlobUploadChunk, layerUploadDispatcher) var err error app.driver, err = factory.Create(configuration.Storage.Type(), configuration.Storage.Parameters()) if err != nil { // TODO(stevvooe): Move the creation of a service into a protected // method, where this is created lazily. Its status can be queried via // a health check. panic(err) } app.driver, err = applyStorageMiddleware(app.driver, configuration.Middleware["storage"]) if err != nil { panic(err) } app.configureEvents(&configuration) app.registry = storage.NewRegistryWithDriver(app.driver) app.registry, err = applyRegistryMiddleware(app.registry, configuration.Middleware["registry"]) if err != nil { panic(err) } authType := configuration.Auth.Type() if authType != "" { accessController, err := auth.GetAccessController(configuration.Auth.Type(), configuration.Auth.Parameters()) if err != nil { panic(fmt.Sprintf("unable to configure authorization (%s): %v", authType, err)) } app.accessController = accessController } return app } // register a handler with the application, by route name. The handler will be // passed through the application filters and context will be constructed at // request time. func (app *App) register(routeName string, dispatch dispatchFunc) { // TODO(stevvooe): This odd dispatcher/route registration is by-product of // some limitations in the gorilla/mux router. We are using it to keep // routing consistent between the client and server, but we may want to // replace it with manual routing and structure-based dispatch for better // control over the request execution. app.router.GetRoute(routeName).Handler(app.dispatcher(dispatch)) } // configureEvents prepares the event sink for action. func (app *App) configureEvents(configuration *configuration.Configuration) { // Configure all of the endpoint sinks. var sinks []notifications.Sink for _, endpoint := range configuration.Notifications.Endpoints { if endpoint.Disabled { ctxu.GetLogger(app).Infof("endpoint %s disabled, skipping", endpoint.Name) continue } ctxu.GetLogger(app).Infof("configuring endpoint %v (%v), timeout=%s, headers=%v", endpoint.Name, endpoint.URL, endpoint.Timeout, endpoint.Headers) endpoint := notifications.NewEndpoint(endpoint.Name, endpoint.URL, notifications.EndpointConfig{ Timeout: endpoint.Timeout, Threshold: endpoint.Threshold, Backoff: endpoint.Backoff, Headers: endpoint.Headers, }) sinks = append(sinks, endpoint) } // NOTE(stevvooe): Moving to a new queueing implementation is as easy as // replacing broadcaster with a rabbitmq implementation. It's recommended // that the registry instances also act as the workers to keep deployment // simple. app.events.sink = notifications.NewBroadcaster(sinks...) // Populate registry event source hostname, err := os.Hostname() if err != nil { hostname = configuration.HTTP.Addr } else { // try to pick the port off the config _, port, err := net.SplitHostPort(configuration.HTTP.Addr) if err == nil { hostname = net.JoinHostPort(hostname, port) } } app.events.source = notifications.SourceRecord{ Addr: hostname, InstanceID: app.InstanceID, } } func (app *App) ServeHTTP(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() // ensure that request body is always closed. // Set a header with the Docker Distribution API Version for all responses. w.Header().Add("Docker-Distribution-API-Version", "registry/2.0") app.router.ServeHTTP(w, r) } // dispatchFunc takes a context and request and returns a constructed handler // for the route. The dispatcher will use this to dynamically create request // specific handlers for each endpoint without creating a new router for each // request. type dispatchFunc func(ctx *Context, r *http.Request) http.Handler // TODO(stevvooe): dispatchers should probably have some validation error // chain with proper error reporting. // singleStatusResponseWriter only allows the first status to be written to be // the valid request status. The current use case of this class should be // factored out. type singleStatusResponseWriter struct { http.ResponseWriter status int } func (ssrw *singleStatusResponseWriter) WriteHeader(status int) { if ssrw.status != 0 { return } ssrw.status = status ssrw.ResponseWriter.WriteHeader(status) } func (ssrw *singleStatusResponseWriter) Flush() { if flusher, ok := ssrw.ResponseWriter.(http.Flusher); ok { flusher.Flush() } } // dispatcher returns a handler that constructs a request specific context and // handler, using the dispatch factory function. func (app *App) dispatcher(dispatch dispatchFunc) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { context := app.context(w, r) defer func() { ctxu.GetResponseLogger(context).Infof("response completed") }() if err := app.authorized(w, r, context); err != nil { ctxu.GetLogger(context).Errorf("error authorizing context: %v", err) return } if app.nameRequired(r) { repository, err := app.registry.Repository(context, getName(context)) if err != nil { ctxu.GetLogger(context).Errorf("error resolving repository: %v", err) switch err := err.(type) { case distribution.ErrRepositoryUnknown: context.Errors.Push(v2.ErrorCodeNameUnknown, err) case distribution.ErrRepositoryNameInvalid: context.Errors.Push(v2.ErrorCodeNameInvalid, err) } w.WriteHeader(http.StatusBadRequest) serveJSON(w, context.Errors) return } // assign and decorate the authorized repository with an event bridge. context.Repository = notifications.Listen( repository, app.eventBridge(context, r)) context.Repository, err = applyRepoMiddleware(context.Repository, app.Config.Middleware["repository"]) if err != nil { ctxu.GetLogger(context).Errorf("error initializing repository middleware: %v", err) context.Errors.Push(v2.ErrorCodeUnknown, err) w.WriteHeader(http.StatusInternalServerError) serveJSON(w, context.Errors) return } } handler := dispatch(context, r) ssrw := &singleStatusResponseWriter{ResponseWriter: w} handler.ServeHTTP(ssrw, r) // Automated error response handling here. Handlers may return their // own errors if they need different behavior (such as range errors // for layer upload). if context.Errors.Len() > 0 { if ssrw.status == 0 { w.WriteHeader(http.StatusBadRequest) } serveJSON(w, context.Errors) } }) } // context constructs the context object for the application. This only be // called once per request. func (app *App) context(w http.ResponseWriter, r *http.Request) *Context { ctx := ctxu.WithRequest(app, r) ctx, w = ctxu.WithResponseWriter(ctx, w) ctx = ctxu.WithVars(ctx, r) ctx = ctxu.WithLogger(ctx, ctxu.GetRequestLogger(ctx)) ctx = ctxu.WithLogger(ctx, ctxu.GetLogger(ctx, "vars.name", "vars.reference", "vars.digest", "vars.uuid")) context := &Context{ App: app, Context: ctx, urlBuilder: v2.NewURLBuilderFromRequest(r), } return context } // authorized checks if the request can proceed with access to the requested // repository. If it succeeds, the context may access the requested // repository. An error will be returned if access is not available. func (app *App) authorized(w http.ResponseWriter, r *http.Request, context *Context) error { ctxu.GetLogger(context).Debug("authorizing request") repo := getName(context) if app.accessController == nil { return nil // access controller is not enabled. } var accessRecords []auth.Access if repo != "" { accessRecords = appendAccessRecords(accessRecords, r.Method, repo) } else { // Only allow the name not to be set on the base route. if app.nameRequired(r) { // For this to be properly secured, repo must always be set for a // resource that may make a modification. The only condition under // which name is not set and we still allow access is when the // base route is accessed. This section prevents us from making // that mistake elsewhere in the code, allowing any operation to // proceed. w.Header().Set("Content-Type", "application/json; charset=utf-8") w.WriteHeader(http.StatusForbidden) var errs v2.Errors errs.Push(v2.ErrorCodeUnauthorized) serveJSON(w, errs) return fmt.Errorf("forbidden: no repository name") } } ctx, err := app.accessController.Authorized(context.Context, accessRecords...) if err != nil { switch err := err.(type) { case auth.Challenge: w.Header().Set("Content-Type", "application/json; charset=utf-8") err.ServeHTTP(w, r) var errs v2.Errors errs.Push(v2.ErrorCodeUnauthorized, accessRecords) serveJSON(w, errs) default: // This condition is a potential security problem either in // the configuration or whatever is backing the access // controller. Just return a bad request with no information // to avoid exposure. The request should not proceed. ctxu.GetLogger(context).Errorf("error checking authorization: %v", err) w.WriteHeader(http.StatusBadRequest) } return err } // TODO(stevvooe): This pattern needs to be cleaned up a bit. One context // should be replaced by another, rather than replacing the context on a // mutable object. context.Context = ctx return nil } // eventBridge returns a bridge for the current request, configured with the // correct actor and source. func (app *App) eventBridge(ctx *Context, r *http.Request) notifications.Listener { actor := notifications.ActorRecord{ Name: getUserName(ctx, r), } request := notifications.NewRequestRecord(ctxu.GetRequestID(ctx), r) return notifications.NewBridge(ctx.urlBuilder, app.events.source, actor, request, app.events.sink) } // nameRequired returns true if the route requires a name. func (app *App) nameRequired(r *http.Request) bool { route := mux.CurrentRoute(r) return route == nil || route.GetName() != v2.RouteNameBase } // apiBase implements a simple yes-man for doing overall checks against the // api. This can support auth roundtrips to support docker login. func apiBase(w http.ResponseWriter, r *http.Request) { const emptyJSON = "{}" // Provide a simple /v2/ 200 OK response with empty json response. w.Header().Set("Content-Type", "application/json; charset=utf-8") w.Header().Set("Content-Length", fmt.Sprint(len(emptyJSON))) fmt.Fprint(w, emptyJSON) } // appendAccessRecords checks the method and adds the appropriate Access records to the records list. func appendAccessRecords(records []auth.Access, method string, repo string) []auth.Access { resource := auth.Resource{ Type: "repository", Name: repo, } switch method { case "GET", "HEAD": records = append(records, auth.Access{ Resource: resource, Action: "pull", }) case "POST", "PUT", "PATCH": records = append(records, auth.Access{ Resource: resource, Action: "pull", }, auth.Access{ Resource: resource, Action: "push", }) case "DELETE": // DELETE access requires full admin rights, which is represented // as "*". This may not be ideal. records = append(records, auth.Access{ Resource: resource, Action: "*", }) } return records } // applyRegistryMiddleware wraps a registry instance with the configured middlewares func applyRegistryMiddleware(registry distribution.Registry, middlewares []configuration.Middleware) (distribution.Registry, error) { for _, mw := range middlewares { rmw, err := registrymiddleware.Get(mw.Name, mw.Options, registry) if err != nil { return nil, fmt.Errorf("unable to configure registry middleware (%s): %s", mw.Name, err) } registry = rmw } return registry, nil } // applyRepoMiddleware wraps a repository with the configured middlewares func applyRepoMiddleware(repository distribution.Repository, middlewares []configuration.Middleware) (distribution.Repository, error) { for _, mw := range middlewares { rmw, err := repositorymiddleware.Get(mw.Name, mw.Options, repository) if err != nil { return nil, err } repository = rmw } return repository, nil } // applyStorageMiddleware wraps a storage driver with the configured middlewares func applyStorageMiddleware(driver storagedriver.StorageDriver, middlewares []configuration.Middleware) (storagedriver.StorageDriver, error) { for _, mw := range middlewares { smw, err := storagemiddleware.Get(mw.Name, mw.Options, driver) if err != nil { return nil, fmt.Errorf("unable to configure storage middleware (%s): %v", mw.Name, err) } driver = smw } return driver, nil } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/handlers/context.go0000644000175000017500000000504312502424227026244 0ustar tianontianonpackage handlers import ( "fmt" "net/http" "github.com/docker/distribution" ctxu "github.com/docker/distribution/context" "github.com/docker/distribution/digest" "github.com/docker/distribution/registry/api/v2" "golang.org/x/net/context" ) // Context should contain the request specific context for use in across // handlers. Resources that don't need to be shared across handlers should not // be on this object. type Context struct { // App points to the application structure that created this context. *App context.Context // Repository is the repository for the current request. All requests // should be scoped to a single repository. This field may be nil. Repository distribution.Repository // Errors is a collection of errors encountered during the request to be // returned to the client API. If errors are added to the collection, the // handler *must not* start the response via http.ResponseWriter. Errors v2.Errors urlBuilder *v2.URLBuilder // TODO(stevvooe): The goal is too completely factor this context and // dispatching out of the web application. Ideally, we should lean on // context.Context for injection of these resources. } // Value overrides context.Context.Value to ensure that calls are routed to // correct context. func (ctx *Context) Value(key interface{}) interface{} { return ctx.Context.Value(key) } func getName(ctx context.Context) (name string) { return ctxu.GetStringValue(ctx, "vars.name") } func getReference(ctx context.Context) (reference string) { return ctxu.GetStringValue(ctx, "vars.reference") } var errDigestNotAvailable = fmt.Errorf("digest not available in context") func getDigest(ctx context.Context) (dgst digest.Digest, err error) { dgstStr := ctxu.GetStringValue(ctx, "vars.digest") if dgstStr == "" { ctxu.GetLogger(ctx).Errorf("digest not available") return "", errDigestNotAvailable } d, err := digest.ParseDigest(dgstStr) if err != nil { ctxu.GetLogger(ctx).Errorf("error parsing digest=%q: %v", dgstStr, err) return "", err } return d, nil } func getUploadUUID(ctx context.Context) (uuid string) { return ctxu.GetStringValue(ctx, "vars.uuid") } // getUserName attempts to resolve a username from the context and request. If // a username cannot be resolved, the empty string is returned. func getUserName(ctx context.Context, r *http.Request) string { username := ctxu.GetStringValue(ctx, "auth.user.name") // Fallback to request user with basic auth if username == "" { var ok bool uname, _, ok := basicAuth(r) if ok { username = uname } } return username } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/handlers/images.go0000644000175000017500000001475612502424227026040 0ustar tianontianonpackage handlers import ( "encoding/json" "fmt" "net/http" "strings" "github.com/docker/distribution" ctxu "github.com/docker/distribution/context" "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest" "github.com/docker/distribution/registry/api/v2" "github.com/gorilla/handlers" "golang.org/x/net/context" ) // imageManifestDispatcher takes the request context and builds the // appropriate handler for handling image manifest requests. func imageManifestDispatcher(ctx *Context, r *http.Request) http.Handler { imageManifestHandler := &imageManifestHandler{ Context: ctx, } reference := getReference(ctx) dgst, err := digest.ParseDigest(reference) if err != nil { // We just have a tag imageManifestHandler.Tag = reference } else { imageManifestHandler.Digest = dgst } return handlers.MethodHandler{ "GET": http.HandlerFunc(imageManifestHandler.GetImageManifest), "PUT": http.HandlerFunc(imageManifestHandler.PutImageManifest), "DELETE": http.HandlerFunc(imageManifestHandler.DeleteImageManifest), } } // imageManifestHandler handles http operations on image manifests. type imageManifestHandler struct { *Context // One of tag or digest gets set, depending on what is present in context. Tag string Digest digest.Digest } // GetImageManifest fetches the image manifest from the storage backend, if it exists. func (imh *imageManifestHandler) GetImageManifest(w http.ResponseWriter, r *http.Request) { ctxu.GetLogger(imh).Debug("GetImageManifest") manifests := imh.Repository.Manifests() var ( sm *manifest.SignedManifest err error ) if imh.Tag != "" { sm, err = manifests.GetByTag(imh.Tag) } else { sm, err = manifests.Get(imh.Digest) } if err != nil { imh.Errors.Push(v2.ErrorCodeManifestUnknown, err) w.WriteHeader(http.StatusNotFound) return } // Get the digest, if we don't already have it. if imh.Digest == "" { dgst, err := digestManifest(imh, sm) if err != nil { imh.Errors.Push(v2.ErrorCodeDigestInvalid, err) w.WriteHeader(http.StatusBadRequest) return } imh.Digest = dgst } w.Header().Set("Content-Type", "application/json; charset=utf-8") w.Header().Set("Content-Length", fmt.Sprint(len(sm.Raw))) w.Header().Set("Docker-Content-Digest", imh.Digest.String()) w.Write(sm.Raw) } // PutImageManifest validates and stores and image in the registry. func (imh *imageManifestHandler) PutImageManifest(w http.ResponseWriter, r *http.Request) { ctxu.GetLogger(imh).Debug("PutImageManifest") manifests := imh.Repository.Manifests() dec := json.NewDecoder(r.Body) var manifest manifest.SignedManifest if err := dec.Decode(&manifest); err != nil { imh.Errors.Push(v2.ErrorCodeManifestInvalid, err) w.WriteHeader(http.StatusBadRequest) return } dgst, err := digestManifest(imh, &manifest) if err != nil { imh.Errors.Push(v2.ErrorCodeDigestInvalid, err) w.WriteHeader(http.StatusBadRequest) return } // Validate manifest tag or digest matches payload if imh.Tag != "" { if manifest.Tag != imh.Tag { ctxu.GetLogger(imh).Errorf("invalid tag on manifest payload: %q != %q", manifest.Tag, imh.Tag) imh.Errors.Push(v2.ErrorCodeTagInvalid) w.WriteHeader(http.StatusBadRequest) return } imh.Digest = dgst } else if imh.Digest != "" { if dgst != imh.Digest { ctxu.GetLogger(imh).Errorf("payload digest does match: %q != %q", dgst, imh.Digest) imh.Errors.Push(v2.ErrorCodeDigestInvalid) w.WriteHeader(http.StatusBadRequest) return } } else { imh.Errors.Push(v2.ErrorCodeTagInvalid, "no tag or digest specified") w.WriteHeader(http.StatusBadRequest) return } if err := manifests.Put(&manifest); err != nil { // TODO(stevvooe): These error handling switches really need to be // handled by an app global mapper. switch err := err.(type) { case distribution.ErrManifestVerification: for _, verificationError := range err { switch verificationError := verificationError.(type) { case distribution.ErrUnknownLayer: imh.Errors.Push(v2.ErrorCodeBlobUnknown, verificationError.FSLayer) case distribution.ErrManifestUnverified: imh.Errors.Push(v2.ErrorCodeManifestUnverified) default: if verificationError == digest.ErrDigestInvalidFormat { // TODO(stevvooe): We need to really need to move all // errors to types. Its much more straightforward. imh.Errors.Push(v2.ErrorCodeDigestInvalid) } else { imh.Errors.PushErr(verificationError) } } } default: imh.Errors.PushErr(err) } w.WriteHeader(http.StatusBadRequest) return } // Construct a canonical url for the uploaded manifest. location, err := imh.urlBuilder.BuildManifestURL(imh.Repository.Name(), imh.Digest.String()) if err != nil { // NOTE(stevvooe): Given the behavior above, this absurdly unlikely to // happen. We'll log the error here but proceed as if it worked. Worst // case, we set an empty location header. ctxu.GetLogger(imh).Errorf("error building manifest url from digest: %v", err) } w.Header().Set("Location", location) w.Header().Set("Docker-Content-Digest", imh.Digest.String()) w.WriteHeader(http.StatusAccepted) } // DeleteImageManifest removes the image with the given tag from the registry. func (imh *imageManifestHandler) DeleteImageManifest(w http.ResponseWriter, r *http.Request) { ctxu.GetLogger(imh).Debug("DeleteImageManifest") // TODO(stevvooe): Unfortunately, at this point, manifest deletes are // unsupported. There are issues with schema version 1 that make removing // tag index entries a serious problem in eventually consistent storage. // Once we work out schema version 2, the full deletion system will be // worked out and we can add support back. imh.Errors.Push(v2.ErrorCodeUnsupported) w.WriteHeader(http.StatusBadRequest) } // digestManifest takes a digest of the given manifest. This belongs somewhere // better but we'll wait for a refactoring cycle to find that real somewhere. func digestManifest(ctx context.Context, sm *manifest.SignedManifest) (digest.Digest, error) { p, err := sm.Payload() if err != nil { if !strings.Contains(err.Error(), "missing signature key") { ctxu.GetLogger(ctx).Errorf("error getting manifest payload: %v", err) return "", err } // NOTE(stevvooe): There are no signatures but we still have a // payload. The request will fail later but this is not the // responsibility of this part of the code. p = sm.Raw } dgst, err := digest.FromBytes(p) if err != nil { ctxu.GetLogger(ctx).Errorf("error digesting manifest: %v", err) return "", err } return dgst, err } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/handlers/tags.go0000644000175000017500000000252612502424227025521 0ustar tianontianonpackage handlers import ( "encoding/json" "net/http" "github.com/docker/distribution" "github.com/docker/distribution/registry/api/v2" "github.com/gorilla/handlers" ) // tagsDispatcher constructs the tags handler api endpoint. func tagsDispatcher(ctx *Context, r *http.Request) http.Handler { tagsHandler := &tagsHandler{ Context: ctx, } return handlers.MethodHandler{ "GET": http.HandlerFunc(tagsHandler.GetTags), } } // tagsHandler handles requests for lists of tags under a repository name. type tagsHandler struct { *Context } type tagsAPIResponse struct { Name string `json:"name"` Tags []string `json:"tags"` } // GetTags returns a json list of tags for a specific image name. func (th *tagsHandler) GetTags(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() manifests := th.Repository.Manifests() tags, err := manifests.Tags() if err != nil { switch err := err.(type) { case distribution.ErrRepositoryUnknown: w.WriteHeader(404) th.Errors.Push(v2.ErrorCodeNameUnknown, map[string]string{"name": th.Repository.Name()}) default: th.Errors.PushErr(err) } return } w.Header().Set("Content-Type", "application/json; charset=utf-8") enc := json.NewEncoder(w) if err := enc.Encode(tagsAPIResponse{ Name: th.Repository.Name(), Tags: tags, }); err != nil { th.Errors.PushErr(err) return } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/handlers/layer.go0000644000175000017500000000345512502424227025701 0ustar tianontianonpackage handlers import ( "net/http" "github.com/docker/distribution" ctxu "github.com/docker/distribution/context" "github.com/docker/distribution/digest" "github.com/docker/distribution/registry/api/v2" "github.com/gorilla/handlers" ) // layerDispatcher uses the request context to build a layerHandler. func layerDispatcher(ctx *Context, r *http.Request) http.Handler { dgst, err := getDigest(ctx) if err != nil { if err == errDigestNotAvailable { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) ctx.Errors.Push(v2.ErrorCodeDigestInvalid, err) }) } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx.Errors.Push(v2.ErrorCodeDigestInvalid, err) }) } layerHandler := &layerHandler{ Context: ctx, Digest: dgst, } return handlers.MethodHandler{ "GET": http.HandlerFunc(layerHandler.GetLayer), "HEAD": http.HandlerFunc(layerHandler.GetLayer), } } // layerHandler serves http layer requests. type layerHandler struct { *Context Digest digest.Digest } // GetLayer fetches the binary data from backend storage returns it in the // response. func (lh *layerHandler) GetLayer(w http.ResponseWriter, r *http.Request) { ctxu.GetLogger(lh).Debug("GetImageLayer") layers := lh.Repository.Layers() layer, err := layers.Fetch(lh.Digest) if err != nil { switch err := err.(type) { case distribution.ErrUnknownLayer: w.WriteHeader(http.StatusNotFound) lh.Errors.Push(v2.ErrorCodeBlobUnknown, err.FSLayer) default: lh.Errors.Push(v2.ErrorCodeUnknown, err) } return } handler, err := layer.Handler(r) if err != nil { ctxu.GetLogger(lh).Debugf("unexpected error getting layer HTTP handler: %s", err) lh.Errors.Push(v2.ErrorCodeUnknown, err) return } handler.ServeHTTP(w, r) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/auth/0000755000175000017500000000000012502424227023370 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/auth/silly/0000755000175000017500000000000012502424227024524 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/auth/silly/access.go0000644000175000017500000000530612502424227026320 0ustar tianontianon// Package silly provides a simple authentication scheme that checks for the // existence of an Authorization header and issues access if is present and // non-empty. // // This package is present as an example implementation of a minimal // auth.AccessController and for testing. This is not suitable for any kind of // production security. package silly import ( "fmt" "net/http" "strings" ctxu "github.com/docker/distribution/context" "github.com/docker/distribution/registry/auth" "golang.org/x/net/context" ) // accessController provides a simple implementation of auth.AccessController // that simply checks for a non-empty Authorization header. It is useful for // demonstration and testing. type accessController struct { realm string service string } var _ auth.AccessController = &accessController{} func newAccessController(options map[string]interface{}) (auth.AccessController, error) { realm, present := options["realm"] if _, ok := realm.(string); !present || !ok { return nil, fmt.Errorf(`"realm" must be set for silly access controller`) } service, present := options["service"] if _, ok := service.(string); !present || !ok { return nil, fmt.Errorf(`"service" must be set for silly access controller`) } return &accessController{realm: realm.(string), service: service.(string)}, nil } // Authorized simply checks for the existence of the authorization header, // responding with a bearer challenge if it doesn't exist. func (ac *accessController) Authorized(ctx context.Context, accessRecords ...auth.Access) (context.Context, error) { req, err := ctxu.GetRequest(ctx) if err != nil { return nil, err } if req.Header.Get("Authorization") == "" { challenge := challenge{ realm: ac.realm, service: ac.service, } if len(accessRecords) > 0 { var scopes []string for _, access := range accessRecords { scopes = append(scopes, fmt.Sprintf("%s:%s:%s", access.Type, access.Resource.Name, access.Action)) } challenge.scope = strings.Join(scopes, " ") } return nil, &challenge } return context.WithValue(ctx, "auth.user", auth.UserInfo{Name: "silly"}), nil } type challenge struct { realm string service string scope string } func (ch *challenge) ServeHTTP(w http.ResponseWriter, r *http.Request) { header := fmt.Sprintf("Bearer realm=%q,service=%q", ch.realm, ch.service) if ch.scope != "" { header = fmt.Sprintf("%s,scope=%q", header, ch.scope) } w.Header().Set("WWW-Authenticate", header) w.WriteHeader(http.StatusUnauthorized) } func (ch *challenge) Error() string { return fmt.Sprintf("silly authentication challenge: %#v", ch) } // init registers the silly auth backend. func init() { auth.Register("silly", auth.InitFunc(newAccessController)) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/auth/silly/access_test.go0000644000175000017500000000334512502424227027360 0ustar tianontianonpackage silly import ( "net/http" "net/http/httptest" "testing" "github.com/docker/distribution/registry/auth" "golang.org/x/net/context" ) func TestSillyAccessController(t *testing.T) { ac := &accessController{ realm: "test-realm", service: "test-service", } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ctx := context.WithValue(nil, "http.request", r) authCtx, err := ac.Authorized(ctx) if err != nil { switch err := err.(type) { case auth.Challenge: err.ServeHTTP(w, r) return default: t.Fatalf("unexpected error authorizing request: %v", err) } } userInfo, ok := authCtx.Value("auth.user").(auth.UserInfo) if !ok { t.Fatal("silly accessController did not set auth.user context") } if userInfo.Name != "silly" { t.Fatalf("expected user name %q, got %q", "silly", userInfo.Name) } w.WriteHeader(http.StatusNoContent) })) resp, err := http.Get(server.URL) if err != nil { t.Fatalf("unexpected error during GET: %v", err) } defer resp.Body.Close() // Request should not be authorized if resp.StatusCode != http.StatusUnauthorized { t.Fatalf("unexpected response status: %v != %v", resp.StatusCode, http.StatusUnauthorized) } req, err := http.NewRequest("GET", server.URL, nil) if err != nil { t.Fatalf("unexpected error creating new request: %v", err) } req.Header.Set("Authorization", "seriously, anything") resp, err = http.DefaultClient.Do(req) if err != nil { t.Fatalf("unexpected error during GET: %v", err) } defer resp.Body.Close() // Request should not be authorized if resp.StatusCode != http.StatusNoContent { t.Fatalf("unexpected response status: %v != %v", resp.StatusCode, http.StatusNoContent) } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/auth/auth.go0000644000175000017500000001124012502424227024656 0ustar tianontianon// Package auth defines a standard interface for request access controllers. // // An access controller has a simple interface with a single `Authorized` // method which checks that a given request is authorized to perform one or // more actions on one or more resources. This method should return a non-nil // error if the requset is not authorized. // // An implementation registers its access controller by name with a constructor // which accepts an options map for configuring the access controller. // // options := map[string]interface{}{"sillySecret": "whysosilly?"} // accessController, _ := auth.GetAccessController("silly", options) // // This `accessController` can then be used in a request handler like so: // // func updateOrder(w http.ResponseWriter, r *http.Request) { // orderNumber := r.FormValue("orderNumber") // resource := auth.Resource{Type: "customerOrder", Name: orderNumber} // access := auth.Access{Resource: resource, Action: "update"} // // if ctx, err := accessController.Authorized(ctx, access); err != nil { // if challenge, ok := err.(auth.Challenge) { // // Let the challenge write the response. // challenge.ServeHTTP(w, r) // } else { // // Some other error. // } // } // } // package auth import ( "fmt" "net/http" "golang.org/x/net/context" ) // UserInfo carries information about // an autenticated/authorized client. type UserInfo struct { Name string } // Resource describes a resource by type and name. type Resource struct { Type string Name string } // Access describes a specific action that is // requested or allowed for a given recource. type Access struct { Resource Action string } // Challenge is a special error type which is used for HTTP 401 Unauthorized // responses and is able to write the response with WWW-Authenticate challenge // header values based on the error. type Challenge interface { error // ServeHTTP prepares the request to conduct the appropriate challenge // response. For most implementations, simply calling ServeHTTP should be // sufficient. Because no body is written, users may write a custom body after // calling ServeHTTP, but any headers must be written before the call and may // be overwritten. ServeHTTP(w http.ResponseWriter, r *http.Request) } // AccessController controls access to registry resources based on a request // and required access levels for a request. Implementations can support both // complete denial and http authorization challenges. type AccessController interface { // Authorized returns a non-nil error if the context is granted access and // returns a new authorized context. If one or more Access structs are // provided, the requested access will be compared with what is available // to the context. The given context will contain a "http.request" key with // a `*http.Request` value. If the error is non-nil, access should always // be denied. The error may be of type Challenge, in which case the caller // may have the Challenge handle the request or choose what action to take // based on the Challenge header or response status. The returned context // object should have a "auth.user" value set to a UserInfo struct. Authorized(ctx context.Context, access ...Access) (context.Context, error) } // WithUser returns a context with the authorized user info. func WithUser(ctx context.Context, user UserInfo) context.Context { return userInfoContext{ Context: ctx, user: user, } } type userInfoContext struct { context.Context user UserInfo } func (uic userInfoContext) Value(key interface{}) interface{} { switch key { case "auth.user": return uic.user case "auth.user.name": return uic.user.Name } return uic.Context.Value(key) } // InitFunc is the type of an AccessController factory function and is used // to register the constructor for different AccesController backends. type InitFunc func(options map[string]interface{}) (AccessController, error) var accessControllers map[string]InitFunc func init() { accessControllers = make(map[string]InitFunc) } // Register is used to register an InitFunc for // an AccessController backend with the given name. func Register(name string, initFunc InitFunc) error { if _, exists := accessControllers[name]; exists { return fmt.Errorf("name already registered: %s", name) } accessControllers[name] = initFunc return nil } // GetAccessController constructs an AccessController // with the given options using the named backend. func GetAccessController(name string, options map[string]interface{}) (AccessController, error) { if initFunc, exists := accessControllers[name]; exists { return initFunc(options) } return nil, fmt.Errorf("no access controller registered with name: %s", name) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/auth/token/0000755000175000017500000000000012502424227024510 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/auth/token/util.go0000644000175000017500000000275012502424227026020 0ustar tianontianonpackage token import ( "encoding/base64" "errors" "strings" ) // joseBase64UrlEncode encodes the given data using the standard base64 url // encoding format but with all trailing '=' characters ommitted in accordance // with the jose specification. // http://tools.ietf.org/html/draft-ietf-jose-json-web-signature-31#section-2 func joseBase64UrlEncode(b []byte) string { return strings.TrimRight(base64.URLEncoding.EncodeToString(b), "=") } // joseBase64UrlDecode decodes the given string using the standard base64 url // decoder but first adds the appropriate number of trailing '=' characters in // accordance with the jose specification. // http://tools.ietf.org/html/draft-ietf-jose-json-web-signature-31#section-2 func joseBase64UrlDecode(s string) ([]byte, error) { switch len(s) % 4 { case 0: case 2: s += "==" case 3: s += "=" default: return nil, errors.New("illegal base64url string") } return base64.URLEncoding.DecodeString(s) } // actionSet is a special type of stringSet. type actionSet struct { stringSet } func newActionSet(actions ...string) actionSet { return actionSet{newStringSet(actions...)} } // Contains calls StringSet.Contains() for // either "*" or the given action string. func (s actionSet) contains(action string) bool { return s.stringSet.contains("*") || s.stringSet.contains(action) } // contains returns true if q is found in ss. func contains(ss []string, q string) bool { for _, s := range ss { if s == q { return true } } return false } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/auth/token/token_test.go0000644000175000017500000002257212502424227027226 0ustar tianontianonpackage token import ( "crypto" "crypto/rand" "crypto/x509" "encoding/base64" "encoding/json" "encoding/pem" "fmt" "io/ioutil" "net/http" "os" "strings" "testing" "time" "github.com/docker/distribution/registry/auth" "github.com/docker/libtrust" "golang.org/x/net/context" ) func makeRootKeys(numKeys int) ([]libtrust.PrivateKey, error) { keys := make([]libtrust.PrivateKey, 0, numKeys) for i := 0; i < numKeys; i++ { key, err := libtrust.GenerateECP256PrivateKey() if err != nil { return nil, err } keys = append(keys, key) } return keys, nil } func makeSigningKeyWithChain(rootKey libtrust.PrivateKey, depth int) (libtrust.PrivateKey, error) { if depth == 0 { // Don't need to build a chain. return rootKey, nil } var ( x5c = make([]string, depth) parentKey = rootKey key libtrust.PrivateKey cert *x509.Certificate err error ) for depth > 0 { if key, err = libtrust.GenerateECP256PrivateKey(); err != nil { return nil, err } if cert, err = libtrust.GenerateCACert(parentKey, key); err != nil { return nil, err } depth-- x5c[depth] = base64.StdEncoding.EncodeToString(cert.Raw) parentKey = key } key.AddExtendedField("x5c", x5c) return key, nil } func makeRootCerts(rootKeys []libtrust.PrivateKey) ([]*x509.Certificate, error) { certs := make([]*x509.Certificate, 0, len(rootKeys)) for _, key := range rootKeys { cert, err := libtrust.GenerateCACert(key, key) if err != nil { return nil, err } certs = append(certs, cert) } return certs, nil } func makeTrustedKeyMap(rootKeys []libtrust.PrivateKey) map[string]libtrust.PublicKey { trustedKeys := make(map[string]libtrust.PublicKey, len(rootKeys)) for _, key := range rootKeys { trustedKeys[key.KeyID()] = key.PublicKey() } return trustedKeys } func makeTestToken(issuer, audience string, access []*ResourceActions, rootKey libtrust.PrivateKey, depth int) (*Token, error) { signingKey, err := makeSigningKeyWithChain(rootKey, depth) if err != nil { return nil, fmt.Errorf("unable to amke signing key with chain: %s", err) } rawJWK, err := signingKey.PublicKey().MarshalJSON() if err != nil { return nil, fmt.Errorf("unable to marshal signing key to JSON: %s", err) } joseHeader := &Header{ Type: "JWT", SigningAlg: "ES256", RawJWK: json.RawMessage(rawJWK), } now := time.Now() randomBytes := make([]byte, 15) if _, err = rand.Read(randomBytes); err != nil { return nil, fmt.Errorf("unable to read random bytes for jwt id: %s", err) } claimSet := &ClaimSet{ Issuer: issuer, Subject: "foo", Audience: audience, Expiration: now.Add(5 * time.Minute).Unix(), NotBefore: now.Unix(), IssuedAt: now.Unix(), JWTID: base64.URLEncoding.EncodeToString(randomBytes), Access: access, } var joseHeaderBytes, claimSetBytes []byte if joseHeaderBytes, err = json.Marshal(joseHeader); err != nil { return nil, fmt.Errorf("unable to marshal jose header: %s", err) } if claimSetBytes, err = json.Marshal(claimSet); err != nil { return nil, fmt.Errorf("unable to marshal claim set: %s", err) } encodedJoseHeader := joseBase64UrlEncode(joseHeaderBytes) encodedClaimSet := joseBase64UrlEncode(claimSetBytes) encodingToSign := fmt.Sprintf("%s.%s", encodedJoseHeader, encodedClaimSet) var signatureBytes []byte if signatureBytes, _, err = signingKey.Sign(strings.NewReader(encodingToSign), crypto.SHA256); err != nil { return nil, fmt.Errorf("unable to sign jwt payload: %s", err) } signature := joseBase64UrlEncode(signatureBytes) tokenString := fmt.Sprintf("%s.%s", encodingToSign, signature) return NewToken(tokenString) } // This test makes 4 tokens with a varying number of intermediate // certificates ranging from no intermediate chain to a length of 3 // intermediates. func TestTokenVerify(t *testing.T) { var ( numTokens = 4 issuer = "test-issuer" audience = "test-audience" access = []*ResourceActions{ { Type: "repository", Name: "foo/bar", Actions: []string{"pull", "push"}, }, } ) rootKeys, err := makeRootKeys(numTokens) if err != nil { t.Fatal(err) } rootCerts, err := makeRootCerts(rootKeys) if err != nil { t.Fatal(err) } rootPool := x509.NewCertPool() for _, rootCert := range rootCerts { rootPool.AddCert(rootCert) } trustedKeys := makeTrustedKeyMap(rootKeys) tokens := make([]*Token, 0, numTokens) for i := 0; i < numTokens; i++ { token, err := makeTestToken(issuer, audience, access, rootKeys[i], i) if err != nil { t.Fatal(err) } tokens = append(tokens, token) } verifyOps := VerifyOptions{ TrustedIssuers: []string{issuer}, AcceptedAudiences: []string{audience}, Roots: rootPool, TrustedKeys: trustedKeys, } for _, token := range tokens { if err := token.Verify(verifyOps); err != nil { t.Fatal(err) } } } func writeTempRootCerts(rootKeys []libtrust.PrivateKey) (filename string, err error) { rootCerts, err := makeRootCerts(rootKeys) if err != nil { return "", err } tempFile, err := ioutil.TempFile("", "rootCertBundle") if err != nil { return "", err } defer tempFile.Close() for _, cert := range rootCerts { if err = pem.Encode(tempFile, &pem.Block{ Type: "CERTIFICATE", Bytes: cert.Raw, }); err != nil { os.Remove(tempFile.Name()) return "", err } } return tempFile.Name(), nil } // TestAccessController tests complete integration of the token auth package. // It starts by mocking the options for a token auth accessController which // it creates. It then tries a few mock requests: // - don't supply a token; should error with challenge // - supply an invalid token; should error with challenge // - supply a token with insufficient access; should error with challenge // - supply a valid token; should not error func TestAccessController(t *testing.T) { // Make 2 keys; only the first is to be a trusted root key. rootKeys, err := makeRootKeys(2) if err != nil { t.Fatal(err) } rootCertBundleFilename, err := writeTempRootCerts(rootKeys[:1]) if err != nil { t.Fatal(err) } defer os.Remove(rootCertBundleFilename) realm := "https://auth.example.com/token/" issuer := "test-issuer.example.com" service := "test-service.example.com" options := map[string]interface{}{ "realm": realm, "issuer": issuer, "service": service, "rootcertbundle": rootCertBundleFilename, } accessController, err := newAccessController(options) if err != nil { t.Fatal(err) } // 1. Make a mock http.Request with no token. req, err := http.NewRequest("GET", "http://example.com/foo", nil) if err != nil { t.Fatal(err) } testAccess := auth.Access{ Resource: auth.Resource{ Type: "foo", Name: "bar", }, Action: "baz", } ctx := context.WithValue(nil, "http.request", req) authCtx, err := accessController.Authorized(ctx, testAccess) challenge, ok := err.(auth.Challenge) if !ok { t.Fatal("accessController did not return a challenge") } if challenge.Error() != ErrTokenRequired.Error() { t.Fatalf("accessControler did not get expected error - got %s - expected %s", challenge, ErrTokenRequired) } if authCtx != nil { t.Fatalf("expected nil auth context but got %s", authCtx) } // 2. Supply an invalid token. token, err := makeTestToken( issuer, service, []*ResourceActions{{ Type: testAccess.Type, Name: testAccess.Name, Actions: []string{testAccess.Action}, }}, rootKeys[1], 1, // Everything is valid except the key which signed it. ) if err != nil { t.Fatal(err) } req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token.compactRaw())) authCtx, err = accessController.Authorized(ctx, testAccess) challenge, ok = err.(auth.Challenge) if !ok { t.Fatal("accessController did not return a challenge") } if challenge.Error() != ErrInvalidToken.Error() { t.Fatalf("accessControler did not get expected error - got %s - expected %s", challenge, ErrTokenRequired) } if authCtx != nil { t.Fatalf("expected nil auth context but got %s", authCtx) } // 3. Supply a token with insufficient access. token, err = makeTestToken( issuer, service, []*ResourceActions{}, // No access specified. rootKeys[0], 1, ) if err != nil { t.Fatal(err) } req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token.compactRaw())) authCtx, err = accessController.Authorized(ctx, testAccess) challenge, ok = err.(auth.Challenge) if !ok { t.Fatal("accessController did not return a challenge") } if challenge.Error() != ErrInsufficientScope.Error() { t.Fatalf("accessControler did not get expected error - got %s - expected %s", challenge, ErrInsufficientScope) } if authCtx != nil { t.Fatalf("expected nil auth context but got %s", authCtx) } // 4. Supply the token we need, or deserve, or whatever. token, err = makeTestToken( issuer, service, []*ResourceActions{{ Type: testAccess.Type, Name: testAccess.Name, Actions: []string{testAccess.Action}, }}, rootKeys[0], 1, ) if err != nil { t.Fatal(err) } req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token.compactRaw())) authCtx, err = accessController.Authorized(ctx, testAccess) if err != nil { t.Fatalf("accessController returned unexpected error: %s", err) } userInfo, ok := authCtx.Value("auth.user").(auth.UserInfo) if !ok { t.Fatal("token accessController did not set auth.user context") } if userInfo.Name != "foo" { t.Fatalf("expected user name %q, got %q", "foo", userInfo.Name) } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/auth/token/token.go0000644000175000017500000002364612502424227026172 0ustar tianontianonpackage token import ( "crypto" "crypto/x509" "encoding/base64" "encoding/json" "errors" "fmt" "strings" "time" log "github.com/Sirupsen/logrus" "github.com/docker/libtrust" "github.com/docker/distribution/registry/auth" ) const ( // TokenSeparator is the value which separates the header, claims, and // signature in the compact serialization of a JSON Web Token. TokenSeparator = "." ) // Errors used by token parsing and verification. var ( ErrMalformedToken = errors.New("malformed token") ErrInvalidToken = errors.New("invalid token") ) // ResourceActions stores allowed actions on a named and typed resource. type ResourceActions struct { Type string `json:"type"` Name string `json:"name"` Actions []string `json:"actions"` } // ClaimSet describes the main section of a JSON Web Token. type ClaimSet struct { // Public claims Issuer string `json:"iss"` Subject string `json:"sub"` Audience string `json:"aud"` Expiration int64 `json:"exp"` NotBefore int64 `json:"nbf"` IssuedAt int64 `json:"iat"` JWTID string `json:"jti"` // Private claims Access []*ResourceActions `json:"access"` } // Header describes the header section of a JSON Web Token. type Header struct { Type string `json:"typ"` SigningAlg string `json:"alg"` KeyID string `json:"kid,omitempty"` X5c []string `json:"x5c,omitempty"` RawJWK json.RawMessage `json:"jwk,omitempty"` } // Token describes a JSON Web Token. type Token struct { Raw string Header *Header Claims *ClaimSet Signature []byte } // VerifyOptions is used to specify // options when verifying a JSON Web Token. type VerifyOptions struct { TrustedIssuers []string AcceptedAudiences []string Roots *x509.CertPool TrustedKeys map[string]libtrust.PublicKey } // NewToken parses the given raw token string // and constructs an unverified JSON Web Token. func NewToken(rawToken string) (*Token, error) { parts := strings.Split(rawToken, TokenSeparator) if len(parts) != 3 { return nil, ErrMalformedToken } var ( rawHeader, rawClaims = parts[0], parts[1] headerJSON, claimsJSON []byte err error ) defer func() { if err != nil { log.Errorf("error while unmarshalling raw token: %s", err) } }() if headerJSON, err = joseBase64UrlDecode(rawHeader); err != nil { err = fmt.Errorf("unable to decode header: %s", err) return nil, ErrMalformedToken } if claimsJSON, err = joseBase64UrlDecode(rawClaims); err != nil { err = fmt.Errorf("unable to decode claims: %s", err) return nil, ErrMalformedToken } token := new(Token) token.Header = new(Header) token.Claims = new(ClaimSet) token.Raw = strings.Join(parts[:2], TokenSeparator) if token.Signature, err = joseBase64UrlDecode(parts[2]); err != nil { err = fmt.Errorf("unable to decode signature: %s", err) return nil, ErrMalformedToken } if err = json.Unmarshal(headerJSON, token.Header); err != nil { return nil, ErrMalformedToken } if err = json.Unmarshal(claimsJSON, token.Claims); err != nil { return nil, ErrMalformedToken } return token, nil } // Verify attempts to verify this token using the given options. // Returns a nil error if the token is valid. func (t *Token) Verify(verifyOpts VerifyOptions) error { // Verify that the Issuer claim is a trusted authority. if !contains(verifyOpts.TrustedIssuers, t.Claims.Issuer) { log.Errorf("token from untrusted issuer: %q", t.Claims.Issuer) return ErrInvalidToken } // Verify that the Audience claim is allowed. if !contains(verifyOpts.AcceptedAudiences, t.Claims.Audience) { log.Errorf("token intended for another audience: %q", t.Claims.Audience) return ErrInvalidToken } // Verify that the token is currently usable and not expired. currentUnixTime := time.Now().Unix() if !(t.Claims.NotBefore <= currentUnixTime && currentUnixTime <= t.Claims.Expiration) { log.Errorf("token not to be used before %d or after %d - currently %d", t.Claims.NotBefore, t.Claims.Expiration, currentUnixTime) return ErrInvalidToken } // Verify the token signature. if len(t.Signature) == 0 { log.Error("token has no signature") return ErrInvalidToken } // Verify that the signing key is trusted. signingKey, err := t.VerifySigningKey(verifyOpts) if err != nil { log.Error(err) return ErrInvalidToken } // Finally, verify the signature of the token using the key which signed it. if err := signingKey.Verify(strings.NewReader(t.Raw), t.Header.SigningAlg, t.Signature); err != nil { log.Errorf("unable to verify token signature: %s", err) return ErrInvalidToken } return nil } // VerifySigningKey attempts to get the key which was used to sign this token. // The token header should contain either of these 3 fields: // `x5c` - The x509 certificate chain for the signing key. Needs to be // verified. // `jwk` - The JSON Web Key representation of the signing key. // May contain its own `x5c` field which needs to be verified. // `kid` - The unique identifier for the key. This library interprets it // as a libtrust fingerprint. The key itself can be looked up in // the trustedKeys field of the given verify options. // Each of these methods are tried in that order of preference until the // signing key is found or an error is returned. func (t *Token) VerifySigningKey(verifyOpts VerifyOptions) (signingKey libtrust.PublicKey, err error) { // First attempt to get an x509 certificate chain from the header. var ( x5c = t.Header.X5c rawJWK = t.Header.RawJWK keyID = t.Header.KeyID ) switch { case len(x5c) > 0: signingKey, err = parseAndVerifyCertChain(x5c, verifyOpts.Roots) case len(rawJWK) > 0: signingKey, err = parseAndVerifyRawJWK(rawJWK, verifyOpts) case len(keyID) > 0: signingKey = verifyOpts.TrustedKeys[keyID] if signingKey == nil { err = fmt.Errorf("token signed by untrusted key with ID: %q", keyID) } default: err = errors.New("unable to get token signing key") } return } func parseAndVerifyCertChain(x5c []string, roots *x509.CertPool) (leafKey libtrust.PublicKey, err error) { if len(x5c) == 0 { return nil, errors.New("empty x509 certificate chain") } // Ensure the first element is encoded correctly. leafCertDer, err := base64.StdEncoding.DecodeString(x5c[0]) if err != nil { return nil, fmt.Errorf("unable to decode leaf certificate: %s", err) } // And that it is a valid x509 certificate. leafCert, err := x509.ParseCertificate(leafCertDer) if err != nil { return nil, fmt.Errorf("unable to parse leaf certificate: %s", err) } // The rest of the certificate chain are intermediate certificates. intermediates := x509.NewCertPool() for i := 1; i < len(x5c); i++ { intermediateCertDer, err := base64.StdEncoding.DecodeString(x5c[i]) if err != nil { return nil, fmt.Errorf("unable to decode intermediate certificate: %s", err) } intermediateCert, err := x509.ParseCertificate(intermediateCertDer) if err != nil { return nil, fmt.Errorf("unable to parse intermediate certificate: %s", err) } intermediates.AddCert(intermediateCert) } verifyOpts := x509.VerifyOptions{ Intermediates: intermediates, Roots: roots, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, } // TODO: this call returns certificate chains which we ignore for now, but // we should check them for revocations if we have the ability later. if _, err = leafCert.Verify(verifyOpts); err != nil { return nil, fmt.Errorf("unable to verify certificate chain: %s", err) } // Get the public key from the leaf certificate. leafCryptoKey, ok := leafCert.PublicKey.(crypto.PublicKey) if !ok { return nil, errors.New("unable to get leaf cert public key value") } leafKey, err = libtrust.FromCryptoPublicKey(leafCryptoKey) if err != nil { return nil, fmt.Errorf("unable to make libtrust public key from leaf certificate: %s", err) } return } func parseAndVerifyRawJWK(rawJWK json.RawMessage, verifyOpts VerifyOptions) (pubKey libtrust.PublicKey, err error) { pubKey, err = libtrust.UnmarshalPublicKeyJWK([]byte(rawJWK)) if err != nil { return nil, fmt.Errorf("unable to decode raw JWK value: %s", err) } // Check to see if the key includes a certificate chain. x5cVal, ok := pubKey.GetExtendedField("x5c").([]interface{}) if !ok { // The JWK should be one of the trusted root keys. if _, trusted := verifyOpts.TrustedKeys[pubKey.KeyID()]; !trusted { return nil, errors.New("untrusted JWK with no certificate chain") } // The JWK is one of the trusted keys. return } // Ensure each item in the chain is of the correct type. x5c := make([]string, len(x5cVal)) for i, val := range x5cVal { certString, ok := val.(string) if !ok || len(certString) == 0 { return nil, errors.New("malformed certificate chain") } x5c[i] = certString } // Ensure that the x509 certificate chain can // be verified up to one of our trusted roots. leafKey, err := parseAndVerifyCertChain(x5c, verifyOpts.Roots) if err != nil { return nil, fmt.Errorf("could not verify JWK certificate chain: %s", err) } // Verify that the public key in the leaf cert *is* the signing key. if pubKey.KeyID() != leafKey.KeyID() { return nil, errors.New("leaf certificate public key ID does not match JWK key ID") } return } // accessSet returns a set of actions available for the resource // actions listed in the `access` section of this token. func (t *Token) accessSet() accessSet { if t.Claims == nil { return nil } accessSet := make(accessSet, len(t.Claims.Access)) for _, resourceActions := range t.Claims.Access { resource := auth.Resource{ Type: resourceActions.Type, Name: resourceActions.Name, } set, exists := accessSet[resource] if !exists { set = newActionSet() accessSet[resource] = set } for _, action := range resourceActions.Actions { set.add(action) } } return accessSet } func (t *Token) compactRaw() string { return fmt.Sprintf("%s.%s", t.Raw, joseBase64UrlEncode(t.Signature)) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/auth/token/accesscontroller.go0000644000175000017500000001650312502424227030411 0ustar tianontianonpackage token import ( "crypto" "crypto/x509" "encoding/pem" "errors" "fmt" "io/ioutil" "net/http" "os" "strings" ctxu "github.com/docker/distribution/context" "github.com/docker/distribution/registry/auth" "github.com/docker/libtrust" "golang.org/x/net/context" ) // accessSet maps a typed, named resource to // a set of actions requested or authorized. type accessSet map[auth.Resource]actionSet // newAccessSet constructs an accessSet from // a variable number of auth.Access items. func newAccessSet(accessItems ...auth.Access) accessSet { accessSet := make(accessSet, len(accessItems)) for _, access := range accessItems { resource := auth.Resource{ Type: access.Type, Name: access.Name, } set, exists := accessSet[resource] if !exists { set = newActionSet() accessSet[resource] = set } set.add(access.Action) } return accessSet } // contains returns whether or not the given access is in this accessSet. func (s accessSet) contains(access auth.Access) bool { actionSet, ok := s[access.Resource] if ok { return actionSet.contains(access.Action) } return false } // scopeParam returns a collection of scopes which can // be used for a WWW-Authenticate challenge parameter. // See https://tools.ietf.org/html/rfc6750#section-3 func (s accessSet) scopeParam() string { scopes := make([]string, 0, len(s)) for resource, actionSet := range s { actions := strings.Join(actionSet.keys(), ",") scopes = append(scopes, fmt.Sprintf("%s:%s:%s", resource.Type, resource.Name, actions)) } return strings.Join(scopes, " ") } // Errors used and exported by this package. var ( ErrInsufficientScope = errors.New("insufficient scope") ErrTokenRequired = errors.New("authorization token required") ) // authChallenge implements the auth.Challenge interface. type authChallenge struct { err error realm string service string accessSet accessSet } // Error returns the internal error string for this authChallenge. func (ac *authChallenge) Error() string { return ac.err.Error() } // Status returns the HTTP Response Status Code for this authChallenge. func (ac *authChallenge) Status() int { return http.StatusUnauthorized } // challengeParams constructs the value to be used in // the WWW-Authenticate response challenge header. // See https://tools.ietf.org/html/rfc6750#section-3 func (ac *authChallenge) challengeParams() string { str := fmt.Sprintf("Bearer realm=%q,service=%q", ac.realm, ac.service) if scope := ac.accessSet.scopeParam(); scope != "" { str = fmt.Sprintf("%s,scope=%q", str, scope) } if ac.err == ErrInvalidToken || ac.err == ErrMalformedToken { str = fmt.Sprintf("%s,error=%q", str, "invalid_token") } else if ac.err == ErrInsufficientScope { str = fmt.Sprintf("%s,error=%q", str, "insufficient_scope") } return str } // SetHeader sets the WWW-Authenticate value for the given header. func (ac *authChallenge) SetHeader(header http.Header) { header.Add("WWW-Authenticate", ac.challengeParams()) } // ServeHttp handles writing the challenge response // by setting the challenge header and status code. func (ac *authChallenge) ServeHTTP(w http.ResponseWriter, r *http.Request) { ac.SetHeader(w.Header()) w.WriteHeader(ac.Status()) } // accessController implements the auth.AccessController interface. type accessController struct { realm string issuer string service string rootCerts *x509.CertPool trustedKeys map[string]libtrust.PublicKey } // tokenAccessOptions is a convenience type for handling // options to the contstructor of an accessController. type tokenAccessOptions struct { realm string issuer string service string rootCertBundle string } // checkOptions gathers the necessary options // for an accessController from the given map. func checkOptions(options map[string]interface{}) (tokenAccessOptions, error) { var opts tokenAccessOptions keys := []string{"realm", "issuer", "service", "rootcertbundle"} vals := make([]string, 0, len(keys)) for _, key := range keys { val, ok := options[key].(string) if !ok { return opts, fmt.Errorf("token auth requires a valid option string: %q", key) } vals = append(vals, val) } opts.realm, opts.issuer, opts.service, opts.rootCertBundle = vals[0], vals[1], vals[2], vals[3] return opts, nil } // newAccessController creates an accessController using the given options. func newAccessController(options map[string]interface{}) (auth.AccessController, error) { config, err := checkOptions(options) if err != nil { return nil, err } fp, err := os.Open(config.rootCertBundle) if err != nil { return nil, fmt.Errorf("unable to open token auth root certificate bundle file %q: %s", config.rootCertBundle, err) } defer fp.Close() rawCertBundle, err := ioutil.ReadAll(fp) if err != nil { return nil, fmt.Errorf("unable to read token auth root certificate bundle file %q: %s", config.rootCertBundle, err) } var rootCerts []*x509.Certificate pemBlock, rawCertBundle := pem.Decode(rawCertBundle) for pemBlock != nil { cert, err := x509.ParseCertificate(pemBlock.Bytes) if err != nil { return nil, fmt.Errorf("unable to parse token auth root certificate: %s", err) } rootCerts = append(rootCerts, cert) pemBlock, rawCertBundle = pem.Decode(rawCertBundle) } if len(rootCerts) == 0 { return nil, errors.New("token auth requires at least one token signing root certificate") } rootPool := x509.NewCertPool() trustedKeys := make(map[string]libtrust.PublicKey, len(rootCerts)) for _, rootCert := range rootCerts { rootPool.AddCert(rootCert) pubKey, err := libtrust.FromCryptoPublicKey(crypto.PublicKey(rootCert.PublicKey)) if err != nil { return nil, fmt.Errorf("unable to get public key from token auth root certificate: %s", err) } trustedKeys[pubKey.KeyID()] = pubKey } return &accessController{ realm: config.realm, issuer: config.issuer, service: config.service, rootCerts: rootPool, trustedKeys: trustedKeys, }, nil } // Authorized handles checking whether the given request is authorized // for actions on resources described by the given access items. func (ac *accessController) Authorized(ctx context.Context, accessItems ...auth.Access) (context.Context, error) { challenge := &authChallenge{ realm: ac.realm, service: ac.service, accessSet: newAccessSet(accessItems...), } req, err := ctxu.GetRequest(ctx) if err != nil { return nil, err } parts := strings.Split(req.Header.Get("Authorization"), " ") if len(parts) != 2 || strings.ToLower(parts[0]) != "bearer" { challenge.err = ErrTokenRequired return nil, challenge } rawToken := parts[1] token, err := NewToken(rawToken) if err != nil { challenge.err = err return nil, challenge } verifyOpts := VerifyOptions{ TrustedIssuers: []string{ac.issuer}, AcceptedAudiences: []string{ac.service}, Roots: ac.rootCerts, TrustedKeys: ac.trustedKeys, } if err = token.Verify(verifyOpts); err != nil { challenge.err = err return nil, challenge } accessSet := token.accessSet() for _, access := range accessItems { if !accessSet.contains(access) { challenge.err = ErrInsufficientScope return nil, challenge } } return auth.WithUser(ctx, auth.UserInfo{Name: token.Claims.Subject}), nil } // init handles registering the token auth backend. func init() { auth.Register("token", auth.InitFunc(newAccessController)) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/auth/token/stringset.go0000644000175000017500000000140512502424227027061 0ustar tianontianonpackage token // StringSet is a useful type for looking up strings. type stringSet map[string]struct{} // NewStringSet creates a new StringSet with the given strings. func newStringSet(keys ...string) stringSet { ss := make(stringSet, len(keys)) ss.add(keys...) return ss } // Add inserts the given keys into this StringSet. func (ss stringSet) add(keys ...string) { for _, key := range keys { ss[key] = struct{}{} } } // Contains returns whether the given key is in this StringSet. func (ss stringSet) contains(key string) bool { _, ok := ss[key] return ok } // Keys returns a slice of all keys in this StringSet. func (ss stringSet) keys() []string { keys := make([]string, 0, len(ss)) for key := range ss { keys = append(keys, key) } return keys } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/doc.go0000644000175000017500000000016412502424227023524 0ustar tianontianon// Package registry is a placeholder package for registry interface // destinations and utilities. package registry distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/client/0000755000175000017500000000000012502424227023705 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/client/client_test.go0000644000175000017500000002501312502424227026552 0ustar tianontianonpackage client import ( "encoding/json" "fmt" "io/ioutil" "net/http" "net/http/httptest" "sync" "testing" "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest" "github.com/docker/distribution/testutil" ) type testBlob struct { digest digest.Digest contents []byte } func TestRangeHeaderParser(t *testing.T) { const ( malformedRangeHeader = "bytes=0-A/C" emptyRangeHeader = "" rFirst = 100 rSecond = 200 ) var ( wellformedRangeHeader = fmt.Sprintf("bytes=0-%d/%d", rFirst, rSecond) ) if _, _, err := parseRangeHeader(malformedRangeHeader); err == nil { t.Fatalf("malformedRangeHeader: error expected, got nil") } if _, _, err := parseRangeHeader(emptyRangeHeader); err == nil { t.Fatalf("emptyRangeHeader: error expected, got nil") } first, second, err := parseRangeHeader(wellformedRangeHeader) if err != nil { t.Fatalf("wellformedRangeHeader: unexpected error %v", err) } if first != rFirst || second != rSecond { t.Fatalf("Range has been parsed unproperly: %d/%d", first, second) } } func TestPush(t *testing.T) { name := "hello/world" tag := "sometag" testBlobs := []testBlob{ { digest: "tarsum.v2+sha256:12345", contents: []byte("some contents"), }, { digest: "tarsum.v2+sha256:98765", contents: []byte("some other contents"), }, } uploadLocations := make([]string, len(testBlobs)) blobs := make([]manifest.FSLayer, len(testBlobs)) history := make([]manifest.History, len(testBlobs)) for i, blob := range testBlobs { // TODO(bbland): this is returning the same location for all uploads, // because we can't know which blob will get which location. // It's sort of okay because we're using unique digests, but this needs // to change at some point. uploadLocations[i] = fmt.Sprintf("/v2/%s/blobs/test-uuid", name) blobs[i] = manifest.FSLayer{BlobSum: blob.digest} history[i] = manifest.History{V1Compatibility: blob.digest.String()} } m := &manifest.SignedManifest{ Manifest: manifest.Manifest{ Name: name, Tag: tag, Architecture: "x86", FSLayers: blobs, History: history, Versioned: manifest.Versioned{ SchemaVersion: 1, }, }, } var err error m.Raw, err = json.Marshal(m) blobRequestResponseMappings := make([]testutil.RequestResponseMapping, 2*len(testBlobs)) for i, blob := range testBlobs { blobRequestResponseMappings[2*i] = testutil.RequestResponseMapping{ Request: testutil.Request{ Method: "POST", Route: "/v2/" + name + "/blobs/uploads/", }, Response: testutil.Response{ StatusCode: http.StatusAccepted, Headers: http.Header(map[string][]string{ "Location": {uploadLocations[i]}, }), }, } blobRequestResponseMappings[2*i+1] = testutil.RequestResponseMapping{ Request: testutil.Request{ Method: "PUT", Route: uploadLocations[i], QueryParams: map[string][]string{ "digest": {blob.digest.String()}, }, Body: blob.contents, }, Response: testutil.Response{ StatusCode: http.StatusCreated, }, } } handler := testutil.NewHandler(append(blobRequestResponseMappings, testutil.RequestResponseMapping{ Request: testutil.Request{ Method: "PUT", Route: "/v2/" + name + "/manifests/" + tag, Body: m.Raw, }, Response: testutil.Response{ StatusCode: http.StatusOK, }, })) var server *httptest.Server // HACK(stevvooe): Super hack to follow: the request response map approach // above does not let us correctly format the location header to the // server url. This handler intercepts and re-writes the location header // to the server url. hack := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w = &headerInterceptingResponseWriter{ResponseWriter: w, serverURL: server.URL} handler.ServeHTTP(w, r) }) server = httptest.NewServer(hack) client, err := New(server.URL) if err != nil { t.Fatalf("error creating client: %v", err) } objectStore := &memoryObjectStore{ mutex: new(sync.Mutex), manifestStorage: make(map[string]*manifest.SignedManifest), layerStorage: make(map[digest.Digest]Layer), } for _, blob := range testBlobs { l, err := objectStore.Layer(blob.digest) if err != nil { t.Fatal(err) } writer, err := l.Writer() if err != nil { t.Fatal(err) } writer.SetSize(len(blob.contents)) writer.Write(blob.contents) writer.Close() } objectStore.WriteManifest(name, tag, m) err = Push(client, objectStore, name, tag) if err != nil { t.Fatal(err) } } func TestPull(t *testing.T) { name := "hello/world" tag := "sometag" testBlobs := []testBlob{ { digest: "tarsum.v2+sha256:12345", contents: []byte("some contents"), }, { digest: "tarsum.v2+sha256:98765", contents: []byte("some other contents"), }, } blobs := make([]manifest.FSLayer, len(testBlobs)) history := make([]manifest.History, len(testBlobs)) for i, blob := range testBlobs { blobs[i] = manifest.FSLayer{BlobSum: blob.digest} history[i] = manifest.History{V1Compatibility: blob.digest.String()} } m := &manifest.SignedManifest{ Manifest: manifest.Manifest{ Name: name, Tag: tag, Architecture: "x86", FSLayers: blobs, History: history, Versioned: manifest.Versioned{ SchemaVersion: 1, }, }, } manifestBytes, err := json.Marshal(m) blobRequestResponseMappings := make([]testutil.RequestResponseMapping, len(testBlobs)) for i, blob := range testBlobs { blobRequestResponseMappings[i] = testutil.RequestResponseMapping{ Request: testutil.Request{ Method: "GET", Route: "/v2/" + name + "/blobs/" + blob.digest.String(), }, Response: testutil.Response{ StatusCode: http.StatusOK, Body: blob.contents, }, } } handler := testutil.NewHandler(append(blobRequestResponseMappings, testutil.RequestResponseMapping{ Request: testutil.Request{ Method: "GET", Route: "/v2/" + name + "/manifests/" + tag, }, Response: testutil.Response{ StatusCode: http.StatusOK, Body: manifestBytes, }, })) server := httptest.NewServer(handler) client, err := New(server.URL) if err != nil { t.Fatalf("error creating client: %v", err) } objectStore := &memoryObjectStore{ mutex: new(sync.Mutex), manifestStorage: make(map[string]*manifest.SignedManifest), layerStorage: make(map[digest.Digest]Layer), } err = Pull(client, objectStore, name, tag) if err != nil { t.Fatal(err) } m, err = objectStore.Manifest(name, tag) if err != nil { t.Fatal(err) } mBytes, err := json.Marshal(m) if err != nil { t.Fatal(err) } if string(mBytes) != string(manifestBytes) { t.Fatal("Incorrect manifest") } for _, blob := range testBlobs { l, err := objectStore.Layer(blob.digest) if err != nil { t.Fatal(err) } reader, err := l.Reader() if err != nil { t.Fatal(err) } defer reader.Close() blobBytes, err := ioutil.ReadAll(reader) if err != nil { t.Fatal(err) } if string(blobBytes) != string(blob.contents) { t.Fatal("Incorrect blob") } } } func TestPullResume(t *testing.T) { name := "hello/world" tag := "sometag" testBlobs := []testBlob{ { digest: "tarsum.v2+sha256:12345", contents: []byte("some contents"), }, { digest: "tarsum.v2+sha256:98765", contents: []byte("some other contents"), }, } layers := make([]manifest.FSLayer, len(testBlobs)) history := make([]manifest.History, len(testBlobs)) for i, layer := range testBlobs { layers[i] = manifest.FSLayer{BlobSum: layer.digest} history[i] = manifest.History{V1Compatibility: layer.digest.String()} } m := &manifest.Manifest{ Name: name, Tag: tag, Architecture: "x86", FSLayers: layers, History: history, Versioned: manifest.Versioned{ SchemaVersion: 1, }, } manifestBytes, err := json.Marshal(m) layerRequestResponseMappings := make([]testutil.RequestResponseMapping, 2*len(testBlobs)) for i, blob := range testBlobs { layerRequestResponseMappings[2*i] = testutil.RequestResponseMapping{ Request: testutil.Request{ Method: "GET", Route: "/v2/" + name + "/blobs/" + blob.digest.String(), }, Response: testutil.Response{ StatusCode: http.StatusOK, Body: blob.contents[:len(blob.contents)/2], Headers: http.Header(map[string][]string{ "Content-Length": {fmt.Sprint(len(blob.contents))}, }), }, } layerRequestResponseMappings[2*i+1] = testutil.RequestResponseMapping{ Request: testutil.Request{ Method: "GET", Route: "/v2/" + name + "/blobs/" + blob.digest.String(), }, Response: testutil.Response{ StatusCode: http.StatusOK, Body: blob.contents[len(blob.contents)/2:], }, } } for i := 0; i < 3; i++ { layerRequestResponseMappings = append(layerRequestResponseMappings, testutil.RequestResponseMapping{ Request: testutil.Request{ Method: "GET", Route: "/v2/" + name + "/manifests/" + tag, }, Response: testutil.Response{ StatusCode: http.StatusOK, Body: manifestBytes, }, }) } handler := testutil.NewHandler(layerRequestResponseMappings) server := httptest.NewServer(handler) client, err := New(server.URL) if err != nil { t.Fatalf("error creating client: %v", err) } objectStore := &memoryObjectStore{ mutex: new(sync.Mutex), manifestStorage: make(map[string]*manifest.SignedManifest), layerStorage: make(map[digest.Digest]Layer), } for attempts := 0; attempts < 3; attempts++ { err = Pull(client, objectStore, name, tag) if err == nil { break } } if err != nil { t.Fatal(err) } sm, err := objectStore.Manifest(name, tag) if err != nil { t.Fatal(err) } mBytes, err := json.Marshal(sm) if err != nil { t.Fatal(err) } if string(mBytes) != string(manifestBytes) { t.Fatal("Incorrect manifest") } for _, blob := range testBlobs { l, err := objectStore.Layer(blob.digest) if err != nil { t.Fatal(err) } reader, err := l.Reader() if err != nil { t.Fatal(err) } defer reader.Close() layerBytes, err := ioutil.ReadAll(reader) if err != nil { t.Fatal(err) } if string(layerBytes) != string(blob.contents) { t.Fatal("Incorrect blob") } } } // headerInterceptingResponseWriter is a hacky workaround to re-write the // location header to have the server url. type headerInterceptingResponseWriter struct { http.ResponseWriter serverURL string } func (hirw *headerInterceptingResponseWriter) WriteHeader(status int) { location := hirw.Header().Get("Location") if location != "" { hirw.Header().Set("Location", hirw.serverURL+location) } hirw.ResponseWriter.WriteHeader(status) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/client/objectstore.go0000644000175000017500000001353712502424227026570 0ustar tianontianonpackage client import ( "bytes" "fmt" "io" "sync" "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest" ) var ( // ErrLayerAlreadyExists is returned when attempting to create a layer with // a tarsum that is already in use. ErrLayerAlreadyExists = fmt.Errorf("Layer already exists") // ErrLayerLocked is returned when attempting to write to a layer which is // currently being written to. ErrLayerLocked = fmt.Errorf("Layer locked") ) // ObjectStore is an interface which is designed to approximate the docker // engine storage. This interface is subject to change to conform to the // future requirements of the engine. type ObjectStore interface { // Manifest retrieves the image manifest stored at the given repository name // and tag Manifest(name, tag string) (*manifest.SignedManifest, error) // WriteManifest stores an image manifest at the given repository name and // tag WriteManifest(name, tag string, manifest *manifest.SignedManifest) error // Layer returns a handle to a layer for reading and writing Layer(dgst digest.Digest) (Layer, error) } // Layer is a generic image layer interface. // A Layer may not be written to if it is already complete. type Layer interface { // Reader returns a LayerReader or an error if the layer has not been // written to or is currently being written to. Reader() (LayerReader, error) // Writer returns a LayerWriter or an error if the layer has been fully // written to or is currently being written to. Writer() (LayerWriter, error) // Wait blocks until the Layer can be read from. Wait() error } // LayerReader is a read-only handle to a Layer, which exposes the CurrentSize // and full Size in addition to implementing the io.ReadCloser interface. type LayerReader interface { io.ReadCloser // CurrentSize returns the number of bytes written to the underlying Layer CurrentSize() int // Size returns the full size of the underlying Layer Size() int } // LayerWriter is a write-only handle to a Layer, which exposes the CurrentSize // and full Size in addition to implementing the io.WriteCloser interface. // SetSize must be called on this LayerWriter before it can be written to. type LayerWriter interface { io.WriteCloser // CurrentSize returns the number of bytes written to the underlying Layer CurrentSize() int // Size returns the full size of the underlying Layer Size() int // SetSize sets the full size of the underlying Layer. // This must be called before any calls to Write SetSize(int) error } // memoryObjectStore is an in-memory implementation of the ObjectStore interface type memoryObjectStore struct { mutex *sync.Mutex manifestStorage map[string]*manifest.SignedManifest layerStorage map[digest.Digest]Layer } func (objStore *memoryObjectStore) Manifest(name, tag string) (*manifest.SignedManifest, error) { objStore.mutex.Lock() defer objStore.mutex.Unlock() manifest, ok := objStore.manifestStorage[name+":"+tag] if !ok { return nil, fmt.Errorf("No manifest found with Name: %q, Tag: %q", name, tag) } return manifest, nil } func (objStore *memoryObjectStore) WriteManifest(name, tag string, manifest *manifest.SignedManifest) error { objStore.mutex.Lock() defer objStore.mutex.Unlock() objStore.manifestStorage[name+":"+tag] = manifest return nil } func (objStore *memoryObjectStore) Layer(dgst digest.Digest) (Layer, error) { objStore.mutex.Lock() defer objStore.mutex.Unlock() layer, ok := objStore.layerStorage[dgst] if !ok { layer = &memoryLayer{cond: sync.NewCond(new(sync.Mutex))} objStore.layerStorage[dgst] = layer } return layer, nil } type memoryLayer struct { cond *sync.Cond contents []byte expectedSize int writing bool } func (ml *memoryLayer) Reader() (LayerReader, error) { ml.cond.L.Lock() defer ml.cond.L.Unlock() if ml.contents == nil { return nil, fmt.Errorf("Layer has not been written to yet") } if ml.writing { return nil, ErrLayerLocked } return &memoryLayerReader{ml: ml, reader: bytes.NewReader(ml.contents)}, nil } func (ml *memoryLayer) Writer() (LayerWriter, error) { ml.cond.L.Lock() defer ml.cond.L.Unlock() if ml.contents != nil { if ml.writing { return nil, ErrLayerLocked } if ml.expectedSize == len(ml.contents) { return nil, ErrLayerAlreadyExists } } else { ml.contents = make([]byte, 0) } ml.writing = true return &memoryLayerWriter{ml: ml, buffer: bytes.NewBuffer(ml.contents)}, nil } func (ml *memoryLayer) Wait() error { ml.cond.L.Lock() defer ml.cond.L.Unlock() if ml.contents == nil { return fmt.Errorf("No writer to wait on") } for ml.writing { ml.cond.Wait() } return nil } type memoryLayerReader struct { ml *memoryLayer reader *bytes.Reader } func (mlr *memoryLayerReader) Read(p []byte) (int, error) { return mlr.reader.Read(p) } func (mlr *memoryLayerReader) Close() error { return nil } func (mlr *memoryLayerReader) CurrentSize() int { return len(mlr.ml.contents) } func (mlr *memoryLayerReader) Size() int { return mlr.ml.expectedSize } type memoryLayerWriter struct { ml *memoryLayer buffer *bytes.Buffer } func (mlw *memoryLayerWriter) Write(p []byte) (int, error) { if mlw.ml.expectedSize == 0 { return 0, fmt.Errorf("Must set size before writing to layer") } wrote, err := mlw.buffer.Write(p) mlw.ml.contents = mlw.buffer.Bytes() return wrote, err } func (mlw *memoryLayerWriter) Close() error { mlw.ml.cond.L.Lock() defer mlw.ml.cond.L.Unlock() return mlw.close() } func (mlw *memoryLayerWriter) close() error { mlw.ml.writing = false mlw.ml.cond.Broadcast() return nil } func (mlw *memoryLayerWriter) CurrentSize() int { return len(mlw.ml.contents) } func (mlw *memoryLayerWriter) Size() int { return mlw.ml.expectedSize } func (mlw *memoryLayerWriter) SetSize(size int) error { if !mlw.ml.writing { return fmt.Errorf("Layer is closed for writing") } mlw.ml.expectedSize = size return nil } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/client/errors.go0000644000175000017500000000435212502424227025554 0ustar tianontianonpackage client import ( "fmt" "github.com/docker/distribution/digest" ) // RepositoryNotFoundError is returned when making an operation against a // repository that does not exist in the registry. type RepositoryNotFoundError struct { Name string } func (e *RepositoryNotFoundError) Error() string { return fmt.Sprintf("No repository found with Name: %s", e.Name) } // ImageManifestNotFoundError is returned when making an operation against a // given image manifest that does not exist in the registry. type ImageManifestNotFoundError struct { Name string Tag string } func (e *ImageManifestNotFoundError) Error() string { return fmt.Sprintf("No manifest found with Name: %s, Tag: %s", e.Name, e.Tag) } // BlobNotFoundError is returned when making an operation against a given image // layer that does not exist in the registry. type BlobNotFoundError struct { Name string Digest digest.Digest } func (e *BlobNotFoundError) Error() string { return fmt.Sprintf("No blob found with Name: %s, Digest: %s", e.Name, e.Digest) } // BlobUploadNotFoundError is returned when making a blob upload operation against an // invalid blob upload location url. // This may be the result of using a cancelled, completed, or stale upload // location. type BlobUploadNotFoundError struct { Location string } func (e *BlobUploadNotFoundError) Error() string { return fmt.Sprintf("No blob upload found at Location: %s", e.Location) } // BlobUploadInvalidRangeError is returned when attempting to upload an image // blob chunk that is out of order. // This provides the known BlobSize and LastValidRange which can be used to // resume the upload. type BlobUploadInvalidRangeError struct { Location string LastValidRange int BlobSize int } func (e *BlobUploadInvalidRangeError) Error() string { return fmt.Sprintf( "Invalid range provided for upload at Location: %s. Last Valid Range: %d, Blob Size: %d", e.Location, e.LastValidRange, e.BlobSize) } // UnexpectedHTTPStatusError is returned when an unexpected HTTP status is // returned when making a registry api call. type UnexpectedHTTPStatusError struct { Status string } func (e *UnexpectedHTTPStatusError) Error() string { return fmt.Sprintf("Received unexpected HTTP status: %s", e.Status) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/client/push.go0000644000175000017500000000667212502424227025226 0ustar tianontianonpackage client import ( "fmt" log "github.com/Sirupsen/logrus" "github.com/docker/distribution/manifest" ) // simultaneousLayerPushWindow is the size of the parallel layer push window. // A layer may not be pushed until the layer preceeding it by the length of the // push window has been successfully pushed. const simultaneousLayerPushWindow = 4 type pushFunction func(fsLayer manifest.FSLayer) error // Push implements a client push workflow for the image defined by the given // name and tag pair, using the given ObjectStore for local manifest and layer // storage func Push(c Client, objectStore ObjectStore, name, tag string) error { manifest, err := objectStore.Manifest(name, tag) if err != nil { log.WithFields(log.Fields{ "error": err, "name": name, "tag": tag, }).Info("No image found") return err } errChans := make([]chan error, len(manifest.FSLayers)) for i := range manifest.FSLayers { errChans[i] = make(chan error) } cancelCh := make(chan struct{}) // Iterate over each layer in the manifest, simultaneously pushing no more // than simultaneousLayerPushWindow layers at a time. If an error is // received from a layer push, we abort the push. for i := 0; i < len(manifest.FSLayers)+simultaneousLayerPushWindow; i++ { dependentLayer := i - simultaneousLayerPushWindow if dependentLayer >= 0 { err := <-errChans[dependentLayer] if err != nil { log.WithField("error", err).Warn("Push aborted") close(cancelCh) return err } } if i < len(manifest.FSLayers) { go func(i int) { select { case errChans[i] <- pushLayer(c, objectStore, name, manifest.FSLayers[i]): case <-cancelCh: // recv broadcast notification about cancelation } }(i) } } err = c.PutImageManifest(name, tag, manifest) if err != nil { log.WithFields(log.Fields{ "error": err, "manifest": manifest, }).Warn("Unable to upload manifest") return err } return nil } func pushLayer(c Client, objectStore ObjectStore, name string, fsLayer manifest.FSLayer) error { log.WithField("layer", fsLayer).Info("Pushing layer") layer, err := objectStore.Layer(fsLayer.BlobSum) if err != nil { log.WithFields(log.Fields{ "error": err, "layer": fsLayer, }).Warn("Unable to read local layer") return err } layerReader, err := layer.Reader() if err != nil { log.WithFields(log.Fields{ "error": err, "layer": fsLayer, }).Warn("Unable to read local layer") return err } defer layerReader.Close() if layerReader.CurrentSize() != layerReader.Size() { log.WithFields(log.Fields{ "layer": fsLayer, "currentSize": layerReader.CurrentSize(), "size": layerReader.Size(), }).Warn("Local layer incomplete") return fmt.Errorf("Local layer incomplete") } length, err := c.BlobLength(name, fsLayer.BlobSum) if err != nil { log.WithFields(log.Fields{ "error": err, "layer": fsLayer, }).Warn("Unable to check existence of remote layer") return err } if length >= 0 { log.WithField("layer", fsLayer).Info("Layer already exists") return nil } location, err := c.InitiateBlobUpload(name) if err != nil { log.WithFields(log.Fields{ "error": err, "layer": fsLayer, }).Warn("Unable to upload layer") return err } err = c.UploadBlob(location, layerReader, int(layerReader.CurrentSize()), fsLayer.BlobSum) if err != nil { log.WithFields(log.Fields{ "error": err, "layer": fsLayer, }).Warn("Unable to upload layer") return err } return nil } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/client/client.go0000644000175000017500000003677012502424227025527 0ustar tianontianonpackage client import ( "bytes" "encoding/json" "fmt" "io" "net/http" "regexp" "strconv" "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest" "github.com/docker/distribution/registry/api/v2" ) // Client implements the client interface to the registry http api type Client interface { // GetImageManifest returns an image manifest for the image at the given // name, tag pair. GetImageManifest(name, tag string) (*manifest.SignedManifest, error) // PutImageManifest uploads an image manifest for the image at the given // name, tag pair. PutImageManifest(name, tag string, imageManifest *manifest.SignedManifest) error // DeleteImage removes the image at the given name, tag pair. DeleteImage(name, tag string) error // ListImageTags returns a list of all image tags with the given repository // name. ListImageTags(name string) ([]string, error) // BlobLength returns the length of the blob stored at the given name, // digest pair. // Returns a length value of -1 on error or if the blob does not exist. BlobLength(name string, dgst digest.Digest) (int, error) // GetBlob returns the blob stored at the given name, digest pair in the // form of an io.ReadCloser with the length of this blob. // A nonzero byteOffset can be provided to receive a partial blob beginning // at the given offset. GetBlob(name string, dgst digest.Digest, byteOffset int) (io.ReadCloser, int, error) // InitiateBlobUpload starts a blob upload in the given repository namespace // and returns a unique location url to use for other blob upload methods. InitiateBlobUpload(name string) (string, error) // GetBlobUploadStatus returns the byte offset and length of the blob at the // given upload location. GetBlobUploadStatus(location string) (int, int, error) // UploadBlob uploads a full blob to the registry. UploadBlob(location string, blob io.ReadCloser, length int, dgst digest.Digest) error // UploadBlobChunk uploads a blob chunk with a given length and startByte to // the registry. // FinishChunkedBlobUpload must be called to finalize this upload. UploadBlobChunk(location string, blobChunk io.ReadCloser, length, startByte int) error // FinishChunkedBlobUpload completes a chunked blob upload at a given // location. FinishChunkedBlobUpload(location string, length int, dgst digest.Digest) error // CancelBlobUpload deletes all content at the unfinished blob upload // location and invalidates any future calls to this blob upload. CancelBlobUpload(location string) error } var ( patternRangeHeader = regexp.MustCompile("bytes=0-(\\d+)/(\\d+)") ) // New returns a new Client which operates against a registry with the // given base endpoint // This endpoint should not include /v2/ or any part of the url after this. func New(endpoint string) (Client, error) { ub, err := v2.NewURLBuilderFromString(endpoint) if err != nil { return nil, err } return &clientImpl{ endpoint: endpoint, ub: ub, }, nil } // clientImpl is the default implementation of the Client interface type clientImpl struct { endpoint string ub *v2.URLBuilder } // TODO(bbland): use consistent route generation between server and client func (r *clientImpl) GetImageManifest(name, tag string) (*manifest.SignedManifest, error) { manifestURL, err := r.ub.BuildManifestURL(name, tag) if err != nil { return nil, err } response, err := http.Get(manifestURL) if err != nil { return nil, err } defer response.Body.Close() // TODO(bbland): handle other status codes, like 5xx errors switch { case response.StatusCode == http.StatusOK: break case response.StatusCode == http.StatusNotFound: return nil, &ImageManifestNotFoundError{Name: name, Tag: tag} case response.StatusCode >= 400 && response.StatusCode < 500: var errs v2.Errors decoder := json.NewDecoder(response.Body) err = decoder.Decode(&errs) if err != nil { return nil, err } return nil, &errs default: return nil, &UnexpectedHTTPStatusError{Status: response.Status} } decoder := json.NewDecoder(response.Body) manifest := new(manifest.SignedManifest) err = decoder.Decode(manifest) if err != nil { return nil, err } return manifest, nil } func (r *clientImpl) PutImageManifest(name, tag string, manifest *manifest.SignedManifest) error { manifestURL, err := r.ub.BuildManifestURL(name, tag) if err != nil { return err } putRequest, err := http.NewRequest("PUT", manifestURL, bytes.NewReader(manifest.Raw)) if err != nil { return err } response, err := http.DefaultClient.Do(putRequest) if err != nil { return err } defer response.Body.Close() // TODO(bbland): handle other status codes, like 5xx errors switch { case response.StatusCode == http.StatusOK || response.StatusCode == http.StatusAccepted: return nil case response.StatusCode >= 400 && response.StatusCode < 500: var errors v2.Errors decoder := json.NewDecoder(response.Body) err = decoder.Decode(&errors) if err != nil { return err } return &errors default: return &UnexpectedHTTPStatusError{Status: response.Status} } } func (r *clientImpl) DeleteImage(name, tag string) error { manifestURL, err := r.ub.BuildManifestURL(name, tag) if err != nil { return err } deleteRequest, err := http.NewRequest("DELETE", manifestURL, nil) if err != nil { return err } response, err := http.DefaultClient.Do(deleteRequest) if err != nil { return err } defer response.Body.Close() // TODO(bbland): handle other status codes, like 5xx errors switch { case response.StatusCode == http.StatusNoContent: break case response.StatusCode == http.StatusNotFound: return &ImageManifestNotFoundError{Name: name, Tag: tag} case response.StatusCode >= 400 && response.StatusCode < 500: var errs v2.Errors decoder := json.NewDecoder(response.Body) err = decoder.Decode(&errs) if err != nil { return err } return &errs default: return &UnexpectedHTTPStatusError{Status: response.Status} } return nil } func (r *clientImpl) ListImageTags(name string) ([]string, error) { tagsURL, err := r.ub.BuildTagsURL(name) if err != nil { return nil, err } response, err := http.Get(tagsURL) if err != nil { return nil, err } defer response.Body.Close() // TODO(bbland): handle other status codes, like 5xx errors switch { case response.StatusCode == http.StatusOK: break case response.StatusCode == http.StatusNotFound: return nil, &RepositoryNotFoundError{Name: name} case response.StatusCode >= 400 && response.StatusCode < 500: var errs v2.Errors decoder := json.NewDecoder(response.Body) err = decoder.Decode(&errs) if err != nil { return nil, err } return nil, &errs default: return nil, &UnexpectedHTTPStatusError{Status: response.Status} } tags := struct { Tags []string `json:"tags"` }{} decoder := json.NewDecoder(response.Body) err = decoder.Decode(&tags) if err != nil { return nil, err } return tags.Tags, nil } func (r *clientImpl) BlobLength(name string, dgst digest.Digest) (int, error) { blobURL, err := r.ub.BuildBlobURL(name, dgst) if err != nil { return -1, err } response, err := http.Head(blobURL) if err != nil { return -1, err } defer response.Body.Close() // TODO(bbland): handle other status codes, like 5xx errors switch { case response.StatusCode == http.StatusOK: lengthHeader := response.Header.Get("Content-Length") length, err := strconv.ParseInt(lengthHeader, 10, 64) if err != nil { return -1, err } return int(length), nil case response.StatusCode == http.StatusNotFound: return -1, nil case response.StatusCode >= 400 && response.StatusCode < 500: var errs v2.Errors decoder := json.NewDecoder(response.Body) err = decoder.Decode(&errs) if err != nil { return -1, err } return -1, &errs default: return -1, &UnexpectedHTTPStatusError{Status: response.Status} } } func (r *clientImpl) GetBlob(name string, dgst digest.Digest, byteOffset int) (io.ReadCloser, int, error) { blobURL, err := r.ub.BuildBlobURL(name, dgst) if err != nil { return nil, 0, err } getRequest, err := http.NewRequest("GET", blobURL, nil) if err != nil { return nil, 0, err } getRequest.Header.Add("Range", fmt.Sprintf("%d-", byteOffset)) response, err := http.DefaultClient.Do(getRequest) if err != nil { return nil, 0, err } // TODO(bbland): handle other status codes, like 5xx errors switch { case response.StatusCode == http.StatusOK: lengthHeader := response.Header.Get("Content-Length") length, err := strconv.ParseInt(lengthHeader, 10, 0) if err != nil { return nil, 0, err } return response.Body, int(length), nil case response.StatusCode == http.StatusNotFound: response.Body.Close() return nil, 0, &BlobNotFoundError{Name: name, Digest: dgst} case response.StatusCode >= 400 && response.StatusCode < 500: var errs v2.Errors decoder := json.NewDecoder(response.Body) err = decoder.Decode(&errs) if err != nil { return nil, 0, err } return nil, 0, &errs default: response.Body.Close() return nil, 0, &UnexpectedHTTPStatusError{Status: response.Status} } } func (r *clientImpl) InitiateBlobUpload(name string) (string, error) { uploadURL, err := r.ub.BuildBlobUploadURL(name) if err != nil { return "", err } postRequest, err := http.NewRequest("POST", uploadURL, nil) if err != nil { return "", err } response, err := http.DefaultClient.Do(postRequest) if err != nil { return "", err } defer response.Body.Close() // TODO(bbland): handle other status codes, like 5xx errors switch { case response.StatusCode == http.StatusAccepted: return response.Header.Get("Location"), nil // case response.StatusCode == http.StatusNotFound: // return case response.StatusCode >= 400 && response.StatusCode < 500: var errs v2.Errors decoder := json.NewDecoder(response.Body) err = decoder.Decode(&errs) if err != nil { return "", err } return "", &errs default: return "", &UnexpectedHTTPStatusError{Status: response.Status} } } func (r *clientImpl) GetBlobUploadStatus(location string) (int, int, error) { response, err := http.Get(location) if err != nil { return 0, 0, err } defer response.Body.Close() // TODO(bbland): handle other status codes, like 5xx errors switch { case response.StatusCode == http.StatusNoContent: return parseRangeHeader(response.Header.Get("Range")) case response.StatusCode == http.StatusNotFound: return 0, 0, &BlobUploadNotFoundError{Location: location} case response.StatusCode >= 400 && response.StatusCode < 500: var errs v2.Errors decoder := json.NewDecoder(response.Body) err = decoder.Decode(&errs) if err != nil { return 0, 0, err } return 0, 0, &errs default: return 0, 0, &UnexpectedHTTPStatusError{Status: response.Status} } } func (r *clientImpl) UploadBlob(location string, blob io.ReadCloser, length int, dgst digest.Digest) error { defer blob.Close() putRequest, err := http.NewRequest("PUT", location, blob) if err != nil { return err } values := putRequest.URL.Query() values.Set("digest", dgst.String()) putRequest.URL.RawQuery = values.Encode() putRequest.Header.Set("Content-Type", "application/octet-stream") putRequest.Header.Set("Content-Length", fmt.Sprint(length)) response, err := http.DefaultClient.Do(putRequest) if err != nil { return err } defer response.Body.Close() // TODO(bbland): handle other status codes, like 5xx errors switch { case response.StatusCode == http.StatusCreated: return nil case response.StatusCode == http.StatusNotFound: return &BlobUploadNotFoundError{Location: location} case response.StatusCode >= 400 && response.StatusCode < 500: var errs v2.Errors decoder := json.NewDecoder(response.Body) err = decoder.Decode(&errs) if err != nil { return err } return &errs default: return &UnexpectedHTTPStatusError{Status: response.Status} } } func (r *clientImpl) UploadBlobChunk(location string, blobChunk io.ReadCloser, length, startByte int) error { defer blobChunk.Close() putRequest, err := http.NewRequest("PUT", location, blobChunk) if err != nil { return err } endByte := startByte + length putRequest.Header.Set("Content-Type", "application/octet-stream") putRequest.Header.Set("Content-Length", fmt.Sprint(length)) putRequest.Header.Set("Content-Range", fmt.Sprintf("%d-%d/%d", startByte, endByte, endByte)) response, err := http.DefaultClient.Do(putRequest) if err != nil { return err } defer response.Body.Close() // TODO(bbland): handle other status codes, like 5xx errors switch { case response.StatusCode == http.StatusAccepted: return nil case response.StatusCode == http.StatusRequestedRangeNotSatisfiable: lastValidRange, blobSize, err := parseRangeHeader(response.Header.Get("Range")) if err != nil { return err } return &BlobUploadInvalidRangeError{ Location: location, LastValidRange: lastValidRange, BlobSize: blobSize, } case response.StatusCode == http.StatusNotFound: return &BlobUploadNotFoundError{Location: location} case response.StatusCode >= 400 && response.StatusCode < 500: var errs v2.Errors decoder := json.NewDecoder(response.Body) err = decoder.Decode(&errs) if err != nil { return err } return &errs default: return &UnexpectedHTTPStatusError{Status: response.Status} } } func (r *clientImpl) FinishChunkedBlobUpload(location string, length int, dgst digest.Digest) error { putRequest, err := http.NewRequest("PUT", location, nil) if err != nil { return err } values := putRequest.URL.Query() values.Set("digest", dgst.String()) putRequest.URL.RawQuery = values.Encode() putRequest.Header.Set("Content-Type", "application/octet-stream") putRequest.Header.Set("Content-Length", "0") putRequest.Header.Set("Content-Range", fmt.Sprintf("%d-%d/%d", length, length, length)) response, err := http.DefaultClient.Do(putRequest) if err != nil { return err } defer response.Body.Close() // TODO(bbland): handle other status codes, like 5xx errors switch { case response.StatusCode == http.StatusCreated: return nil case response.StatusCode == http.StatusNotFound: return &BlobUploadNotFoundError{Location: location} case response.StatusCode >= 400 && response.StatusCode < 500: var errs v2.Errors decoder := json.NewDecoder(response.Body) err = decoder.Decode(&errs) if err != nil { return err } return &errs default: return &UnexpectedHTTPStatusError{Status: response.Status} } } func (r *clientImpl) CancelBlobUpload(location string) error { deleteRequest, err := http.NewRequest("DELETE", location, nil) if err != nil { return err } response, err := http.DefaultClient.Do(deleteRequest) if err != nil { return err } defer response.Body.Close() // TODO(bbland): handle other status codes, like 5xx errors switch { case response.StatusCode == http.StatusNoContent: return nil case response.StatusCode == http.StatusNotFound: return &BlobUploadNotFoundError{Location: location} case response.StatusCode >= 400 && response.StatusCode < 500: var errs v2.Errors decoder := json.NewDecoder(response.Body) err = decoder.Decode(&errs) if err != nil { return err } return &errs default: return &UnexpectedHTTPStatusError{Status: response.Status} } } // parseRangeHeader parses out the offset and length from a returned Range // header func parseRangeHeader(byteRangeHeader string) (int, int, error) { submatches := patternRangeHeader.FindStringSubmatch(byteRangeHeader) if submatches == nil || len(submatches) < 3 { return 0, 0, fmt.Errorf("Malformed Range header") } offset, err := strconv.Atoi(submatches[1]) if err != nil { return 0, 0, err } length, err := strconv.Atoi(submatches[2]) if err != nil { return 0, 0, err } return offset, length, nil } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/client/pull.go0000644000175000017500000001004312502424227025206 0ustar tianontianonpackage client import ( "fmt" "io" log "github.com/Sirupsen/logrus" "github.com/docker/distribution/manifest" ) // simultaneousLayerPullWindow is the size of the parallel layer pull window. // A layer may not be pulled until the layer preceeding it by the length of the // pull window has been successfully pulled. const simultaneousLayerPullWindow = 4 // Pull implements a client pull workflow for the image defined by the given // name and tag pair, using the given ObjectStore for local manifest and layer // storage func Pull(c Client, objectStore ObjectStore, name, tag string) error { manifest, err := c.GetImageManifest(name, tag) if err != nil { return err } log.WithField("manifest", manifest).Info("Pulled manifest") if len(manifest.FSLayers) != len(manifest.History) { return fmt.Errorf("Length of history not equal to number of layers") } if len(manifest.FSLayers) == 0 { return fmt.Errorf("Image has no layers") } errChans := make([]chan error, len(manifest.FSLayers)) for i := range manifest.FSLayers { errChans[i] = make(chan error) } // To avoid leak of goroutines we must notify // pullLayer goroutines about a cancelation, // otherwise they will lock forever. cancelCh := make(chan struct{}) // Iterate over each layer in the manifest, simultaneously pulling no more // than simultaneousLayerPullWindow layers at a time. If an error is // received from a layer pull, we abort the push. for i := 0; i < len(manifest.FSLayers)+simultaneousLayerPullWindow; i++ { dependentLayer := i - simultaneousLayerPullWindow if dependentLayer >= 0 { err := <-errChans[dependentLayer] if err != nil { log.WithField("error", err).Warn("Pull aborted") close(cancelCh) return err } } if i < len(manifest.FSLayers) { go func(i int) { select { case errChans[i] <- pullLayer(c, objectStore, name, manifest.FSLayers[i]): case <-cancelCh: // no chance to recv until cancelCh's closed } }(i) } } err = objectStore.WriteManifest(name, tag, manifest) if err != nil { log.WithFields(log.Fields{ "error": err, "manifest": manifest, }).Warn("Unable to write image manifest") return err } return nil } func pullLayer(c Client, objectStore ObjectStore, name string, fsLayer manifest.FSLayer) error { log.WithField("layer", fsLayer).Info("Pulling layer") layer, err := objectStore.Layer(fsLayer.BlobSum) if err != nil { log.WithFields(log.Fields{ "error": err, "layer": fsLayer, }).Warn("Unable to write local layer") return err } layerWriter, err := layer.Writer() if err == ErrLayerAlreadyExists { log.WithField("layer", fsLayer).Info("Layer already exists") return nil } if err == ErrLayerLocked { log.WithField("layer", fsLayer).Info("Layer download in progress, waiting") layer.Wait() return nil } if err != nil { log.WithFields(log.Fields{ "error": err, "layer": fsLayer, }).Warn("Unable to write local layer") return err } defer layerWriter.Close() if layerWriter.CurrentSize() > 0 { log.WithFields(log.Fields{ "layer": fsLayer, "currentSize": layerWriter.CurrentSize(), "size": layerWriter.Size(), }).Info("Layer partially downloaded, resuming") } layerReader, length, err := c.GetBlob(name, fsLayer.BlobSum, layerWriter.CurrentSize()) if err != nil { log.WithFields(log.Fields{ "error": err, "layer": fsLayer, }).Warn("Unable to download layer") return err } defer layerReader.Close() layerWriter.SetSize(layerWriter.CurrentSize() + length) _, err = io.Copy(layerWriter, layerReader) if err != nil { log.WithFields(log.Fields{ "error": err, "layer": fsLayer, }).Warn("Unable to download layer") return err } if layerWriter.CurrentSize() != layerWriter.Size() { log.WithFields(log.Fields{ "size": layerWriter.Size(), "currentSize": layerWriter.CurrentSize(), "layer": fsLayer, }).Warn("Layer invalid size") return fmt.Errorf( "Wrote incorrect number of bytes for layer %v. Expected %d, Wrote %d", fsLayer, layerWriter.Size(), layerWriter.CurrentSize(), ) } return nil } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/middleware/0000755000175000017500000000000012502424227024544 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/middleware/repository/0000755000175000017500000000000012502424227026763 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/middleware/repository/middleware.go0000644000175000017500000000224712502424227031434 0ustar tianontianonpackage middleware import ( "fmt" "github.com/docker/distribution" ) // InitFunc is the type of a RepositoryMiddleware factory function and is // used to register the constructor for different RepositoryMiddleware backends. type InitFunc func(repository distribution.Repository, options map[string]interface{}) (distribution.Repository, error) var middlewares map[string]InitFunc // Register is used to register an InitFunc for // a RepositoryMiddleware backend with the given name. func Register(name string, initFunc InitFunc) error { if middlewares == nil { middlewares = make(map[string]InitFunc) } if _, exists := middlewares[name]; exists { return fmt.Errorf("name already registered: %s", name) } middlewares[name] = initFunc return nil } // Get constructs a RepositoryMiddleware with the given options using the named backend. func Get(name string, options map[string]interface{}, repository distribution.Repository) (distribution.Repository, error) { if middlewares != nil { if initFunc, exists := middlewares[name]; exists { return initFunc(repository, options) } } return nil, fmt.Errorf("no repository middleware registered with name: %s", name) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/middleware/registry/0000755000175000017500000000000012502424227026414 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/middleware/registry/middleware.go0000644000175000017500000000221712502424227031062 0ustar tianontianonpackage middleware import ( "fmt" "github.com/docker/distribution" ) // InitFunc is the type of a RegistryMiddleware factory function and is // used to register the constructor for different RegistryMiddleware backends. type InitFunc func(registry distribution.Registry, options map[string]interface{}) (distribution.Registry, error) var middlewares map[string]InitFunc // Register is used to register an InitFunc for // a RegistryMiddleware backend with the given name. func Register(name string, initFunc InitFunc) error { if middlewares == nil { middlewares = make(map[string]InitFunc) } if _, exists := middlewares[name]; exists { return fmt.Errorf("name already registered: %s", name) } middlewares[name] = initFunc return nil } // Get constructs a RegistryMiddleware with the given options using the named backend. func Get(name string, options map[string]interface{}, registry distribution.Registry) (distribution.Registry, error) { if middlewares != nil { if initFunc, exists := middlewares[name]; exists { return initFunc(registry, options) } } return nil, fmt.Errorf("no registry middleware registered with name: %s", name) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/0000755000175000017500000000000012502424227024073 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/paths_test.go0000644000175000017500000000761412502424227026610 0ustar tianontianonpackage storage import ( "testing" "github.com/docker/distribution/digest" ) func TestPathMapper(t *testing.T) { pm := &pathMapper{ root: "/pathmapper-test", } for _, testcase := range []struct { spec pathSpec expected string err error }{ { spec: manifestRevisionPathSpec{ name: "foo/bar", revision: "sha256:abcdef0123456789", }, expected: "/pathmapper-test/repositories/foo/bar/_manifests/revisions/sha256/abcdef0123456789", }, { spec: manifestRevisionLinkPathSpec{ name: "foo/bar", revision: "sha256:abcdef0123456789", }, expected: "/pathmapper-test/repositories/foo/bar/_manifests/revisions/sha256/abcdef0123456789/link", }, { spec: manifestSignatureLinkPathSpec{ name: "foo/bar", revision: "sha256:abcdef0123456789", signature: "sha256:abcdef0123456789", }, expected: "/pathmapper-test/repositories/foo/bar/_manifests/revisions/sha256/abcdef0123456789/signatures/sha256/abcdef0123456789/link", }, { spec: manifestSignaturesPathSpec{ name: "foo/bar", revision: "sha256:abcdef0123456789", }, expected: "/pathmapper-test/repositories/foo/bar/_manifests/revisions/sha256/abcdef0123456789/signatures", }, { spec: manifestTagsPathSpec{ name: "foo/bar", }, expected: "/pathmapper-test/repositories/foo/bar/_manifests/tags", }, { spec: manifestTagPathSpec{ name: "foo/bar", tag: "thetag", }, expected: "/pathmapper-test/repositories/foo/bar/_manifests/tags/thetag", }, { spec: manifestTagCurrentPathSpec{ name: "foo/bar", tag: "thetag", }, expected: "/pathmapper-test/repositories/foo/bar/_manifests/tags/thetag/current/link", }, { spec: manifestTagIndexPathSpec{ name: "foo/bar", tag: "thetag", }, expected: "/pathmapper-test/repositories/foo/bar/_manifests/tags/thetag/index", }, { spec: manifestTagIndexEntryPathSpec{ name: "foo/bar", tag: "thetag", revision: "sha256:abcdef0123456789", }, expected: "/pathmapper-test/repositories/foo/bar/_manifests/tags/thetag/index/sha256/abcdef0123456789", }, { spec: manifestTagIndexEntryLinkPathSpec{ name: "foo/bar", tag: "thetag", revision: "sha256:abcdef0123456789", }, expected: "/pathmapper-test/repositories/foo/bar/_manifests/tags/thetag/index/sha256/abcdef0123456789/link", }, { spec: layerLinkPathSpec{ name: "foo/bar", digest: "tarsum.v1+test:abcdef", }, expected: "/pathmapper-test/repositories/foo/bar/_layers/tarsum/v1/test/abcdef/link", }, { spec: blobDataPathSpec{ digest: digest.Digest("tarsum.dev+sha512:abcdefabcdefabcdef908909909"), }, expected: "/pathmapper-test/blobs/tarsum/dev/sha512/ab/abcdefabcdefabcdef908909909/data", }, { spec: blobDataPathSpec{ digest: digest.Digest("tarsum.v1+sha256:abcdefabcdefabcdef908909909"), }, expected: "/pathmapper-test/blobs/tarsum/v1/sha256/ab/abcdefabcdefabcdef908909909/data", }, { spec: uploadDataPathSpec{ name: "foo/bar", uuid: "asdf-asdf-asdf-adsf", }, expected: "/pathmapper-test/repositories/foo/bar/_uploads/asdf-asdf-asdf-adsf/data", }, { spec: uploadStartedAtPathSpec{ name: "foo/bar", uuid: "asdf-asdf-asdf-adsf", }, expected: "/pathmapper-test/repositories/foo/bar/_uploads/asdf-asdf-asdf-adsf/startedat", }, } { p, err := pm.path(testcase.spec) if err != nil { t.Fatalf("unexpected generating path (%T): %v", testcase.spec, err) } if p != testcase.expected { t.Fatalf("unexpected path generated (%T): %q != %q", testcase.spec, p, testcase.expected) } } // Add a few test cases to ensure we cover some errors // Specify a path that requires a revision and get a digest validation error. badpath, err := pm.path(manifestSignaturesPathSpec{ name: "foo/bar", }) if err == nil { t.Fatalf("expected an error when mapping an invalid revision: %s", badpath) } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/manifeststore_test.go0000644000175000017500000001633612502424227030355 0ustar tianontianonpackage storage import ( "bytes" "io" "reflect" "testing" "github.com/docker/distribution" "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest" "github.com/docker/distribution/registry/storage/driver" "github.com/docker/distribution/registry/storage/driver/inmemory" "github.com/docker/distribution/testutil" "github.com/docker/libtrust" "golang.org/x/net/context" ) type manifestStoreTestEnv struct { ctx context.Context driver driver.StorageDriver registry distribution.Registry repository distribution.Repository name string tag string } func newManifestStoreTestEnv(t *testing.T, name, tag string) *manifestStoreTestEnv { ctx := context.Background() driver := inmemory.New() registry := NewRegistryWithDriver(driver) repo, err := registry.Repository(ctx, name) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } return &manifestStoreTestEnv{ ctx: ctx, driver: driver, registry: registry, repository: repo, name: name, tag: tag, } } func TestManifestStorage(t *testing.T) { env := newManifestStoreTestEnv(t, "foo/bar", "thetag") ms := env.repository.Manifests() exists, err := ms.ExistsByTag(env.tag) if err != nil { t.Fatalf("unexpected error checking manifest existence: %v", err) } if exists { t.Fatalf("manifest should not exist") } if _, err := ms.GetByTag(env.tag); true { switch err.(type) { case distribution.ErrManifestUnknown: break default: t.Fatalf("expected manifest unknown error: %#v", err) } } m := manifest.Manifest{ Versioned: manifest.Versioned{ SchemaVersion: 1, }, Name: env.name, Tag: env.tag, } // Build up some test layers and add them to the manifest, saving the // readseekers for upload later. testLayers := map[digest.Digest]io.ReadSeeker{} for i := 0; i < 2; i++ { rs, ds, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("unexpected error generating test layer file") } dgst := digest.Digest(ds) testLayers[digest.Digest(dgst)] = rs m.FSLayers = append(m.FSLayers, manifest.FSLayer{ BlobSum: dgst, }) } pk, err := libtrust.GenerateECP256PrivateKey() if err != nil { t.Fatalf("unexpected error generating private key: %v", err) } sm, err := manifest.Sign(&m, pk) if err != nil { t.Fatalf("error signing manifest: %v", err) } err = ms.Put(sm) if err == nil { t.Fatalf("expected errors putting manifest") } // TODO(stevvooe): We expect errors describing all of the missing layers. // Now, upload the layers that were missing! for dgst, rs := range testLayers { upload, err := env.repository.Layers().Upload() if err != nil { t.Fatalf("unexpected error creating test upload: %v", err) } if _, err := io.Copy(upload, rs); err != nil { t.Fatalf("unexpected error copying to upload: %v", err) } if _, err := upload.Finish(dgst); err != nil { t.Fatalf("unexpected error finishing upload: %v", err) } } if err = ms.Put(sm); err != nil { t.Fatalf("unexpected error putting manifest: %v", err) } exists, err = ms.ExistsByTag(env.tag) if err != nil { t.Fatalf("unexpected error checking manifest existence: %v", err) } if !exists { t.Fatalf("manifest should exist") } fetchedManifest, err := ms.GetByTag(env.tag) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } if !reflect.DeepEqual(fetchedManifest, sm) { t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedManifest, sm) } fetchedJWS, err := libtrust.ParsePrettySignature(fetchedManifest.Raw, "signatures") if err != nil { t.Fatalf("unexpected error parsing jws: %v", err) } payload, err := fetchedJWS.Payload() if err != nil { t.Fatalf("unexpected error extracting payload: %v", err) } // Now that we have a payload, take a moment to check that the manifest is // return by the payload digest. dgst, err := digest.FromBytes(payload) if err != nil { t.Fatalf("error getting manifest digest: %v", err) } exists, err = ms.Exists(dgst) if err != nil { t.Fatalf("error checking manifest existence by digest: %v", err) } if !exists { t.Fatalf("manifest %s should exist", dgst) } fetchedByDigest, err := ms.Get(dgst) if err != nil { t.Fatalf("unexpected error fetching manifest by digest: %v", err) } if !reflect.DeepEqual(fetchedByDigest, fetchedManifest) { t.Fatalf("fetched manifest not equal: %#v != %#v", fetchedByDigest, fetchedManifest) } sigs, err := fetchedJWS.Signatures() if err != nil { t.Fatalf("unable to extract signatures: %v", err) } if len(sigs) != 1 { t.Fatalf("unexpected number of signatures: %d != %d", len(sigs), 1) } // Grabs the tags and check that this tagged manifest is present tags, err := ms.Tags() if err != nil { t.Fatalf("unexpected error fetching tags: %v", err) } if len(tags) != 1 { t.Fatalf("unexpected tags returned: %v", tags) } if tags[0] != env.tag { t.Fatalf("unexpected tag found in tags: %v != %v", tags, []string{env.tag}) } // Now, push the same manifest with a different key pk2, err := libtrust.GenerateECP256PrivateKey() if err != nil { t.Fatalf("unexpected error generating private key: %v", err) } sm2, err := manifest.Sign(&m, pk2) if err != nil { t.Fatalf("unexpected error signing manifest: %v", err) } jws2, err := libtrust.ParsePrettySignature(sm2.Raw, "signatures") if err != nil { t.Fatalf("error parsing signature: %v", err) } sigs2, err := jws2.Signatures() if err != nil { t.Fatalf("unable to extract signatures: %v", err) } if len(sigs2) != 1 { t.Fatalf("unexpected number of signatures: %d != %d", len(sigs2), 1) } if err = ms.Put(sm2); err != nil { t.Fatalf("unexpected error putting manifest: %v", err) } fetched, err := ms.GetByTag(env.tag) if err != nil { t.Fatalf("unexpected error fetching manifest: %v", err) } if _, err := manifest.Verify(fetched); err != nil { t.Fatalf("unexpected error verifying manifest: %v", err) } // Assemble our payload and two signatures to get what we expect! expectedJWS, err := libtrust.NewJSONSignature(payload, sigs[0], sigs2[0]) if err != nil { t.Fatalf("unexpected error merging jws: %v", err) } expectedSigs, err := expectedJWS.Signatures() if err != nil { t.Fatalf("unexpected error getting expected signatures: %v", err) } receivedJWS, err := libtrust.ParsePrettySignature(fetched.Raw, "signatures") if err != nil { t.Fatalf("unexpected error parsing jws: %v", err) } receivedPayload, err := receivedJWS.Payload() if err != nil { t.Fatalf("unexpected error extracting received payload: %v", err) } if !bytes.Equal(receivedPayload, payload) { t.Fatalf("payloads are not equal") } receivedSigs, err := receivedJWS.Signatures() if err != nil { t.Fatalf("error getting signatures: %v", err) } for i, sig := range receivedSigs { if !bytes.Equal(sig, expectedSigs[i]) { t.Fatalf("mismatched signatures from remote: %v != %v", string(sig), string(expectedSigs[i])) } } // TODO(stevvooe): Currently, deletes are not supported due to some // complexity around managing tag indexes. We'll add this support back in // when the manifest format has settled. For now, we expect an error for // all deletes. if err := ms.Delete(dgst); err == nil { t.Fatalf("unexpected an error deleting manifest by digest: %v", err) } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/layerstore.go0000644000175000017500000001024512502424227026615 0ustar tianontianonpackage storage import ( "time" "code.google.com/p/go-uuid/uuid" "github.com/docker/distribution" ctxu "github.com/docker/distribution/context" "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest" storagedriver "github.com/docker/distribution/registry/storage/driver" ) type layerStore struct { repository *repository } func (ls *layerStore) Exists(digest digest.Digest) (bool, error) { ctxu.GetLogger(ls.repository.ctx).Debug("(*layerStore).Exists") // Because this implementation just follows blob links, an existence check // is pretty cheap by starting and closing a fetch. _, err := ls.Fetch(digest) if err != nil { switch err.(type) { case distribution.ErrUnknownLayer: return false, nil } return false, err } return true, nil } func (ls *layerStore) Fetch(dgst digest.Digest) (distribution.Layer, error) { ctxu.GetLogger(ls.repository.ctx).Debug("(*layerStore).Fetch") bp, err := ls.path(dgst) if err != nil { return nil, err } fr, err := newFileReader(ls.repository.driver, bp) if err != nil { return nil, err } return &layerReader{ fileReader: *fr, digest: dgst, }, nil } // Upload begins a layer upload, returning a handle. If the layer upload // is already in progress or the layer has already been uploaded, this // will return an error. func (ls *layerStore) Upload() (distribution.LayerUpload, error) { ctxu.GetLogger(ls.repository.ctx).Debug("(*layerStore).Upload") // NOTE(stevvooe): Consider the issues with allowing concurrent upload of // the same two layers. Should it be disallowed? For now, we allow both // parties to proceed and the the first one uploads the layer. uuid := uuid.New() startedAt := time.Now().UTC() path, err := ls.repository.registry.pm.path(uploadDataPathSpec{ name: ls.repository.Name(), uuid: uuid, }) if err != nil { return nil, err } startedAtPath, err := ls.repository.registry.pm.path(uploadStartedAtPathSpec{ name: ls.repository.Name(), uuid: uuid, }) if err != nil { return nil, err } // Write a startedat file for this upload if err := ls.repository.driver.PutContent(startedAtPath, []byte(startedAt.Format(time.RFC3339))); err != nil { return nil, err } return ls.newLayerUpload(uuid, path, startedAt) } // Resume continues an in progress layer upload, returning the current // state of the upload. func (ls *layerStore) Resume(uuid string) (distribution.LayerUpload, error) { ctxu.GetLogger(ls.repository.ctx).Debug("(*layerStore).Resume") startedAtPath, err := ls.repository.registry.pm.path(uploadStartedAtPathSpec{ name: ls.repository.Name(), uuid: uuid, }) if err != nil { return nil, err } startedAtBytes, err := ls.repository.driver.GetContent(startedAtPath) if err != nil { switch err := err.(type) { case storagedriver.PathNotFoundError: return nil, distribution.ErrLayerUploadUnknown default: return nil, err } } startedAt, err := time.Parse(time.RFC3339, string(startedAtBytes)) if err != nil { return nil, err } path, err := ls.repository.pm.path(uploadDataPathSpec{ name: ls.repository.Name(), uuid: uuid, }) if err != nil { return nil, err } return ls.newLayerUpload(uuid, path, startedAt) } // newLayerUpload allocates a new upload controller with the given state. func (ls *layerStore) newLayerUpload(uuid, path string, startedAt time.Time) (distribution.LayerUpload, error) { fw, err := newFileWriter(ls.repository.driver, path) if err != nil { return nil, err } return &layerWriter{ layerStore: ls, uuid: uuid, startedAt: startedAt, bufferedFileWriter: *fw, }, nil } func (ls *layerStore) path(dgst digest.Digest) (string, error) { // We must traverse this path through the link to enforce ownership. layerLinkPath, err := ls.repository.registry.pm.path(layerLinkPathSpec{name: ls.repository.Name(), digest: dgst}) if err != nil { return "", err } blobPath, err := ls.repository.blobStore.resolve(layerLinkPath) if err != nil { switch err := err.(type) { case storagedriver.PathNotFoundError: return "", distribution.ErrUnknownLayer{ FSLayer: manifest.FSLayer{BlobSum: dgst}, } default: return "", err } } return blobPath, nil } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/layerwriter.go0000644000175000017500000001557312502424227027006 0ustar tianontianonpackage storage import ( "fmt" "io" "path" "time" "github.com/Sirupsen/logrus" "github.com/docker/distribution" ctxu "github.com/docker/distribution/context" "github.com/docker/distribution/digest" storagedriver "github.com/docker/distribution/registry/storage/driver" ) var _ distribution.LayerUpload = &layerWriter{} // layerWriter is used to control the various aspects of resumable // layer upload. It implements the LayerUpload interface. type layerWriter struct { layerStore *layerStore uuid string startedAt time.Time // implementes io.WriteSeeker, io.ReaderFrom and io.Closer to satisy // LayerUpload Interface bufferedFileWriter } var _ distribution.LayerUpload = &layerWriter{} // UUID returns the identifier for this upload. func (lw *layerWriter) UUID() string { return lw.uuid } func (lw *layerWriter) StartedAt() time.Time { return lw.startedAt } // Finish marks the upload as completed, returning a valid handle to the // uploaded layer. The final size and checksum are validated against the // contents of the uploaded layer. The checksum should be provided in the // format :. func (lw *layerWriter) Finish(digest digest.Digest) (distribution.Layer, error) { ctxu.GetLogger(lw.layerStore.repository.ctx).Debug("(*layerWriter).Finish") if err := lw.bufferedFileWriter.Close(); err != nil { return nil, err } canonical, err := lw.validateLayer(digest) if err != nil { return nil, err } if err := lw.moveLayer(canonical); err != nil { // TODO(stevvooe): Cleanup? return nil, err } // Link the layer blob into the repository. if err := lw.linkLayer(canonical, digest); err != nil { return nil, err } if err := lw.removeResources(); err != nil { return nil, err } return lw.layerStore.Fetch(canonical) } // Cancel the layer upload process. func (lw *layerWriter) Cancel() error { ctxu.GetLogger(lw.layerStore.repository.ctx).Debug("(*layerWriter).Cancel") if err := lw.removeResources(); err != nil { return err } lw.Close() return nil } // validateLayer checks the layer data against the digest, returning an error // if it does not match. The canonical digest is returned. func (lw *layerWriter) validateLayer(dgst digest.Digest) (digest.Digest, error) { digestVerifier, err := digest.NewDigestVerifier(dgst) if err != nil { return "", err } // TODO(stevvooe): Store resumable hash calculations in upload directory // in driver. Something like a file at path /resumablehash/ // with the hash state up to that point would be perfect. The hasher would // then only have to fetch the difference. // Read the file from the backend driver and validate it. fr, err := newFileReader(lw.bufferedFileWriter.driver, lw.path) if err != nil { return "", err } tr := io.TeeReader(fr, digestVerifier) // TODO(stevvooe): This is one of the places we need a Digester write // sink. Instead, its read driven. This might be okay. // Calculate an updated digest with the latest version. canonical, err := digest.FromReader(tr) if err != nil { return "", err } if !digestVerifier.Verified() { return "", distribution.ErrLayerInvalidDigest{ Digest: dgst, Reason: fmt.Errorf("content does not match digest"), } } return canonical, nil } // moveLayer moves the data into its final, hash-qualified destination, // identified by dgst. The layer should be validated before commencing the // move. func (lw *layerWriter) moveLayer(dgst digest.Digest) error { blobPath, err := lw.layerStore.repository.registry.pm.path(blobDataPathSpec{ digest: dgst, }) if err != nil { return err } // Check for existence if _, err := lw.driver.Stat(blobPath); err != nil { switch err := err.(type) { case storagedriver.PathNotFoundError: break // ensure that it doesn't exist. default: return err } } else { // If the path exists, we can assume that the content has already // been uploaded, since the blob storage is content-addressable. // While it may be corrupted, detection of such corruption belongs // elsewhere. return nil } // If no data was received, we may not actually have a file on disk. Check // the size here and write a zero-length file to blobPath if this is the // case. For the most part, this should only ever happen with zero-length // tars. if _, err := lw.driver.Stat(lw.path); err != nil { switch err := err.(type) { case storagedriver.PathNotFoundError: // HACK(stevvooe): This is slightly dangerous: if we verify above, // get a hash, then the underlying file is deleted, we risk moving // a zero-length blob into a nonzero-length blob location. To // prevent this horrid thing, we employ the hack of only allowing // to this happen for the zero tarsum. if dgst == digest.DigestSha256EmptyTar { return lw.driver.PutContent(blobPath, []byte{}) } // We let this fail during the move below. logrus. WithField("upload.uuid", lw.UUID()). WithField("digest", dgst).Warnf("attempted to move zero-length content with non-zero digest") default: return err // unrelated error } } return lw.driver.Move(lw.path, blobPath) } // linkLayer links a valid, written layer blob into the registry under the // named repository for the upload controller. func (lw *layerWriter) linkLayer(canonical digest.Digest, aliases ...digest.Digest) error { dgsts := append([]digest.Digest{canonical}, aliases...) // Don't make duplicate links. seenDigests := make(map[digest.Digest]struct{}, len(dgsts)) for _, dgst := range dgsts { if _, seen := seenDigests[dgst]; seen { continue } seenDigests[dgst] = struct{}{} layerLinkPath, err := lw.layerStore.repository.registry.pm.path(layerLinkPathSpec{ name: lw.layerStore.repository.Name(), digest: dgst, }) if err != nil { return err } if err := lw.layerStore.repository.registry.driver.PutContent(layerLinkPath, []byte(canonical)); err != nil { return err } } return nil } // removeResources should clean up all resources associated with the upload // instance. An error will be returned if the clean up cannot proceed. If the // resources are already not present, no error will be returned. func (lw *layerWriter) removeResources() error { dataPath, err := lw.layerStore.repository.registry.pm.path(uploadDataPathSpec{ name: lw.layerStore.repository.Name(), uuid: lw.uuid, }) if err != nil { return err } // Resolve and delete the containing directory, which should include any // upload related files. dirPath := path.Dir(dataPath) if err := lw.driver.Delete(dirPath); err != nil { switch err := err.(type) { case storagedriver.PathNotFoundError: break // already gone! default: // This should be uncommon enough such that returning an error // should be okay. At this point, the upload should be mostly // complete, but perhaps the backend became unaccessible. logrus.Errorf("unable to delete layer upload resources %q: %v", dirPath, err) return err } } return nil } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/layerreader.go0000644000175000017500000000307712502424227026730 0ustar tianontianonpackage storage import ( "net/http" "time" "github.com/docker/distribution" "github.com/docker/distribution/digest" "github.com/docker/distribution/registry/storage/driver" ) // layerReader implements Layer and provides facilities for reading and // seeking. type layerReader struct { fileReader digest digest.Digest } var _ distribution.Layer = &layerReader{} func (lr *layerReader) Digest() digest.Digest { return lr.digest } func (lr *layerReader) Length() int64 { return lr.size } func (lr *layerReader) CreatedAt() time.Time { return lr.modtime } // Close the layer. Should be called when the resource is no longer needed. func (lr *layerReader) Close() error { return lr.closeWithErr(distribution.ErrLayerClosed) } func (lr *layerReader) Handler(r *http.Request) (h http.Handler, err error) { var handlerFunc http.HandlerFunc redirectURL, err := lr.fileReader.driver.URLFor(lr.path, map[string]interface{}{"method": r.Method}) switch err { case nil: handlerFunc = func(w http.ResponseWriter, r *http.Request) { // Redirect to storage URL. http.Redirect(w, r, redirectURL, http.StatusTemporaryRedirect) } case driver.ErrUnsupportedMethod: handlerFunc = func(w http.ResponseWriter, r *http.Request) { // Fallback to serving the content directly. http.ServeContent(w, r, lr.digest.String(), lr.CreatedAt(), lr) } default: // Some unexpected error. return nil, err } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Docker-Content-Digest", lr.digest.String()) handlerFunc.ServeHTTP(w, r) }), nil } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/registry.go0000644000175000017500000000472412502424227026301 0ustar tianontianonpackage storage import ( "github.com/docker/distribution" "github.com/docker/distribution/registry/api/v2" storagedriver "github.com/docker/distribution/registry/storage/driver" "golang.org/x/net/context" ) // registry is the top-level implementation of Registry for use in the storage // package. All instances should descend from this object. type registry struct { driver storagedriver.StorageDriver pm *pathMapper blobStore *blobStore } // NewRegistryWithDriver creates a new registry instance from the provided // driver. The resulting registry may be shared by multiple goroutines but is // cheap to allocate. func NewRegistryWithDriver(driver storagedriver.StorageDriver) distribution.Registry { bs := &blobStore{} reg := ®istry{ driver: driver, blobStore: bs, // TODO(sday): This should be configurable. pm: defaultPathMapper, } reg.blobStore.registry = reg return reg } // Repository returns an instance of the repository tied to the registry. // Instances should not be shared between goroutines but are cheap to // allocate. In general, they should be request scoped. func (reg *registry) Repository(ctx context.Context, name string) (distribution.Repository, error) { if err := v2.ValidateRespositoryName(name); err != nil { return nil, distribution.ErrRepositoryNameInvalid{ Name: name, Reason: err, } } return &repository{ ctx: ctx, registry: reg, name: name, }, nil } // repository provides name-scoped access to various services. type repository struct { *registry ctx context.Context name string } // Name returns the name of the repository. func (repo *repository) Name() string { return repo.name } // Manifests returns an instance of ManifestService. Instantiation is cheap and // may be context sensitive in the future. The instance should be used similar // to a request local. func (repo *repository) Manifests() distribution.ManifestService { return &manifestStore{ repository: repo, revisionStore: &revisionStore{ repository: repo, }, tagStore: &tagStore{ repository: repo, }, } } // Layers returns an instance of the LayerService. Instantiation is cheap and // may be context sensitive in the future. The instance should be used similar // to a request local. func (repo *repository) Layers() distribution.LayerService { return &layerStore{ repository: repo, } } func (repo *repository) Signatures() distribution.SignatureService { return &signatureStore{ repository: repo, } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/filereader.go0000644000175000017500000001175412502424227026534 0ustar tianontianonpackage storage import ( "bufio" "bytes" "fmt" "io" "io/ioutil" "os" "time" storagedriver "github.com/docker/distribution/registry/storage/driver" ) // TODO(stevvooe): Set an optimal buffer size here. We'll have to // understand the latency characteristics of the underlying network to // set this correctly, so we may want to leave it to the driver. For // out of process drivers, we'll have to optimize this buffer size for // local communication. const fileReaderBufferSize = 4 << 20 // remoteFileReader provides a read seeker interface to files stored in // storagedriver. Used to implement part of layer interface and will be used // to implement read side of LayerUpload. type fileReader struct { driver storagedriver.StorageDriver // identifying fields path string size int64 // size is the total layer size, must be set. modtime time.Time // mutable fields rc io.ReadCloser // remote read closer brd *bufio.Reader // internal buffered io offset int64 // offset is the current read offset err error // terminal error, if set, reader is closed } // newFileReader initializes a file reader for the remote file. The read takes // on the offset and size at the time the reader is created. If the underlying // file changes, one must create a new fileReader. func newFileReader(driver storagedriver.StorageDriver, path string) (*fileReader, error) { rd := &fileReader{ driver: driver, path: path, } // Grab the size of the layer file, ensuring existence. if fi, err := driver.Stat(path); err != nil { switch err := err.(type) { case storagedriver.PathNotFoundError: // NOTE(stevvooe): We really don't care if the file is not // actually present for the reader. If the caller needs to know // whether or not the file exists, they should issue a stat call // on the path. There is still no guarantee, since the file may be // gone by the time the reader is created. The only correct // behavior is to return a reader that immediately returns EOF. default: // Any other error we want propagated up the stack. return nil, err } } else { if fi.IsDir() { return nil, fmt.Errorf("cannot read a directory") } // Fill in file information rd.size = fi.Size() rd.modtime = fi.ModTime() } return rd, nil } func (fr *fileReader) Read(p []byte) (n int, err error) { if fr.err != nil { return 0, fr.err } rd, err := fr.reader() if err != nil { return 0, err } n, err = rd.Read(p) fr.offset += int64(n) // Simulate io.EOR error if we reach filesize. if err == nil && fr.offset >= fr.size { err = io.EOF } return n, err } func (fr *fileReader) Seek(offset int64, whence int) (int64, error) { if fr.err != nil { return 0, fr.err } var err error newOffset := fr.offset switch whence { case os.SEEK_CUR: newOffset += int64(offset) case os.SEEK_END: newOffset = fr.size + int64(offset) case os.SEEK_SET: newOffset = int64(offset) } if newOffset < 0 { err = fmt.Errorf("cannot seek to negative position") } else { if fr.offset != newOffset { fr.reset() } // No problems, set the offset. fr.offset = newOffset } return fr.offset, err } func (fr *fileReader) Close() error { return fr.closeWithErr(fmt.Errorf("fileReader: closed")) } // reader prepares the current reader at the lrs offset, ensuring its buffered // and ready to go. func (fr *fileReader) reader() (io.Reader, error) { if fr.err != nil { return nil, fr.err } if fr.rc != nil { return fr.brd, nil } // If we don't have a reader, open one up. rc, err := fr.driver.ReadStream(fr.path, fr.offset) if err != nil { switch err := err.(type) { case storagedriver.PathNotFoundError: // NOTE(stevvooe): If the path is not found, we simply return a // reader that returns io.EOF. However, we do not set fr.rc, // allowing future attempts at getting a reader to possibly // succeed if the file turns up later. return ioutil.NopCloser(bytes.NewReader([]byte{})), nil default: return nil, err } } fr.rc = rc if fr.brd == nil { // TODO(stevvooe): Set an optimal buffer size here. We'll have to // understand the latency characteristics of the underlying network to // set this correctly, so we may want to leave it to the driver. For // out of process drivers, we'll have to optimize this buffer size for // local communication. fr.brd = bufio.NewReaderSize(fr.rc, fileReaderBufferSize) } else { fr.brd.Reset(fr.rc) } return fr.brd, nil } // resetReader resets the reader, forcing the read method to open up a new // connection and rebuild the buffered reader. This should be called when the // offset and the reader will become out of sync, such as during a seek // operation. func (fr *fileReader) reset() { if fr.err != nil { return } if fr.rc != nil { fr.rc.Close() fr.rc = nil } } func (fr *fileReader) closeWithErr(err error) error { if fr.err != nil { return fr.err } fr.err = err // close and release reader chain if fr.rc != nil { fr.rc.Close() } fr.rc = nil fr.brd = nil return fr.err } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/filereader_test.go0000644000175000017500000001206112502424227027563 0ustar tianontianonpackage storage import ( "bytes" "crypto/rand" "io" mrand "math/rand" "os" "testing" "github.com/docker/distribution/digest" "github.com/docker/distribution/registry/storage/driver/inmemory" ) func TestSimpleRead(t *testing.T) { content := make([]byte, 1<<20) n, err := rand.Read(content) if err != nil { t.Fatalf("unexpected error building random data: %v", err) } if n != len(content) { t.Fatalf("random read did't fill buffer") } dgst, err := digest.FromReader(bytes.NewReader(content)) if err != nil { t.Fatalf("unexpected error digesting random content: %v", err) } driver := inmemory.New() path := "/random" if err := driver.PutContent(path, content); err != nil { t.Fatalf("error putting patterned content: %v", err) } fr, err := newFileReader(driver, path) if err != nil { t.Fatalf("error allocating file reader: %v", err) } verifier, err := digest.NewDigestVerifier(dgst) if err != nil { t.Fatalf("error getting digest verifier: %s", err) } io.Copy(verifier, fr) if !verifier.Verified() { t.Fatalf("unable to verify read data") } } func TestFileReaderSeek(t *testing.T) { driver := inmemory.New() pattern := "01234567890ab" // prime length block repititions := 1024 path := "/patterned" content := bytes.Repeat([]byte(pattern), repititions) if err := driver.PutContent(path, content); err != nil { t.Fatalf("error putting patterned content: %v", err) } fr, err := newFileReader(driver, path) if err != nil { t.Fatalf("unexpected error creating file reader: %v", err) } // Seek all over the place, in blocks of pattern size and make sure we get // the right data. for _, repitition := range mrand.Perm(repititions - 1) { targetOffset := int64(len(pattern) * repitition) // Seek to a multiple of pattern size and read pattern size bytes offset, err := fr.Seek(targetOffset, os.SEEK_SET) if err != nil { t.Fatalf("unexpected error seeking: %v", err) } if offset != targetOffset { t.Fatalf("did not seek to correct offset: %d != %d", offset, targetOffset) } p := make([]byte, len(pattern)) n, err := fr.Read(p) if err != nil { t.Fatalf("error reading pattern: %v", err) } if n != len(pattern) { t.Fatalf("incorrect read length: %d != %d", n, len(pattern)) } if string(p) != pattern { t.Fatalf("incorrect read content: %q != %q", p, pattern) } // Check offset current, err := fr.Seek(0, os.SEEK_CUR) if err != nil { t.Fatalf("error checking current offset: %v", err) } if current != targetOffset+int64(len(pattern)) { t.Fatalf("unexpected offset after read: %v", err) } } start, err := fr.Seek(0, os.SEEK_SET) if err != nil { t.Fatalf("error seeking to start: %v", err) } if start != 0 { t.Fatalf("expected to seek to start: %v != 0", start) } end, err := fr.Seek(0, os.SEEK_END) if err != nil { t.Fatalf("error checking current offset: %v", err) } if end != int64(len(content)) { t.Fatalf("expected to seek to end: %v != %v", end, len(content)) } // 4. Seek before start, ensure error. // seek before start before, err := fr.Seek(-1, os.SEEK_SET) if err == nil { t.Fatalf("error expected, returned offset=%v", before) } // 5. Seek after end, after, err := fr.Seek(1, os.SEEK_END) if err != nil { t.Fatalf("unexpected error expected, returned offset=%v", after) } p := make([]byte, 16) n, err := fr.Read(p) if n != 0 { t.Fatalf("bytes reads %d != %d", n, 0) } if err != io.EOF { t.Fatalf("expected io.EOF, got %v", err) } } // TestFileReaderNonExistentFile ensures the reader behaves as expected with a // missing or zero-length remote file. While the file may not exist, the // reader should not error out on creation and should return 0-bytes from the // read method, with an io.EOF error. func TestFileReaderNonExistentFile(t *testing.T) { driver := inmemory.New() fr, err := newFileReader(driver, "/doesnotexist") if err != nil { t.Fatalf("unexpected error initializing reader: %v", err) } var buf [1024]byte n, err := fr.Read(buf[:]) if n != 0 { t.Fatalf("non-zero byte read reported: %d != 0", n) } if err != io.EOF { t.Fatalf("read on missing file should return io.EOF, got %v", err) } } // TestLayerReadErrors covers the various error return type for different // conditions that can arise when reading a layer. func TestFileReaderErrors(t *testing.T) { // TODO(stevvooe): We need to cover error return types, driven by the // errors returned via the HTTP API. For now, here is a incomplete list: // // 1. Layer Not Found: returned when layer is not found or access is // denied. // 2. Layer Unavailable: returned when link references are unresolved, // but layer is known to the registry. // 3. Layer Invalid: This may more split into more errors, but should be // returned when name or tarsum does not reference a valid error. We // may also need something to communication layer verification errors // for the inline tarsum check. // 4. Timeout: timeouts to backend. Need to better understand these // failure cases and how the storage driver propagates these errors // up the stack. } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/tagstore.go0000644000175000017500000000667012502424227026263 0ustar tianontianonpackage storage import ( "path" "github.com/docker/distribution" "github.com/docker/distribution/digest" storagedriver "github.com/docker/distribution/registry/storage/driver" ) // tagStore provides methods to manage manifest tags in a backend storage driver. type tagStore struct { *repository } // tags lists the manifest tags for the specified repository. func (ts *tagStore) tags() ([]string, error) { p, err := ts.pm.path(manifestTagPathSpec{ name: ts.name, }) if err != nil { return nil, err } var tags []string entries, err := ts.driver.List(p) if err != nil { switch err := err.(type) { case storagedriver.PathNotFoundError: return nil, distribution.ErrRepositoryUnknown{Name: ts.name} default: return nil, err } } for _, entry := range entries { _, filename := path.Split(entry) tags = append(tags, filename) } return tags, nil } // exists returns true if the specified manifest tag exists in the repository. func (ts *tagStore) exists(tag string) (bool, error) { tagPath, err := ts.pm.path(manifestTagCurrentPathSpec{ name: ts.Name(), tag: tag, }) if err != nil { return false, err } exists, err := exists(ts.driver, tagPath) if err != nil { return false, err } return exists, nil } // tag tags the digest with the given tag, updating the the store to point at // the current tag. The digest must point to a manifest. func (ts *tagStore) tag(tag string, revision digest.Digest) error { indexEntryPath, err := ts.pm.path(manifestTagIndexEntryLinkPathSpec{ name: ts.Name(), tag: tag, revision: revision, }) if err != nil { return err } currentPath, err := ts.pm.path(manifestTagCurrentPathSpec{ name: ts.Name(), tag: tag, }) if err != nil { return err } // Link into the index if err := ts.blobStore.link(indexEntryPath, revision); err != nil { return err } // Overwrite the current link return ts.blobStore.link(currentPath, revision) } // resolve the current revision for name and tag. func (ts *tagStore) resolve(tag string) (digest.Digest, error) { currentPath, err := ts.pm.path(manifestTagCurrentPathSpec{ name: ts.Name(), tag: tag, }) if err != nil { return "", err } if exists, err := exists(ts.driver, currentPath); err != nil { return "", err } else if !exists { return "", distribution.ErrManifestUnknown{Name: ts.Name(), Tag: tag} } revision, err := ts.blobStore.readlink(currentPath) if err != nil { return "", err } return revision, nil } // revisions returns all revisions with the specified name and tag. func (ts *tagStore) revisions(tag string) ([]digest.Digest, error) { manifestTagIndexPath, err := ts.pm.path(manifestTagIndexPathSpec{ name: ts.Name(), tag: tag, }) if err != nil { return nil, err } // TODO(stevvooe): Need to append digest alg to get listing of revisions. manifestTagIndexPath = path.Join(manifestTagIndexPath, "sha256") entries, err := ts.driver.List(manifestTagIndexPath) if err != nil { return nil, err } var revisions []digest.Digest for _, entry := range entries { revisions = append(revisions, digest.NewDigestFromHex("sha256", path.Base(entry))) } return revisions, nil } // delete removes the tag from repository, including the history of all // revisions that have the specified tag. func (ts *tagStore) delete(tag string) error { tagPath, err := ts.pm.path(manifestTagPathSpec{ name: ts.Name(), tag: tag, }) if err != nil { return err } return ts.driver.Delete(tagPath) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/doc.go0000644000175000017500000000024012502424227025163 0ustar tianontianon// Package storage contains storage services for use in the registry // application. It should be considered an internal package, as of Go 1.4. package storage distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/filewriter.go0000644000175000017500000001146512502424227026605 0ustar tianontianonpackage storage import ( "bufio" "bytes" "fmt" "io" "os" storagedriver "github.com/docker/distribution/registry/storage/driver" ) const ( fileWriterBufferSize = 5 << 20 ) // fileWriter implements a remote file writer backed by a storage driver. type fileWriter struct { driver storagedriver.StorageDriver // identifying fields path string // mutable fields size int64 // size of the file, aka the current end offset int64 // offset is the current write offset err error // terminal error, if set, reader is closed } type bufferedFileWriter struct { fileWriter bw *bufio.Writer } // fileWriterInterface makes the desired io compliant interface that the // filewriter should implement. type fileWriterInterface interface { io.WriteSeeker io.WriterAt io.ReaderFrom io.Closer } var _ fileWriterInterface = &fileWriter{} // newFileWriter returns a prepared fileWriter for the driver and path. This // could be considered similar to an "open" call on a regular filesystem. func newFileWriter(driver storagedriver.StorageDriver, path string) (*bufferedFileWriter, error) { fw := fileWriter{ driver: driver, path: path, } if fi, err := driver.Stat(path); err != nil { switch err := err.(type) { case storagedriver.PathNotFoundError: // ignore, offset is zero default: return nil, err } } else { if fi.IsDir() { return nil, fmt.Errorf("cannot write to a directory") } fw.size = fi.Size() } buffered := bufferedFileWriter{ fileWriter: fw, } buffered.bw = bufio.NewWriterSize(&buffered.fileWriter, fileWriterBufferSize) return &buffered, nil } // wraps the fileWriter.Write method to buffer small writes func (bfw *bufferedFileWriter) Write(p []byte) (int, error) { return bfw.bw.Write(p) } // wraps fileWriter.Close to ensure the buffer is flushed // before we close the writer. func (bfw *bufferedFileWriter) Close() (err error) { if err = bfw.Flush(); err != nil { return err } err = bfw.fileWriter.Close() return err } // wraps fileWriter.Seek to ensure offset is handled // correctly in respect to pending data in the buffer func (bfw *bufferedFileWriter) Seek(offset int64, whence int) (int64, error) { if err := bfw.Flush(); err != nil { return 0, err } return bfw.fileWriter.Seek(offset, whence) } // wraps bufio.Writer.Flush to allow intermediate flushes // of the bufferedFileWriter func (bfw *bufferedFileWriter) Flush() error { return bfw.bw.Flush() } // Write writes the buffer p at the current write offset. func (fw *fileWriter) Write(p []byte) (n int, err error) { nn, err := fw.readFromAt(bytes.NewReader(p), -1) return int(nn), err } // WriteAt writes p at the specified offset. The underlying offset does not // change. func (fw *fileWriter) WriteAt(p []byte, offset int64) (n int, err error) { nn, err := fw.readFromAt(bytes.NewReader(p), offset) return int(nn), err } // ReadFrom reads reader r until io.EOF writing the contents at the current // offset. func (fw *fileWriter) ReadFrom(r io.Reader) (n int64, err error) { return fw.readFromAt(r, -1) } // Seek moves the write position do the requested offest based on the whence // argument, which can be os.SEEK_CUR, os.SEEK_END, or os.SEEK_SET. func (fw *fileWriter) Seek(offset int64, whence int) (int64, error) { if fw.err != nil { return 0, fw.err } var err error newOffset := fw.offset switch whence { case os.SEEK_CUR: newOffset += int64(offset) case os.SEEK_END: newOffset = fw.size + int64(offset) case os.SEEK_SET: newOffset = int64(offset) } if newOffset < 0 { err = fmt.Errorf("cannot seek to negative position") } else { // No problems, set the offset. fw.offset = newOffset } return fw.offset, err } // Close closes the fileWriter for writing. // Calling it once is valid and correct and it will // return a nil error. Calling it subsequent times will // detect that fw.err has been set and will return the error. func (fw *fileWriter) Close() error { if fw.err != nil { return fw.err } fw.err = fmt.Errorf("filewriter@%v: closed", fw.path) return nil } // readFromAt writes to fw from r at the specified offset. If offset is less // than zero, the value of fw.offset is used and updated after the operation. func (fw *fileWriter) readFromAt(r io.Reader, offset int64) (n int64, err error) { if fw.err != nil { return 0, fw.err } var updateOffset bool if offset < 0 { offset = fw.offset updateOffset = true } nn, err := fw.driver.WriteStream(fw.path, offset, r) if updateOffset { // We should forward the offset, whether or not there was an error. // Basically, we keep the filewriter in sync with the reader's head. If an // error is encountered, the whole thing should be retried but we proceed // from an expected offset, even if the data didn't make it to the // backend. fw.offset += nn if fw.offset > fw.size { fw.size = fw.offset } } return nn, err } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/revisionstore.go0000644000175000017500000000641612502424227027344 0ustar tianontianonpackage storage import ( "encoding/json" "github.com/Sirupsen/logrus" "github.com/docker/distribution" "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest" "github.com/docker/libtrust" ) // revisionStore supports storing and managing manifest revisions. type revisionStore struct { *repository } // exists returns true if the revision is available in the named repository. func (rs *revisionStore) exists(revision digest.Digest) (bool, error) { revpath, err := rs.pm.path(manifestRevisionPathSpec{ name: rs.Name(), revision: revision, }) if err != nil { return false, err } exists, err := exists(rs.driver, revpath) if err != nil { return false, err } return exists, nil } // get retrieves the manifest, keyed by revision digest. func (rs *revisionStore) get(revision digest.Digest) (*manifest.SignedManifest, error) { // Ensure that this revision is available in this repository. if exists, err := rs.exists(revision); err != nil { return nil, err } else if !exists { return nil, distribution.ErrUnknownManifestRevision{ Name: rs.Name(), Revision: revision, } } content, err := rs.blobStore.get(revision) if err != nil { return nil, err } // Fetch the signatures for the manifest signatures, err := rs.Signatures().Get(revision) if err != nil { return nil, err } jsig, err := libtrust.NewJSONSignature(content, signatures...) if err != nil { return nil, err } // Extract the pretty JWS raw, err := jsig.PrettySignature("signatures") if err != nil { return nil, err } var sm manifest.SignedManifest if err := json.Unmarshal(raw, &sm); err != nil { return nil, err } return &sm, nil } // put stores the manifest in the repository, if not already present. Any // updated signatures will be stored, as well. func (rs *revisionStore) put(sm *manifest.SignedManifest) (digest.Digest, error) { // Resolve the payload in the manifest. payload, err := sm.Payload() if err != nil { return "", err } // Digest and store the manifest payload in the blob store. revision, err := rs.blobStore.put(payload) if err != nil { logrus.Errorf("error putting payload into blobstore: %v", err) return "", err } // Link the revision into the repository. if err := rs.link(revision); err != nil { return "", err } // Grab each json signature and store them. signatures, err := sm.Signatures() if err != nil { return "", err } if err := rs.Signatures().Put(revision, signatures...); err != nil { return "", err } return revision, nil } // link links the revision into the repository. func (rs *revisionStore) link(revision digest.Digest) error { revisionPath, err := rs.pm.path(manifestRevisionLinkPathSpec{ name: rs.Name(), revision: revision, }) if err != nil { return err } if exists, err := exists(rs.driver, revisionPath); err != nil { return err } else if exists { // Revision has already been linked! return nil } return rs.blobStore.link(revisionPath, revision) } // delete removes the specified manifest revision from storage. func (rs *revisionStore) delete(revision digest.Digest) error { revisionPath, err := rs.pm.path(manifestRevisionPathSpec{ name: rs.Name(), revision: revision, }) if err != nil { return err } return rs.driver.Delete(revisionPath) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/layer_test.go0000644000175000017500000002327512502424227026606 0ustar tianontianonpackage storage import ( "bytes" "crypto/sha256" "fmt" "io" "io/ioutil" "os" "testing" "github.com/docker/distribution" "github.com/docker/distribution/digest" storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/distribution/registry/storage/driver/inmemory" "github.com/docker/distribution/testutil" "golang.org/x/net/context" ) // TestSimpleLayerUpload covers the layer upload process, exercising common // error paths that might be seen during an upload. func TestSimpleLayerUpload(t *testing.T) { randomDataReader, tarSumStr, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating random reader: %v", err) } dgst := digest.Digest(tarSumStr) if err != nil { t.Fatalf("error allocating upload store: %v", err) } ctx := context.Background() imageName := "foo/bar" driver := inmemory.New() registry := NewRegistryWithDriver(driver) repository, err := registry.Repository(ctx, imageName) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } ls := repository.Layers() h := sha256.New() rd := io.TeeReader(randomDataReader, h) layerUpload, err := ls.Upload() if err != nil { t.Fatalf("unexpected error starting layer upload: %s", err) } // Cancel the upload then restart it if err := layerUpload.Cancel(); err != nil { t.Fatalf("unexpected error during upload cancellation: %v", err) } // Do a resume, get unknown upload layerUpload, err = ls.Resume(layerUpload.UUID()) if err != distribution.ErrLayerUploadUnknown { t.Fatalf("unexpected error resuming upload, should be unkown: %v", err) } // Restart! layerUpload, err = ls.Upload() if err != nil { t.Fatalf("unexpected error starting layer upload: %s", err) } // Get the size of our random tarfile randomDataSize, err := seekerSize(randomDataReader) if err != nil { t.Fatalf("error getting seeker size of random data: %v", err) } nn, err := io.Copy(layerUpload, rd) if err != nil { t.Fatalf("unexpected error uploading layer data: %v", err) } if nn != randomDataSize { t.Fatalf("layer data write incomplete") } offset, err := layerUpload.Seek(0, os.SEEK_CUR) if err != nil { t.Fatalf("unexpected error seeking layer upload: %v", err) } if offset != nn { t.Fatalf("layerUpload not updated with correct offset: %v != %v", offset, nn) } layerUpload.Close() // Do a resume, for good fun layerUpload, err = ls.Resume(layerUpload.UUID()) if err != nil { t.Fatalf("unexpected error resuming upload: %v", err) } sha256Digest := digest.NewDigest("sha256", h) layer, err := layerUpload.Finish(dgst) if err != nil { t.Fatalf("unexpected error finishing layer upload: %v", err) } // After finishing an upload, it should no longer exist. if _, err := ls.Resume(layerUpload.UUID()); err != distribution.ErrLayerUploadUnknown { t.Fatalf("expected layer upload to be unknown, got %v", err) } // Test for existence. exists, err := ls.Exists(layer.Digest()) if err != nil { t.Fatalf("unexpected error checking for existence: %v", err) } if !exists { t.Fatalf("layer should now exist") } h.Reset() nn, err = io.Copy(h, layer) if err != nil { t.Fatalf("error reading layer: %v", err) } if nn != randomDataSize { t.Fatalf("incorrect read length") } if digest.NewDigest("sha256", h) != sha256Digest { t.Fatalf("unexpected digest from uploaded layer: %q != %q", digest.NewDigest("sha256", h), sha256Digest) } } // TestSimpleLayerRead just creates a simple layer file and ensures that basic // open, read, seek, read works. More specific edge cases should be covered in // other tests. func TestSimpleLayerRead(t *testing.T) { ctx := context.Background() imageName := "foo/bar" driver := inmemory.New() registry := NewRegistryWithDriver(driver) repository, err := registry.Repository(ctx, imageName) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } ls := repository.Layers() randomLayerReader, tarSumStr, err := testutil.CreateRandomTarFile() if err != nil { t.Fatalf("error creating random data: %v", err) } dgst := digest.Digest(tarSumStr) // Test for existence. exists, err := ls.Exists(dgst) if err != nil { t.Fatalf("unexpected error checking for existence: %v", err) } if exists { t.Fatalf("layer should not exist") } // Try to get the layer and make sure we get a not found error layer, err := ls.Fetch(dgst) if err == nil { t.Fatalf("error expected fetching unknown layer") } switch err.(type) { case distribution.ErrUnknownLayer: err = nil default: t.Fatalf("unexpected error fetching non-existent layer: %v", err) } randomLayerDigest, err := writeTestLayer(driver, ls.(*layerStore).repository.pm, imageName, dgst, randomLayerReader) if err != nil { t.Fatalf("unexpected error writing test layer: %v", err) } randomLayerSize, err := seekerSize(randomLayerReader) if err != nil { t.Fatalf("error getting seeker size for random layer: %v", err) } layer, err = ls.Fetch(dgst) if err != nil { t.Fatal(err) } defer layer.Close() // Now check the sha digest and ensure its the same h := sha256.New() nn, err := io.Copy(h, layer) if err != nil && err != io.EOF { t.Fatalf("unexpected error copying to hash: %v", err) } if nn != randomLayerSize { t.Fatalf("stored incorrect number of bytes in layer: %d != %d", nn, randomLayerSize) } sha256Digest := digest.NewDigest("sha256", h) if sha256Digest != randomLayerDigest { t.Fatalf("fetched digest does not match: %q != %q", sha256Digest, randomLayerDigest) } // Now seek back the layer, read the whole thing and check against randomLayerData offset, err := layer.Seek(0, os.SEEK_SET) if err != nil { t.Fatalf("error seeking layer: %v", err) } if offset != 0 { t.Fatalf("seek failed: expected 0 offset, got %d", offset) } p, err := ioutil.ReadAll(layer) if err != nil { t.Fatalf("error reading all of layer: %v", err) } if len(p) != int(randomLayerSize) { t.Fatalf("layer data read has different length: %v != %v", len(p), randomLayerSize) } // Reset the randomLayerReader and read back the buffer _, err = randomLayerReader.Seek(0, os.SEEK_SET) if err != nil { t.Fatalf("error resetting layer reader: %v", err) } randomLayerData, err := ioutil.ReadAll(randomLayerReader) if err != nil { t.Fatalf("random layer read failed: %v", err) } if !bytes.Equal(p, randomLayerData) { t.Fatalf("layer data not equal") } } // TestLayerUploadZeroLength uploads zero-length func TestLayerUploadZeroLength(t *testing.T) { ctx := context.Background() imageName := "foo/bar" driver := inmemory.New() registry := NewRegistryWithDriver(driver) repository, err := registry.Repository(ctx, imageName) if err != nil { t.Fatalf("unexpected error getting repo: %v", err) } ls := repository.Layers() upload, err := ls.Upload() if err != nil { t.Fatalf("unexpected error starting upload: %v", err) } io.Copy(upload, bytes.NewReader([]byte{})) dgst, err := digest.FromReader(bytes.NewReader([]byte{})) if err != nil { t.Fatalf("error getting zero digest: %v", err) } if dgst != digest.DigestSha256EmptyTar { // sanity check on zero digest t.Fatalf("digest not as expected: %v != %v", dgst, digest.DigestTarSumV1EmptyTar) } layer, err := upload.Finish(dgst) if err != nil { t.Fatalf("unexpected error finishing upload: %v", err) } if layer.Digest() != dgst { t.Fatalf("unexpected digest: %v != %v", layer.Digest(), dgst) } } // writeRandomLayer creates a random layer under name and tarSum using driver // and pathMapper. An io.ReadSeeker with the data is returned, along with the // sha256 hex digest. func writeRandomLayer(driver storagedriver.StorageDriver, pathMapper *pathMapper, name string) (rs io.ReadSeeker, tarSum digest.Digest, sha256digest digest.Digest, err error) { reader, tarSumStr, err := testutil.CreateRandomTarFile() if err != nil { return nil, "", "", err } tarSum = digest.Digest(tarSumStr) // Now, actually create the layer. randomLayerDigest, err := writeTestLayer(driver, pathMapper, name, tarSum, ioutil.NopCloser(reader)) if _, err := reader.Seek(0, os.SEEK_SET); err != nil { return nil, "", "", err } return reader, tarSum, randomLayerDigest, err } // seekerSize seeks to the end of seeker, checks the size and returns it to // the original state, returning the size. The state of the seeker should be // treated as unknown if an error is returned. func seekerSize(seeker io.ReadSeeker) (int64, error) { current, err := seeker.Seek(0, os.SEEK_CUR) if err != nil { return 0, err } end, err := seeker.Seek(0, os.SEEK_END) if err != nil { return 0, err } resumed, err := seeker.Seek(current, os.SEEK_SET) if err != nil { return 0, err } if resumed != current { return 0, fmt.Errorf("error returning seeker to original state, could not seek back to original location") } return end, nil } // createTestLayer creates a simple test layer in the provided driver under // tarsum dgst, returning the sha256 digest location. This is implemented // peicemeal and should probably be replaced by the uploader when it's ready. func writeTestLayer(driver storagedriver.StorageDriver, pathMapper *pathMapper, name string, dgst digest.Digest, content io.Reader) (digest.Digest, error) { h := sha256.New() rd := io.TeeReader(content, h) p, err := ioutil.ReadAll(rd) if err != nil { return "", nil } blobDigestSHA := digest.NewDigest("sha256", h) blobPath, err := pathMapper.path(blobDataPathSpec{ digest: dgst, }) if err := driver.PutContent(blobPath, p); err != nil { return "", err } if err != nil { return "", err } layerLinkPath, err := pathMapper.path(layerLinkPathSpec{ name: name, digest: dgst, }) if err != nil { return "", err } if err := driver.PutContent(layerLinkPath, []byte(dgst)); err != nil { return "", nil } return blobDigestSHA, err } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/blobstore.go0000644000175000017500000000777712502424227026437 0ustar tianontianonpackage storage import ( "fmt" ctxu "github.com/docker/distribution/context" "github.com/docker/distribution/digest" storagedriver "github.com/docker/distribution/registry/storage/driver" "golang.org/x/net/context" ) // TODO(stevvooe): Currently, the blobStore implementation used by the // manifest store. The layer store should be refactored to better leverage the // blobStore, reducing duplicated code. // blobStore implements a generalized blob store over a driver, supporting the // read side and link management. This object is intentionally a leaky // abstraction, providing utility methods that support creating and traversing // backend links. type blobStore struct { *registry ctx context.Context } // exists reports whether or not the path exists. If the driver returns error // other than storagedriver.PathNotFound, an error may be returned. func (bs *blobStore) exists(dgst digest.Digest) (bool, error) { path, err := bs.path(dgst) if err != nil { return false, err } ok, err := exists(bs.driver, path) if err != nil { return false, err } return ok, nil } // get retrieves the blob by digest, returning it a byte slice. This should // only be used for small objects. func (bs *blobStore) get(dgst digest.Digest) ([]byte, error) { bp, err := bs.path(dgst) if err != nil { return nil, err } return bs.driver.GetContent(bp) } // link links the path to the provided digest by writing the digest into the // target file. func (bs *blobStore) link(path string, dgst digest.Digest) error { if exists, err := bs.exists(dgst); err != nil { return err } else if !exists { return fmt.Errorf("cannot link non-existent blob") } // The contents of the "link" file are the exact string contents of the // digest, which is specified in that package. return bs.driver.PutContent(path, []byte(dgst)) } // linked reads the link at path and returns the content. func (bs *blobStore) linked(path string) ([]byte, error) { linked, err := bs.readlink(path) if err != nil { return nil, err } return bs.get(linked) } // readlink returns the linked digest at path. func (bs *blobStore) readlink(path string) (digest.Digest, error) { content, err := bs.driver.GetContent(path) if err != nil { return "", err } linked, err := digest.ParseDigest(string(content)) if err != nil { return "", err } if exists, err := bs.exists(linked); err != nil { return "", err } else if !exists { return "", fmt.Errorf("link %q invalid: blob %s does not exist", path, linked) } return linked, nil } // resolve reads the digest link at path and returns the blob store link. func (bs *blobStore) resolve(path string) (string, error) { dgst, err := bs.readlink(path) if err != nil { return "", err } return bs.path(dgst) } // put stores the content p in the blob store, calculating the digest. If the // content is already present, only the digest will be returned. This should // only be used for small objects, such as manifests. func (bs *blobStore) put(p []byte) (digest.Digest, error) { dgst, err := digest.FromBytes(p) if err != nil { ctxu.GetLogger(bs.ctx).Errorf("error digesting content: %v, %s", err, string(p)) return "", err } bp, err := bs.path(dgst) if err != nil { return "", err } // If the content already exists, just return the digest. if exists, err := bs.exists(dgst); err != nil { return "", err } else if exists { return dgst, nil } return dgst, bs.driver.PutContent(bp, p) } // path returns the canonical path for the blob identified by digest. The blob // may or may not exist. func (bs *blobStore) path(dgst digest.Digest) (string, error) { bp, err := bs.pm.path(blobDataPathSpec{ digest: dgst, }) if err != nil { return "", err } return bp, nil } // exists provides a utility method to test whether or not func exists(driver storagedriver.StorageDriver, path string) (bool, error) { if _, err := driver.Stat(path); err != nil { switch err := err.(type) { case storagedriver.PathNotFoundError: return false, nil default: return false, err } } return true, nil } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/signaturestore.go0000644000175000017500000000315412502424227027503 0ustar tianontianonpackage storage import ( "path" "github.com/docker/distribution" "github.com/docker/distribution/digest" ) type signatureStore struct { *repository } var _ distribution.SignatureService = &signatureStore{} func (s *signatureStore) Get(dgst digest.Digest) ([][]byte, error) { signaturesPath, err := s.pm.path(manifestSignaturesPathSpec{ name: s.Name(), revision: dgst, }) if err != nil { return nil, err } // Need to append signature digest algorithm to path to get all items. // Perhaps, this should be in the pathMapper but it feels awkward. This // can be eliminated by implementing listAll on drivers. signaturesPath = path.Join(signaturesPath, "sha256") signaturePaths, err := s.driver.List(signaturesPath) if err != nil { return nil, err } var signatures [][]byte for _, sigPath := range signaturePaths { // Append the link portion sigPath = path.Join(sigPath, "link") // TODO(stevvooe): These fetches should be parallelized for performance. p, err := s.blobStore.linked(sigPath) if err != nil { return nil, err } signatures = append(signatures, p) } return signatures, nil } func (s *signatureStore) Put(dgst digest.Digest, signatures ...[]byte) error { for _, signature := range signatures { signatureDigest, err := s.blobStore.put(signature) if err != nil { return err } signaturePath, err := s.pm.path(manifestSignatureLinkPathSpec{ name: s.Name(), revision: dgst, signature: signatureDigest, }) if err != nil { return err } if err := s.blobStore.link(signaturePath, signatureDigest); err != nil { return err } } return nil } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/filewriter_test.go0000644000175000017500000001416212502424227027641 0ustar tianontianonpackage storage import ( "bytes" "crypto/rand" "io" "os" "testing" "github.com/docker/distribution/digest" storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/distribution/registry/storage/driver/inmemory" ) // TestSimpleWrite takes the fileWriter through common write operations // ensuring data integrity. func TestSimpleWrite(t *testing.T) { content := make([]byte, 1<<20) n, err := rand.Read(content) if err != nil { t.Fatalf("unexpected error building random data: %v", err) } if n != len(content) { t.Fatalf("random read did't fill buffer") } dgst, err := digest.FromReader(bytes.NewReader(content)) if err != nil { t.Fatalf("unexpected error digesting random content: %v", err) } driver := inmemory.New() path := "/random" fw, err := newFileWriter(driver, path) if err != nil { t.Fatalf("unexpected error creating fileWriter: %v", err) } defer fw.Close() n, err = fw.Write(content) if err != nil { t.Fatalf("unexpected error writing content: %v", err) } fw.Flush() if n != len(content) { t.Fatalf("unexpected write length: %d != %d", n, len(content)) } fr, err := newFileReader(driver, path) if err != nil { t.Fatalf("unexpected error creating fileReader: %v", err) } defer fr.Close() verifier, err := digest.NewDigestVerifier(dgst) if err != nil { t.Fatalf("unexpected error getting digest verifier: %s", err) } io.Copy(verifier, fr) if !verifier.Verified() { t.Fatalf("unable to verify write data") } // Check the seek position is equal to the content length end, err := fw.Seek(0, os.SEEK_END) if err != nil { t.Fatalf("unexpected error seeking: %v", err) } if end != int64(len(content)) { t.Fatalf("write did not advance offset: %d != %d", end, len(content)) } // Double the content, but use the WriteAt method doubled := append(content, content...) doubledgst, err := digest.FromReader(bytes.NewReader(doubled)) if err != nil { t.Fatalf("unexpected error digesting doubled content: %v", err) } n, err = fw.WriteAt(content, end) if err != nil { t.Fatalf("unexpected error writing content at %d: %v", end, err) } if n != len(content) { t.Fatalf("writeat was short: %d != %d", n, len(content)) } fr, err = newFileReader(driver, path) if err != nil { t.Fatalf("unexpected error creating fileReader: %v", err) } defer fr.Close() verifier, err = digest.NewDigestVerifier(doubledgst) if err != nil { t.Fatalf("unexpected error getting digest verifier: %s", err) } io.Copy(verifier, fr) if !verifier.Verified() { t.Fatalf("unable to verify write data") } // Check that WriteAt didn't update the offset. end, err = fw.Seek(0, os.SEEK_END) if err != nil { t.Fatalf("unexpected error seeking: %v", err) } if end != int64(len(content)) { t.Fatalf("write did not advance offset: %d != %d", end, len(content)) } // Now, we copy from one path to another, running the data through the // fileReader to fileWriter, rather than the driver.Move command to ensure // everything is working correctly. fr, err = newFileReader(driver, path) if err != nil { t.Fatalf("unexpected error creating fileReader: %v", err) } defer fr.Close() fw, err = newFileWriter(driver, "/copied") if err != nil { t.Fatalf("unexpected error creating fileWriter: %v", err) } defer fw.Close() nn, err := io.Copy(fw, fr) if err != nil { t.Fatalf("unexpected error copying data: %v", err) } if nn != int64(len(doubled)) { t.Fatalf("unexpected copy length: %d != %d", nn, len(doubled)) } fr, err = newFileReader(driver, "/copied") if err != nil { t.Fatalf("unexpected error creating fileReader: %v", err) } defer fr.Close() verifier, err = digest.NewDigestVerifier(doubledgst) if err != nil { t.Fatalf("unexpected error getting digest verifier: %s", err) } io.Copy(verifier, fr) if !verifier.Verified() { t.Fatalf("unable to verify write data") } } func TestBufferedFileWriter(t *testing.T) { writer, err := newFileWriter(inmemory.New(), "/random") if err != nil { t.Fatalf("Failed to initialize bufferedFileWriter: %v", err.Error()) } // write one byte and ensure the offset hasn't been incremented. // offset will only get incremented when the buffer gets flushed short := []byte{byte(1)} writer.Write(short) if writer.offset > 0 { t.Fatalf("WriteStream called prematurely") } // write enough data to cause the buffer to flush and confirm // the offset has been incremented long := make([]byte, fileWriterBufferSize) _, err = rand.Read(long) if err != nil { t.Fatalf("unexpected error building random data: %v", err) } for i := range long { long[i] = byte(i) } writer.Write(long) writer.Close() if writer.offset != (fileWriterBufferSize + 1) { t.Fatalf("WriteStream not called when buffer capacity reached") } } func BenchmarkFileWriter(b *testing.B) { b.StopTimer() // not sure how long setup above will take for i := 0; i < b.N; i++ { // Start basic fileWriter initialization fw := fileWriter{ driver: inmemory.New(), path: "/random", } if fi, err := fw.driver.Stat(fw.path); err != nil { switch err := err.(type) { case storagedriver.PathNotFoundError: // ignore, offset is zero default: b.Fatalf("Failed to initialize fileWriter: %v", err.Error()) } } else { if fi.IsDir() { b.Fatalf("Cannot write to a directory") } fw.size = fi.Size() } randomBytes := make([]byte, 1<<20) _, err := rand.Read(randomBytes) if err != nil { b.Fatalf("unexpected error building random data: %v", err) } // End basic file writer initialization b.StartTimer() for j := 0; j < 100; j++ { fw.Write(randomBytes) } b.StopTimer() } } func BenchmarkBufferedFileWriter(b *testing.B) { b.StopTimer() // not sure how long setup above will take for i := 0; i < b.N; i++ { bfw, err := newFileWriter(inmemory.New(), "/random") if err != nil { b.Fatalf("Failed to initialize bufferedFileWriter: %v", err.Error()) } randomBytes := make([]byte, 1<<20) _, err = rand.Read(randomBytes) if err != nil { b.Fatalf("unexpected error building random data: %v", err) } b.StartTimer() for j := 0; j < 100; j++ { bfw.Write(randomBytes) } b.StopTimer() } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/paths.go0000644000175000017500000003517012502424227025547 0ustar tianontianonpackage storage import ( "fmt" "path" "strings" "github.com/docker/distribution/digest" ) const storagePathVersion = "v2" // pathMapper maps paths based on "object names" and their ids. The "object // names" mapped by pathMapper are internal to the storage system. // // The path layout in the storage backend is roughly as follows: // // /v2 // -> repositories/ // ->/ // -> _manifests/ // revisions // -> // -> link // -> signatures // //link // tags/ // -> current/link // -> index // -> //link // -> _layers/ // // -> _uploads/ // data // startedat // -> blob/ // // // The storage backend layout is broken up into a content- addressable blob // store and repositories. The content-addressable blob store holds most data // throughout the backend, keyed by algorithm and digests of the underlying // content. Access to the blob store is controled through links from the // repository to blobstore. // // A repository is made up of layers, manifests and tags. The layers component // is just a directory of layers which are "linked" into a repository. A layer // can only be accessed through a qualified repository name if it is linked in // the repository. Uploads of layers are managed in the uploads directory, // which is key by upload uuid. When all data for an upload is received, the // data is moved into the blob store and the upload directory is deleted. // Abandoned uploads can be garbage collected by reading the startedat file // and removing uploads that have been active for longer than a certain time. // // The third component of the repository directory is the manifests store, // which is made up of a revision store and tag store. Manifests are stored in // the blob store and linked into the revision store. Signatures are separated // from the manifest payload data and linked into the blob store, as well. // While the registry can save all revisions of a manifest, no relationship is // implied as to the ordering of changes to a manifest. The tag store provides // support for name, tag lookups of manifests, using "current/link" under a // named tag directory. An index is maintained to support deletions of all // revisions of a given manifest tag. // // We cover the path formats implemented by this path mapper below. // // Manifests: // // manifestRevisionPathSpec: /v2/repositories//_manifests/revisions/// // manifestRevisionLinkPathSpec: /v2/repositories//_manifests/revisions///link // manifestSignaturesPathSpec: /v2/repositories//_manifests/revisions///signatures/ // manifestSignatureLinkPathSpec: /v2/repositories//_manifests/revisions///signatures///link // // Tags: // // manifestTagsPathSpec: /v2/repositories//_manifests/tags/ // manifestTagPathSpec: /v2/repositories//_manifests/tags// // manifestTagCurrentPathSpec: /v2/repositories//_manifests/tags//current/link // manifestTagIndexPathSpec: /v2/repositories//_manifests/tags//index/ // manifestTagIndexEntryPathSpec: /v2/repositories//_manifests/tags//index/// // manifestTagIndexEntryLinkPathSpec: /v2/repositories//_manifests/tags//index///link // // Layers: // // layerLinkPathSpec: /v2/repositories//_layers/tarsum////link // // Uploads: // // uploadDataPathSpec: /v2/repositories//_uploads//data // uploadStartedAtPathSpec: /v2/repositories//_uploads//startedat // // Blob Store: // // blobPathSpec: /v2/blobs/// // blobDataPathSpec: /v2/blobs////data // // For more information on the semantic meaning of each path and their // contents, please see the path spec documentation. type pathMapper struct { root string version string // should be a constant? } var defaultPathMapper = &pathMapper{ root: "/docker/registry/", version: storagePathVersion, } // path returns the path identified by spec. func (pm *pathMapper) path(spec pathSpec) (string, error) { // Switch on the path object type and return the appropriate path. At // first glance, one may wonder why we don't use an interface to // accomplish this. By keep the formatting separate from the pathSpec, we // keep separate the path generation componentized. These specs could be // passed to a completely different mapper implementation and generate a // different set of paths. // // For example, imagine migrating from one backend to the other: one could // build a filesystem walker that converts a string path in one version, // to an intermediate path object, than can be consumed and mapped by the // other version. rootPrefix := []string{pm.root, pm.version} repoPrefix := append(rootPrefix, "repositories") switch v := spec.(type) { case manifestRevisionPathSpec: components, err := digestPathComponents(v.revision, false) if err != nil { return "", err } return path.Join(append(append(repoPrefix, v.name, "_manifests", "revisions"), components...)...), nil case manifestRevisionLinkPathSpec: root, err := pm.path(manifestRevisionPathSpec{ name: v.name, revision: v.revision, }) if err != nil { return "", err } return path.Join(root, "link"), nil case manifestSignaturesPathSpec: root, err := pm.path(manifestRevisionPathSpec{ name: v.name, revision: v.revision, }) if err != nil { return "", err } return path.Join(root, "signatures"), nil case manifestSignatureLinkPathSpec: root, err := pm.path(manifestSignaturesPathSpec{ name: v.name, revision: v.revision, }) if err != nil { return "", err } signatureComponents, err := digestPathComponents(v.signature, false) if err != nil { return "", err } return path.Join(root, path.Join(append(signatureComponents, "link")...)), nil case manifestTagsPathSpec: return path.Join(append(repoPrefix, v.name, "_manifests", "tags")...), nil case manifestTagPathSpec: root, err := pm.path(manifestTagsPathSpec{ name: v.name, }) if err != nil { return "", err } return path.Join(root, v.tag), nil case manifestTagCurrentPathSpec: root, err := pm.path(manifestTagPathSpec{ name: v.name, tag: v.tag, }) if err != nil { return "", err } return path.Join(root, "current", "link"), nil case manifestTagIndexPathSpec: root, err := pm.path(manifestTagPathSpec{ name: v.name, tag: v.tag, }) if err != nil { return "", err } return path.Join(root, "index"), nil case manifestTagIndexEntryLinkPathSpec: root, err := pm.path(manifestTagIndexEntryPathSpec{ name: v.name, tag: v.tag, revision: v.revision, }) if err != nil { return "", err } return path.Join(root, "link"), nil case manifestTagIndexEntryPathSpec: root, err := pm.path(manifestTagIndexPathSpec{ name: v.name, tag: v.tag, }) if err != nil { return "", err } components, err := digestPathComponents(v.revision, false) if err != nil { return "", err } return path.Join(root, path.Join(components...)), nil case layerLinkPathSpec: components, err := digestPathComponents(v.digest, false) if err != nil { return "", err } layerLinkPathComponents := append(repoPrefix, v.name, "_layers") return path.Join(path.Join(append(layerLinkPathComponents, components...)...), "link"), nil case blobDataPathSpec: components, err := digestPathComponents(v.digest, true) if err != nil { return "", err } components = append(components, "data") blobPathPrefix := append(rootPrefix, "blobs") return path.Join(append(blobPathPrefix, components...)...), nil case uploadDataPathSpec: return path.Join(append(repoPrefix, v.name, "_uploads", v.uuid, "data")...), nil case uploadStartedAtPathSpec: return path.Join(append(repoPrefix, v.name, "_uploads", v.uuid, "startedat")...), nil default: // TODO(sday): This is an internal error. Ensure it doesn't escape (panic?). return "", fmt.Errorf("unknown path spec: %#v", v) } } // pathSpec is a type to mark structs as path specs. There is no // implementation because we'd like to keep the specs and the mappers // decoupled. type pathSpec interface { pathSpec() } // manifestRevisionPathSpec describes the components of the directory path for // a manifest revision. type manifestRevisionPathSpec struct { name string revision digest.Digest } func (manifestRevisionPathSpec) pathSpec() {} // manifestRevisionLinkPathSpec describes the path components required to look // up the data link for a revision of a manifest. If this file is not present, // the manifest blob is not available in the given repo. The contents of this // file should just be the digest. type manifestRevisionLinkPathSpec struct { name string revision digest.Digest } func (manifestRevisionLinkPathSpec) pathSpec() {} // manifestSignaturesPathSpec decribes the path components for the directory // containing all the signatures for the target blob. Entries are named with // the underlying key id. type manifestSignaturesPathSpec struct { name string revision digest.Digest } func (manifestSignaturesPathSpec) pathSpec() {} // manifestSignatureLinkPathSpec decribes the path components used to look up // a signature file by the hash of its blob. type manifestSignatureLinkPathSpec struct { name string revision digest.Digest signature digest.Digest } func (manifestSignatureLinkPathSpec) pathSpec() {} // manifestTagsPathSpec describes the path elements required to point to the // manifest tags directory. type manifestTagsPathSpec struct { name string } func (manifestTagsPathSpec) pathSpec() {} // manifestTagPathSpec describes the path elements required to point to the // manifest tag links files under a repository. These contain a blob id that // can be used to look up the data and signatures. type manifestTagPathSpec struct { name string tag string } func (manifestTagPathSpec) pathSpec() {} // manifestTagCurrentPathSpec describes the link to the current revision for a // given tag. type manifestTagCurrentPathSpec struct { name string tag string } func (manifestTagCurrentPathSpec) pathSpec() {} // manifestTagCurrentPathSpec describes the link to the index of revisions // with the given tag. type manifestTagIndexPathSpec struct { name string tag string } func (manifestTagIndexPathSpec) pathSpec() {} // manifestTagIndexEntryPathSpec contains the entries of the index by revision. type manifestTagIndexEntryPathSpec struct { name string tag string revision digest.Digest } func (manifestTagIndexEntryPathSpec) pathSpec() {} // manifestTagIndexEntryLinkPathSpec describes the link to a revisions of a // manifest with given tag within the index. type manifestTagIndexEntryLinkPathSpec struct { name string tag string revision digest.Digest } func (manifestTagIndexEntryLinkPathSpec) pathSpec() {} // layerLink specifies a path for a layer link, which is a file with a blob // id. The layer link will contain a content addressable blob id reference // into the blob store. The format of the contents is as follows: // // : // // The following example of the file contents is more illustrative: // // sha256:96443a84ce518ac22acb2e985eda402b58ac19ce6f91980bde63726a79d80b36 // // This says indicates that there is a blob with the id/digest, calculated via // sha256 that can be fetched from the blob store. type layerLinkPathSpec struct { name string digest digest.Digest } func (layerLinkPathSpec) pathSpec() {} // blobAlgorithmReplacer does some very simple path sanitization for user // input. Mostly, this is to provide some heirachry for tarsum digests. Paths // should be "safe" before getting this far due to strict digest requirements // but we can add further path conversion here, if needed. var blobAlgorithmReplacer = strings.NewReplacer( "+", "/", ".", "/", ";", "/", ) // // blobPathSpec contains the path for the registry global blob store. // type blobPathSpec struct { // digest digest.Digest // } // func (blobPathSpec) pathSpec() {} // blobDataPathSpec contains the path for the registry global blob store. For // now, this contains layer data, exclusively. type blobDataPathSpec struct { digest digest.Digest } func (blobDataPathSpec) pathSpec() {} // uploadDataPathSpec defines the path parameters of the data file for // uploads. type uploadDataPathSpec struct { name string uuid string } func (uploadDataPathSpec) pathSpec() {} // uploadDataPathSpec defines the path parameters for the file that stores the // start time of an uploads. If it is missing, the upload is considered // unknown. Admittedly, the presence of this file is an ugly hack to make sure // we have a way to cleanup old or stalled uploads that doesn't rely on driver // FileInfo behavior. If we come up with a more clever way to do this, we // should remove this file immediately and rely on the startetAt field from // the client to enforce time out policies. type uploadStartedAtPathSpec struct { name string uuid string } func (uploadStartedAtPathSpec) pathSpec() {} // digestPathComponents provides a consistent path breakdown for a given // digest. For a generic digest, it will be as follows: // // / // // Most importantly, for tarsum, the layout looks like this: // // tarsum/// // // If multilevel is true, the first two bytes of the digest will separate // groups of digest folder. It will be as follows: // // // // func digestPathComponents(dgst digest.Digest, multilevel bool) ([]string, error) { if err := dgst.Validate(); err != nil { return nil, err } algorithm := blobAlgorithmReplacer.Replace(dgst.Algorithm()) hex := dgst.Hex() prefix := []string{algorithm} var suffix []string if multilevel { suffix = append(suffix, hex[:2]) } suffix = append(suffix, hex) if tsi, err := digest.ParseTarSum(dgst.String()); err == nil { // We have a tarsum! version := tsi.Version if version == "" { version = "v0" } prefix = []string{ "tarsum", version, tsi.Algorithm, } } return append(prefix, suffix...), nil } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/manifeststore.go0000644000175000017500000000720612502424227027312 0ustar tianontianonpackage storage import ( "fmt" "github.com/docker/distribution" ctxu "github.com/docker/distribution/context" "github.com/docker/distribution/digest" "github.com/docker/distribution/manifest" "github.com/docker/libtrust" ) type manifestStore struct { repository *repository revisionStore *revisionStore tagStore *tagStore } var _ distribution.ManifestService = &manifestStore{} func (ms *manifestStore) Exists(dgst digest.Digest) (bool, error) { ctxu.GetLogger(ms.repository.ctx).Debug("(*manifestStore).Exists") return ms.revisionStore.exists(dgst) } func (ms *manifestStore) Get(dgst digest.Digest) (*manifest.SignedManifest, error) { ctxu.GetLogger(ms.repository.ctx).Debug("(*manifestStore).Get") return ms.revisionStore.get(dgst) } func (ms *manifestStore) Put(manifest *manifest.SignedManifest) error { ctxu.GetLogger(ms.repository.ctx).Debug("(*manifestStore).Put") // TODO(stevvooe): Add check here to see if the revision is already // present in the repository. If it is, we should merge the signatures, do // a shallow verify (or a full one, doesn't matter) and return an error // indicating what happened. // Verify the manifest. if err := ms.verifyManifest(manifest); err != nil { return err } // Store the revision of the manifest revision, err := ms.revisionStore.put(manifest) if err != nil { return err } // Now, tag the manifest return ms.tagStore.tag(manifest.Tag, revision) } // Delete removes the revision of the specified manfiest. func (ms *manifestStore) Delete(dgst digest.Digest) error { ctxu.GetLogger(ms.repository.ctx).Debug("(*manifestStore).Delete - unsupported") return fmt.Errorf("deletion of manifests not supported") } func (ms *manifestStore) Tags() ([]string, error) { ctxu.GetLogger(ms.repository.ctx).Debug("(*manifestStore).Tags") return ms.tagStore.tags() } func (ms *manifestStore) ExistsByTag(tag string) (bool, error) { ctxu.GetLogger(ms.repository.ctx).Debug("(*manifestStore).ExistsByTag") return ms.tagStore.exists(tag) } func (ms *manifestStore) GetByTag(tag string) (*manifest.SignedManifest, error) { ctxu.GetLogger(ms.repository.ctx).Debug("(*manifestStore).GetByTag") dgst, err := ms.tagStore.resolve(tag) if err != nil { return nil, err } return ms.revisionStore.get(dgst) } // verifyManifest ensures that the manifest content is valid from the // perspective of the registry. It ensures that the signature is valid for the // enclosed payload. As a policy, the registry only tries to store valid // content, leaving trust policies of that content up to consumers. func (ms *manifestStore) verifyManifest(mnfst *manifest.SignedManifest) error { var errs distribution.ErrManifestVerification if mnfst.Name != ms.repository.Name() { // TODO(stevvooe): This needs to be an exported error errs = append(errs, fmt.Errorf("repository name does not match manifest name")) } if _, err := manifest.Verify(mnfst); err != nil { switch err { case libtrust.ErrMissingSignatureKey, libtrust.ErrInvalidJSONContent, libtrust.ErrMissingSignatureKey: errs = append(errs, distribution.ErrManifestUnverified{}) default: if err.Error() == "invalid signature" { // TODO(stevvooe): This should be exported by libtrust errs = append(errs, distribution.ErrManifestUnverified{}) } else { errs = append(errs, err) } } } for _, fsLayer := range mnfst.FSLayers { exists, err := ms.repository.Layers().Exists(fsLayer.BlobSum) if err != nil { errs = append(errs, err) } if !exists { errs = append(errs, distribution.ErrUnknownLayer{FSLayer: fsLayer}) } } if len(errs) != 0 { // TODO(stevvooe): These need to be recoverable by a caller. return errs } return nil } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/0000755000175000017500000000000012502424227025366 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/factory/0000755000175000017500000000000012502424227027035 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/factory/factory.go0000644000175000017500000000523412502424227031037 0ustar tianontianonpackage factory import ( "fmt" storagedriver "github.com/docker/distribution/registry/storage/driver" ) // driverFactories stores an internal mapping between storage driver names and their respective // factories var driverFactories = make(map[string]StorageDriverFactory) // StorageDriverFactory is a factory interface for creating storagedriver.StorageDriver interfaces // Storage drivers should call Register() with a factory to make the driver available by name type StorageDriverFactory interface { // Create returns a new storagedriver.StorageDriver with the given parameters // Parameters will vary by driver and may be ignored // Each parameter key must only consist of lowercase letters and numbers Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error) } // Register makes a storage driver available by the provided name. // If Register is called twice with the same name or if driver factory is nil, it panics. func Register(name string, factory StorageDriverFactory) { if factory == nil { panic("Must not provide nil StorageDriverFactory") } _, registered := driverFactories[name] if registered { panic(fmt.Sprintf("StorageDriverFactory named %s already registered", name)) } driverFactories[name] = factory } // Create a new storagedriver.StorageDriver with the given name and parameters // To run in-process, the StorageDriverFactory must first be registered with the given name // If no in-process drivers are found with the given name, this attempts to create an IPC driver // If no in-process or external drivers are found, an InvalidStorageDriverError is returned func Create(name string, parameters map[string]interface{}) (storagedriver.StorageDriver, error) { driverFactory, ok := driverFactories[name] if !ok { return nil, InvalidStorageDriverError{name} // NOTE(stevvooe): We are disabling storagedriver ipc for now, as the // server and client need to be updated for the changed API calls and // there were some problems libchan hanging. We'll phase this // functionality back in over the next few weeks. // No registered StorageDriverFactory found, try ipc // driverClient, err := ipc.NewDriverClient(name, parameters) // if err != nil { // return nil, InvalidStorageDriverError{name} // } // err = driverClient.Start() // if err != nil { // return nil, err // } // return driverClient, nil } return driverFactory.Create(parameters) } // InvalidStorageDriverError records an attempt to construct an unregistered storage driver type InvalidStorageDriverError struct { Name string } func (err InvalidStorageDriverError) Error() string { return fmt.Sprintf("StorageDriver not registered: %s", err.Name) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/filesystem/0000755000175000017500000000000012502424227027552 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/filesystem/driver.go0000644000175000017500000001607412502424227031404 0ustar tianontianonpackage filesystem import ( "bytes" "fmt" "io" "io/ioutil" "os" "path" "time" storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/distribution/registry/storage/driver/base" "github.com/docker/distribution/registry/storage/driver/factory" ) const driverName = "filesystem" const defaultRootDirectory = "/tmp/registry/storage" func init() { factory.Register(driverName, &filesystemDriverFactory{}) } // filesystemDriverFactory implements the factory.StorageDriverFactory interface type filesystemDriverFactory struct{} func (factory *filesystemDriverFactory) Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error) { return FromParameters(parameters), nil } type driver struct { rootDirectory string } type baseEmbed struct { base.Base } // Driver is a storagedriver.StorageDriver implementation backed by a local // filesystem. All provided paths will be subpaths of the RootDirectory. type Driver struct { baseEmbed } // FromParameters constructs a new Driver with a given parameters map // Optional Parameters: // - rootdirectory func FromParameters(parameters map[string]interface{}) *Driver { var rootDirectory = defaultRootDirectory if parameters != nil { rootDir, ok := parameters["rootdirectory"] if ok { rootDirectory = fmt.Sprint(rootDir) } } return New(rootDirectory) } // New constructs a new Driver with a given rootDirectory func New(rootDirectory string) *Driver { return &Driver{ baseEmbed: baseEmbed{ Base: base.Base{ StorageDriver: &driver{ rootDirectory: rootDirectory, }, }, }, } } // Implement the storagedriver.StorageDriver interface // GetContent retrieves the content stored at "path" as a []byte. func (d *driver) GetContent(path string) ([]byte, error) { rc, err := d.ReadStream(path, 0) if err != nil { return nil, err } defer rc.Close() p, err := ioutil.ReadAll(rc) if err != nil { return nil, err } return p, nil } // PutContent stores the []byte content at a location designated by "path". func (d *driver) PutContent(subPath string, contents []byte) error { if _, err := d.WriteStream(subPath, 0, bytes.NewReader(contents)); err != nil { return err } return os.Truncate(d.fullPath(subPath), int64(len(contents))) } // ReadStream retrieves an io.ReadCloser for the content stored at "path" with a // given byte offset. func (d *driver) ReadStream(path string, offset int64) (io.ReadCloser, error) { file, err := os.OpenFile(d.fullPath(path), os.O_RDONLY, 0644) if err != nil { if os.IsNotExist(err) { return nil, storagedriver.PathNotFoundError{Path: path} } return nil, err } seekPos, err := file.Seek(int64(offset), os.SEEK_SET) if err != nil { file.Close() return nil, err } else if seekPos < int64(offset) { file.Close() return nil, storagedriver.InvalidOffsetError{Path: path, Offset: offset} } return file, nil } // WriteStream stores the contents of the provided io.Reader at a location // designated by the given path. func (d *driver) WriteStream(subPath string, offset int64, reader io.Reader) (nn int64, err error) { // TODO(stevvooe): This needs to be a requirement. // if !path.IsAbs(subPath) { // return fmt.Errorf("absolute path required: %q", subPath) // } fullPath := d.fullPath(subPath) parentDir := path.Dir(fullPath) if err := os.MkdirAll(parentDir, 0755); err != nil { return 0, err } fp, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE, 0644) if err != nil { // TODO(stevvooe): A few missing conditions in storage driver: // 1. What if the path is already a directory? // 2. Should number 1 be exposed explicitly in storagedriver? // 2. Can this path not exist, even if we create above? return 0, err } defer fp.Close() nn, err = fp.Seek(offset, os.SEEK_SET) if err != nil { return 0, err } if nn != offset { return 0, fmt.Errorf("bad seek to %v, expected %v in fp=%v", offset, nn, fp) } return io.Copy(fp, reader) } // Stat retrieves the FileInfo for the given path, including the current size // in bytes and the creation time. func (d *driver) Stat(subPath string) (storagedriver.FileInfo, error) { fullPath := d.fullPath(subPath) fi, err := os.Stat(fullPath) if err != nil { if os.IsNotExist(err) { return nil, storagedriver.PathNotFoundError{Path: subPath} } return nil, err } return fileInfo{ path: subPath, FileInfo: fi, }, nil } // List returns a list of the objects that are direct descendants of the given // path. func (d *driver) List(subPath string) ([]string, error) { if subPath[len(subPath)-1] != '/' { subPath += "/" } fullPath := d.fullPath(subPath) dir, err := os.Open(fullPath) if err != nil { if os.IsNotExist(err) { return nil, storagedriver.PathNotFoundError{Path: subPath} } return nil, err } defer dir.Close() fileNames, err := dir.Readdirnames(0) if err != nil { return nil, err } keys := make([]string, 0, len(fileNames)) for _, fileName := range fileNames { keys = append(keys, path.Join(subPath, fileName)) } return keys, nil } // Move moves an object stored at sourcePath to destPath, removing the original // object. func (d *driver) Move(sourcePath string, destPath string) error { source := d.fullPath(sourcePath) dest := d.fullPath(destPath) if _, err := os.Stat(source); os.IsNotExist(err) { return storagedriver.PathNotFoundError{Path: sourcePath} } if err := os.MkdirAll(path.Dir(dest), 0755); err != nil { return err } err := os.Rename(source, dest) return err } // Delete recursively deletes all objects stored at "path" and its subpaths. func (d *driver) Delete(subPath string) error { fullPath := d.fullPath(subPath) _, err := os.Stat(fullPath) if err != nil && !os.IsNotExist(err) { return err } else if err != nil { return storagedriver.PathNotFoundError{Path: subPath} } err = os.RemoveAll(fullPath) return err } // URLFor returns a URL which may be used to retrieve the content stored at the given path. // May return an UnsupportedMethodErr in certain StorageDriver implementations. func (d *driver) URLFor(path string, options map[string]interface{}) (string, error) { return "", storagedriver.ErrUnsupportedMethod } // fullPath returns the absolute path of a key within the Driver's storage. func (d *driver) fullPath(subPath string) string { return path.Join(d.rootDirectory, subPath) } type fileInfo struct { os.FileInfo path string } var _ storagedriver.FileInfo = fileInfo{} // Path provides the full path of the target of this file info. func (fi fileInfo) Path() string { return fi.path } // Size returns current length in bytes of the file. The return value can // be used to write to the end of the file at path. The value is // meaningless if IsDir returns true. func (fi fileInfo) Size() int64 { if fi.IsDir() { return 0 } return fi.FileInfo.Size() } // ModTime returns the modification time for the file. For backends that // don't have a modification time, the creation time should be returned. func (fi fileInfo) ModTime() time.Time { return fi.FileInfo.ModTime() } // IsDir returns true if the path is a directory. func (fi fileInfo) IsDir() bool { return fi.FileInfo.IsDir() } ././@LongLink0000644000000000000000000000015000000000000011577 Lustar rootrootdistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/filesystem/driver_test.godistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/filesystem/driver_test0000644000175000017500000000136012502424227032027 0ustar tianontianonpackage filesystem import ( "io/ioutil" "os" "testing" storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/distribution/registry/storage/driver/testsuites" . "gopkg.in/check.v1" ) // Hook up gocheck into the "go test" runner. func Test(t *testing.T) { TestingT(t) } func init() { root, err := ioutil.TempDir("", "driver-") if err != nil { panic(err) } defer os.Remove(root) testsuites.RegisterInProcessSuite(func() (storagedriver.StorageDriver, error) { return New(root), nil }, testsuites.NeverSkip) // BUG(stevvooe): IPC is broken so we're disabling for now. Will revisit later. // testsuites.RegisterIPCSuite(driverName, map[string]string{"rootdirectory": root}, testsuites.NeverSkip) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/testsuites/0000755000175000017500000000000012502424227027602 5ustar tianontianon././@LongLink0000644000000000000000000000014700000000000011605 Lustar rootrootdistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/testsuites/testsuites.godistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/testsuites/testsuites.0000644000175000017500000011054112502424227032021 0ustar tianontianonpackage testsuites import ( "bytes" "crypto/sha1" "io" "io/ioutil" "math/rand" "net/http" "os" "path" "sort" "sync" "testing" "time" storagedriver "github.com/docker/distribution/registry/storage/driver" "gopkg.in/check.v1" ) // Test hooks up gocheck into the "go test" runner. func Test(t *testing.T) { check.TestingT(t) } // RegisterInProcessSuite registers an in-process storage driver test suite with // the go test runner. func RegisterInProcessSuite(driverConstructor DriverConstructor, skipCheck SkipCheck) { check.Suite(&DriverSuite{ Constructor: driverConstructor, SkipCheck: skipCheck, }) } // RegisterIPCSuite registers a storage driver test suite which runs the named // driver as a child process with the given parameters. func RegisterIPCSuite(driverName string, ipcParams map[string]string, skipCheck SkipCheck) { panic("ipc testing is disabled for now") // NOTE(stevvooe): IPC testing is disabled for now. Uncomment the code // block before and remove the panic when we phase it back in. // suite := &DriverSuite{ // Constructor: func() (storagedriver.StorageDriver, error) { // d, err := ipc.NewDriverClient(driverName, ipcParams) // if err != nil { // return nil, err // } // err = d.Start() // if err != nil { // return nil, err // } // return d, nil // }, // SkipCheck: skipCheck, // } // suite.Teardown = func() error { // if suite.StorageDriver == nil { // return nil // } // driverClient := suite.StorageDriver.(*ipc.StorageDriverClient) // return driverClient.Stop() // } // check.Suite(suite) } // SkipCheck is a function used to determine if a test suite should be skipped. // If a SkipCheck returns a non-empty skip reason, the suite is skipped with // the given reason. type SkipCheck func() (reason string) // NeverSkip is a default SkipCheck which never skips the suite. var NeverSkip SkipCheck = func() string { return "" } // DriverConstructor is a function which returns a new // storagedriver.StorageDriver. type DriverConstructor func() (storagedriver.StorageDriver, error) // DriverTeardown is a function which cleans up a suite's // storagedriver.StorageDriver. type DriverTeardown func() error // DriverSuite is a gocheck test suite designed to test a // storagedriver.StorageDriver. // The intended way to create a DriverSuite is with RegisterInProcessSuite or // RegisterIPCSuite. type DriverSuite struct { Constructor DriverConstructor Teardown DriverTeardown SkipCheck storagedriver.StorageDriver } // SetUpSuite sets up the gocheck test suite. func (suite *DriverSuite) SetUpSuite(c *check.C) { if reason := suite.SkipCheck(); reason != "" { c.Skip(reason) } d, err := suite.Constructor() c.Assert(err, check.IsNil) suite.StorageDriver = d } // TearDownSuite tears down the gocheck test suite. func (suite *DriverSuite) TearDownSuite(c *check.C) { if suite.Teardown != nil { err := suite.Teardown() c.Assert(err, check.IsNil) } } // TearDownTest tears down the gocheck test. // This causes the suite to abort if any files are left around in the storage // driver. func (suite *DriverSuite) TearDownTest(c *check.C) { files, _ := suite.StorageDriver.List("/") if len(files) > 0 { c.Fatalf("Storage driver did not clean up properly. Offending files: %#v", files) } } // TestValidPaths checks that various valid file paths are accepted by the // storage driver. func (suite *DriverSuite) TestValidPaths(c *check.C) { contents := randomContents(64) validFiles := []string{ "/a", "/2", "/aa", "/a.a", "/0-9/abcdefg", "/abcdefg/z.75", "/abc/1.2.3.4.5-6_zyx/123.z/4", "/docker/docker-registry", "/123.abc", "/abc./abc", "/.abc", "/a--b", "/a-.b", "/_.abc"} for _, filename := range validFiles { err := suite.StorageDriver.PutContent(filename, contents) defer suite.StorageDriver.Delete(firstPart(filename)) c.Assert(err, check.IsNil) received, err := suite.StorageDriver.GetContent(filename) c.Assert(err, check.IsNil) c.Assert(received, check.DeepEquals, contents) } } // TestInvalidPaths checks that various invalid file paths are rejected by the // storage driver. func (suite *DriverSuite) TestInvalidPaths(c *check.C) { contents := randomContents(64) invalidFiles := []string{ "", "/", "abc", "123.abc", "//bcd", "/abc_123/", "/Docker/docker-registry"} for _, filename := range invalidFiles { err := suite.StorageDriver.PutContent(filename, contents) defer suite.StorageDriver.Delete(firstPart(filename)) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.InvalidPathError{}) _, err = suite.StorageDriver.GetContent(filename) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.InvalidPathError{}) } } // TestWriteRead1 tests a simple write-read workflow. func (suite *DriverSuite) TestWriteRead1(c *check.C) { filename := randomPath(32) contents := []byte("a") suite.writeReadCompare(c, filename, contents) } // TestWriteRead2 tests a simple write-read workflow with unicode data. func (suite *DriverSuite) TestWriteRead2(c *check.C) { filename := randomPath(32) contents := []byte("\xc3\x9f") suite.writeReadCompare(c, filename, contents) } // TestWriteRead3 tests a simple write-read workflow with a small string. func (suite *DriverSuite) TestWriteRead3(c *check.C) { filename := randomPath(32) contents := randomContents(32) suite.writeReadCompare(c, filename, contents) } // TestWriteRead4 tests a simple write-read workflow with 1MB of data. func (suite *DriverSuite) TestWriteRead4(c *check.C) { filename := randomPath(32) contents := randomContents(1024 * 1024) suite.writeReadCompare(c, filename, contents) } // TestWriteReadNonUTF8 tests that non-utf8 data may be written to the storage // driver safely. func (suite *DriverSuite) TestWriteReadNonUTF8(c *check.C) { filename := randomPath(32) contents := []byte{0x80, 0x80, 0x80, 0x80} suite.writeReadCompare(c, filename, contents) } // TestTruncate tests that putting smaller contents than an original file does // remove the excess contents. func (suite *DriverSuite) TestTruncate(c *check.C) { filename := randomPath(32) contents := randomContents(1024 * 1024) suite.writeReadCompare(c, filename, contents) contents = randomContents(1024) suite.writeReadCompare(c, filename, contents) } // TestReadNonexistent tests reading content from an empty path. func (suite *DriverSuite) TestReadNonexistent(c *check.C) { filename := randomPath(32) _, err := suite.StorageDriver.GetContent(filename) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) } // TestWriteReadStreams1 tests a simple write-read streaming workflow. func (suite *DriverSuite) TestWriteReadStreams1(c *check.C) { filename := randomPath(32) contents := []byte("a") suite.writeReadCompareStreams(c, filename, contents) } // TestWriteReadStreams2 tests a simple write-read streaming workflow with // unicode data. func (suite *DriverSuite) TestWriteReadStreams2(c *check.C) { filename := randomPath(32) contents := []byte("\xc3\x9f") suite.writeReadCompareStreams(c, filename, contents) } // TestWriteReadStreams3 tests a simple write-read streaming workflow with a // small amount of data. func (suite *DriverSuite) TestWriteReadStreams3(c *check.C) { filename := randomPath(32) contents := randomContents(32) suite.writeReadCompareStreams(c, filename, contents) } // TestWriteReadStreams4 tests a simple write-read streaming workflow with 1MB // of data. func (suite *DriverSuite) TestWriteReadStreams4(c *check.C) { filename := randomPath(32) contents := randomContents(1024 * 1024) suite.writeReadCompareStreams(c, filename, contents) } // TestWriteReadStreamsNonUTF8 tests that non-utf8 data may be written to the // storage driver safely. func (suite *DriverSuite) TestWriteReadStreamsNonUTF8(c *check.C) { filename := randomPath(32) contents := []byte{0x80, 0x80, 0x80, 0x80} suite.writeReadCompareStreams(c, filename, contents) } // TestWriteReadLargeStreams tests that a 5GB file may be written to the storage // driver safely. func (suite *DriverSuite) TestWriteReadLargeStreams(c *check.C) { if testing.Short() { c.Skip("Skipping test in short mode") } filename := randomPath(32) defer suite.StorageDriver.Delete(firstPart(filename)) checksum := sha1.New() var fileSize int64 = 5 * 1024 * 1024 * 1024 contents := newRandReader(fileSize) written, err := suite.StorageDriver.WriteStream(filename, 0, io.TeeReader(contents, checksum)) c.Assert(err, check.IsNil) c.Assert(written, check.Equals, fileSize) reader, err := suite.StorageDriver.ReadStream(filename, 0) c.Assert(err, check.IsNil) writtenChecksum := sha1.New() io.Copy(writtenChecksum, reader) c.Assert(writtenChecksum.Sum(nil), check.DeepEquals, checksum.Sum(nil)) } // TestReadStreamWithOffset tests that the appropriate data is streamed when // reading with a given offset. func (suite *DriverSuite) TestReadStreamWithOffset(c *check.C) { filename := randomPath(32) defer suite.StorageDriver.Delete(firstPart(filename)) chunkSize := int64(32) contentsChunk1 := randomContents(chunkSize) contentsChunk2 := randomContents(chunkSize) contentsChunk3 := randomContents(chunkSize) err := suite.StorageDriver.PutContent(filename, append(append(contentsChunk1, contentsChunk2...), contentsChunk3...)) c.Assert(err, check.IsNil) reader, err := suite.StorageDriver.ReadStream(filename, 0) c.Assert(err, check.IsNil) defer reader.Close() readContents, err := ioutil.ReadAll(reader) c.Assert(err, check.IsNil) c.Assert(readContents, check.DeepEquals, append(append(contentsChunk1, contentsChunk2...), contentsChunk3...)) reader, err = suite.StorageDriver.ReadStream(filename, chunkSize) c.Assert(err, check.IsNil) defer reader.Close() readContents, err = ioutil.ReadAll(reader) c.Assert(err, check.IsNil) c.Assert(readContents, check.DeepEquals, append(contentsChunk2, contentsChunk3...)) reader, err = suite.StorageDriver.ReadStream(filename, chunkSize*2) c.Assert(err, check.IsNil) defer reader.Close() readContents, err = ioutil.ReadAll(reader) c.Assert(err, check.IsNil) c.Assert(readContents, check.DeepEquals, contentsChunk3) // Ensure we get invalid offest for negative offsets. reader, err = suite.StorageDriver.ReadStream(filename, -1) c.Assert(err, check.FitsTypeOf, storagedriver.InvalidOffsetError{}) c.Assert(err.(storagedriver.InvalidOffsetError).Offset, check.Equals, int64(-1)) c.Assert(err.(storagedriver.InvalidOffsetError).Path, check.Equals, filename) c.Assert(reader, check.IsNil) // Read past the end of the content and make sure we get a reader that // returns 0 bytes and io.EOF reader, err = suite.StorageDriver.ReadStream(filename, chunkSize*3) c.Assert(err, check.IsNil) defer reader.Close() buf := make([]byte, chunkSize) n, err := reader.Read(buf) c.Assert(err, check.Equals, io.EOF) c.Assert(n, check.Equals, 0) // Check the N-1 boundary condition, ensuring we get 1 byte then io.EOF. reader, err = suite.StorageDriver.ReadStream(filename, chunkSize*3-1) c.Assert(err, check.IsNil) defer reader.Close() n, err = reader.Read(buf) c.Assert(n, check.Equals, 1) // We don't care whether the io.EOF comes on the this read or the first // zero read, but the only error acceptable here is io.EOF. if err != nil { c.Assert(err, check.Equals, io.EOF) } // Any more reads should result in zero bytes and io.EOF n, err = reader.Read(buf) c.Assert(n, check.Equals, 0) c.Assert(err, check.Equals, io.EOF) } // TestContinueStreamAppendLarge tests that a stream write can be appended to without // corrupting the data with a large chunk size. func (suite *DriverSuite) TestContinueStreamAppendLarge(c *check.C) { suite.testContinueStreamAppend(c, int64(10*1024*1024)) } // TestContinueStreamAppendSmall is the same as TestContinueStreamAppendLarge, but only // with a tiny chunk size in order to test corner cases for some cloud storage drivers. func (suite *DriverSuite) TestContinueStreamAppendSmall(c *check.C) { suite.testContinueStreamAppend(c, int64(32)) } func (suite *DriverSuite) testContinueStreamAppend(c *check.C, chunkSize int64) { filename := randomPath(32) defer suite.StorageDriver.Delete(firstPart(filename)) contentsChunk1 := randomContents(chunkSize) contentsChunk2 := randomContents(chunkSize) contentsChunk3 := randomContents(chunkSize) contentsChunk4 := randomContents(chunkSize) zeroChunk := make([]byte, int64(chunkSize)) fullContents := append(append(contentsChunk1, contentsChunk2...), contentsChunk3...) nn, err := suite.StorageDriver.WriteStream(filename, 0, bytes.NewReader(contentsChunk1)) c.Assert(err, check.IsNil) c.Assert(nn, check.Equals, int64(len(contentsChunk1))) fi, err := suite.StorageDriver.Stat(filename) c.Assert(err, check.IsNil) c.Assert(fi, check.NotNil) c.Assert(fi.Size(), check.Equals, int64(len(contentsChunk1))) nn, err = suite.StorageDriver.WriteStream(filename, fi.Size(), bytes.NewReader(contentsChunk2)) c.Assert(err, check.IsNil) c.Assert(nn, check.Equals, int64(len(contentsChunk2))) fi, err = suite.StorageDriver.Stat(filename) c.Assert(err, check.IsNil) c.Assert(fi, check.NotNil) c.Assert(fi.Size(), check.Equals, 2*chunkSize) // Test re-writing the last chunk nn, err = suite.StorageDriver.WriteStream(filename, fi.Size()-chunkSize, bytes.NewReader(contentsChunk2)) c.Assert(err, check.IsNil) c.Assert(nn, check.Equals, int64(len(contentsChunk2))) fi, err = suite.StorageDriver.Stat(filename) c.Assert(err, check.IsNil) c.Assert(fi, check.NotNil) c.Assert(fi.Size(), check.Equals, 2*chunkSize) nn, err = suite.StorageDriver.WriteStream(filename, fi.Size(), bytes.NewReader(fullContents[fi.Size():])) c.Assert(err, check.IsNil) c.Assert(nn, check.Equals, int64(len(fullContents[fi.Size():]))) received, err := suite.StorageDriver.GetContent(filename) c.Assert(err, check.IsNil) c.Assert(received, check.DeepEquals, fullContents) // Writing past size of file extends file (no offest error). We would like // to write chunk 4 one chunk length past chunk 3. It should be successful // and the resulting file will be 5 chunks long, with a chunk of all // zeros. fullContents = append(fullContents, zeroChunk...) fullContents = append(fullContents, contentsChunk4...) nn, err = suite.StorageDriver.WriteStream(filename, int64(len(fullContents))-chunkSize, bytes.NewReader(contentsChunk4)) c.Assert(err, check.IsNil) c.Assert(nn, check.Equals, chunkSize) fi, err = suite.StorageDriver.Stat(filename) c.Assert(err, check.IsNil) c.Assert(fi, check.NotNil) c.Assert(fi.Size(), check.Equals, int64(len(fullContents))) received, err = suite.StorageDriver.GetContent(filename) c.Assert(err, check.IsNil) c.Assert(len(received), check.Equals, len(fullContents)) c.Assert(received[chunkSize*3:chunkSize*4], check.DeepEquals, zeroChunk) c.Assert(received[chunkSize*4:chunkSize*5], check.DeepEquals, contentsChunk4) c.Assert(received, check.DeepEquals, fullContents) // Ensure that negative offsets return correct error. nn, err = suite.StorageDriver.WriteStream(filename, -1, bytes.NewReader(zeroChunk)) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.InvalidOffsetError{}) c.Assert(err.(storagedriver.InvalidOffsetError).Path, check.Equals, filename) c.Assert(err.(storagedriver.InvalidOffsetError).Offset, check.Equals, int64(-1)) } // TestReadNonexistentStream tests that reading a stream for a nonexistent path // fails. func (suite *DriverSuite) TestReadNonexistentStream(c *check.C) { filename := randomPath(32) _, err := suite.StorageDriver.ReadStream(filename, 0) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) _, err = suite.StorageDriver.ReadStream(filename, 64) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) } // TestList checks the returned list of keys after populating a directory tree. func (suite *DriverSuite) TestList(c *check.C) { rootDirectory := "/" + randomFilename(int64(8+rand.Intn(8))) defer suite.StorageDriver.Delete(rootDirectory) parentDirectory := rootDirectory + "/" + randomFilename(int64(8+rand.Intn(8))) childFiles := make([]string, 50) for i := 0; i < len(childFiles); i++ { childFile := parentDirectory + "/" + randomFilename(int64(8+rand.Intn(8))) childFiles[i] = childFile err := suite.StorageDriver.PutContent(childFile, randomContents(32)) c.Assert(err, check.IsNil) } sort.Strings(childFiles) keys, err := suite.StorageDriver.List("/") c.Assert(err, check.IsNil) c.Assert(keys, check.DeepEquals, []string{rootDirectory}) keys, err = suite.StorageDriver.List(rootDirectory) c.Assert(err, check.IsNil) c.Assert(keys, check.DeepEquals, []string{parentDirectory}) keys, err = suite.StorageDriver.List(parentDirectory) c.Assert(err, check.IsNil) sort.Strings(keys) c.Assert(keys, check.DeepEquals, childFiles) // A few checks to add here (check out #819 for more discussion on this): // 1. Ensure that all paths are absolute. // 2. Ensure that listings only include direct children. // 3. Ensure that we only respond to directory listings that end with a slash (maybe?). } // TestMove checks that a moved object no longer exists at the source path and // does exist at the destination. func (suite *DriverSuite) TestMove(c *check.C) { contents := randomContents(32) sourcePath := randomPath(32) destPath := randomPath(32) defer suite.StorageDriver.Delete(firstPart(sourcePath)) defer suite.StorageDriver.Delete(firstPart(destPath)) err := suite.StorageDriver.PutContent(sourcePath, contents) c.Assert(err, check.IsNil) err = suite.StorageDriver.Move(sourcePath, destPath) c.Assert(err, check.IsNil) received, err := suite.StorageDriver.GetContent(destPath) c.Assert(err, check.IsNil) c.Assert(received, check.DeepEquals, contents) _, err = suite.StorageDriver.GetContent(sourcePath) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) } // TestMoveOverwrite checks that a moved object no longer exists at the source // path and overwrites the contents at the destination. func (suite *DriverSuite) TestMoveOverwrite(c *check.C) { sourcePath := randomPath(32) destPath := randomPath(32) sourceContents := randomContents(32) destContents := randomContents(64) defer suite.StorageDriver.Delete(firstPart(sourcePath)) defer suite.StorageDriver.Delete(firstPart(destPath)) err := suite.StorageDriver.PutContent(sourcePath, sourceContents) c.Assert(err, check.IsNil) err = suite.StorageDriver.PutContent(destPath, destContents) c.Assert(err, check.IsNil) err = suite.StorageDriver.Move(sourcePath, destPath) c.Assert(err, check.IsNil) received, err := suite.StorageDriver.GetContent(destPath) c.Assert(err, check.IsNil) c.Assert(received, check.DeepEquals, sourceContents) _, err = suite.StorageDriver.GetContent(sourcePath) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) } // TestMoveNonexistent checks that moving a nonexistent key fails and does not // delete the data at the destination path. func (suite *DriverSuite) TestMoveNonexistent(c *check.C) { contents := randomContents(32) sourcePath := randomPath(32) destPath := randomPath(32) defer suite.StorageDriver.Delete(firstPart(destPath)) err := suite.StorageDriver.PutContent(destPath, contents) c.Assert(err, check.IsNil) err = suite.StorageDriver.Move(sourcePath, destPath) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) received, err := suite.StorageDriver.GetContent(destPath) c.Assert(err, check.IsNil) c.Assert(received, check.DeepEquals, contents) } // TestDelete checks that the delete operation removes data from the storage // driver func (suite *DriverSuite) TestDelete(c *check.C) { filename := randomPath(32) contents := randomContents(32) defer suite.StorageDriver.Delete(firstPart(filename)) err := suite.StorageDriver.PutContent(filename, contents) c.Assert(err, check.IsNil) err = suite.StorageDriver.Delete(filename) c.Assert(err, check.IsNil) _, err = suite.StorageDriver.GetContent(filename) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) } // TestURLFor checks that the URLFor method functions properly, but only if it // is implemented func (suite *DriverSuite) TestURLFor(c *check.C) { filename := randomPath(32) contents := randomContents(32) defer suite.StorageDriver.Delete(firstPart(filename)) err := suite.StorageDriver.PutContent(filename, contents) c.Assert(err, check.IsNil) url, err := suite.StorageDriver.URLFor(filename, nil) if err == storagedriver.ErrUnsupportedMethod { return } c.Assert(err, check.IsNil) response, err := http.Get(url) c.Assert(err, check.IsNil) defer response.Body.Close() read, err := ioutil.ReadAll(response.Body) c.Assert(err, check.IsNil) c.Assert(read, check.DeepEquals, contents) url, err = suite.StorageDriver.URLFor(filename, map[string]interface{}{"method": "HEAD"}) if err == storagedriver.ErrUnsupportedMethod { return } c.Assert(err, check.IsNil) response, err = http.Head(url) c.Assert(response.StatusCode, check.Equals, 200) c.Assert(response.ContentLength, check.Equals, int64(32)) } // TestDeleteNonexistent checks that removing a nonexistent key fails. func (suite *DriverSuite) TestDeleteNonexistent(c *check.C) { filename := randomPath(32) err := suite.StorageDriver.Delete(filename) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) } // TestDeleteFolder checks that deleting a folder removes all child elements. func (suite *DriverSuite) TestDeleteFolder(c *check.C) { dirname := randomPath(32) filename1 := randomPath(32) filename2 := randomPath(32) filename3 := randomPath(32) contents := randomContents(32) defer suite.StorageDriver.Delete(firstPart(dirname)) err := suite.StorageDriver.PutContent(path.Join(dirname, filename1), contents) c.Assert(err, check.IsNil) err = suite.StorageDriver.PutContent(path.Join(dirname, filename2), contents) c.Assert(err, check.IsNil) err = suite.StorageDriver.PutContent(path.Join(dirname, filename3), contents) c.Assert(err, check.IsNil) err = suite.StorageDriver.Delete(path.Join(dirname, filename1)) c.Assert(err, check.IsNil) _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename1)) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename2)) c.Assert(err, check.IsNil) _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename3)) c.Assert(err, check.IsNil) err = suite.StorageDriver.Delete(dirname) c.Assert(err, check.IsNil) _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename1)) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename2)) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) _, err = suite.StorageDriver.GetContent(path.Join(dirname, filename3)) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) } // TestStatCall runs verifies the implementation of the storagedriver's Stat call. func (suite *DriverSuite) TestStatCall(c *check.C) { content := randomContents(4096) dirPath := randomPath(32) fileName := randomFilename(32) filePath := path.Join(dirPath, fileName) defer suite.StorageDriver.Delete(firstPart(dirPath)) // Call on non-existent file/dir, check error. fi, err := suite.StorageDriver.Stat(dirPath) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) c.Assert(fi, check.IsNil) fi, err = suite.StorageDriver.Stat(filePath) c.Assert(err, check.NotNil) c.Assert(err, check.FitsTypeOf, storagedriver.PathNotFoundError{}) c.Assert(fi, check.IsNil) err = suite.StorageDriver.PutContent(filePath, content) c.Assert(err, check.IsNil) // Call on regular file, check results fi, err = suite.StorageDriver.Stat(filePath) c.Assert(err, check.IsNil) c.Assert(fi, check.NotNil) c.Assert(fi.Path(), check.Equals, filePath) c.Assert(fi.Size(), check.Equals, int64(len(content))) c.Assert(fi.IsDir(), check.Equals, false) createdTime := fi.ModTime() // Sleep and modify the file time.Sleep(time.Second * 10) content = randomContents(4096) err = suite.StorageDriver.PutContent(filePath, content) c.Assert(err, check.IsNil) fi, err = suite.StorageDriver.Stat(filePath) c.Assert(err, check.IsNil) c.Assert(fi, check.NotNil) time.Sleep(time.Second * 5) // allow changes to propagate (eventual consistency) // Check if the modification time is after the creation time. // In case of cloud storage services, storage frontend nodes might have // time drift between them, however that should be solved with sleeping // before update. modTime := fi.ModTime() if !modTime.After(createdTime) { c.Errorf("modtime (%s) is before the creation time (%s)", modTime, createdTime) } // Call on directory (do not check ModTime as dirs don't need to support it) fi, err = suite.StorageDriver.Stat(dirPath) c.Assert(err, check.IsNil) c.Assert(fi, check.NotNil) c.Assert(fi.Path(), check.Equals, dirPath) c.Assert(fi.Size(), check.Equals, int64(0)) c.Assert(fi.IsDir(), check.Equals, true) } // TestPutContentMultipleTimes checks that if storage driver can overwrite the content // in the subsequent puts. Validates that PutContent does not have to work // with an offset like WriteStream does and overwrites the file entirely // rather than writing the data to the [0,len(data)) of the file. func (suite *DriverSuite) TestPutContentMultipleTimes(c *check.C) { filename := randomPath(32) contents := randomContents(4096) defer suite.StorageDriver.Delete(firstPart(filename)) err := suite.StorageDriver.PutContent(filename, contents) c.Assert(err, check.IsNil) contents = randomContents(2048) // upload a different, smaller file err = suite.StorageDriver.PutContent(filename, contents) c.Assert(err, check.IsNil) readContents, err := suite.StorageDriver.GetContent(filename) c.Assert(err, check.IsNil) c.Assert(readContents, check.DeepEquals, contents) } // TestConcurrentStreamReads checks that multiple clients can safely read from // the same file simultaneously with various offsets. func (suite *DriverSuite) TestConcurrentStreamReads(c *check.C) { var filesize int64 = 128 * 1024 * 1024 if testing.Short() { filesize = 10 * 1024 * 1024 c.Log("Reducing file size to 10MB for short mode") } filename := randomPath(32) contents := randomContents(filesize) defer suite.StorageDriver.Delete(firstPart(filename)) err := suite.StorageDriver.PutContent(filename, contents) c.Assert(err, check.IsNil) var wg sync.WaitGroup readContents := func() { defer wg.Done() offset := rand.Int63n(int64(len(contents))) reader, err := suite.StorageDriver.ReadStream(filename, offset) c.Assert(err, check.IsNil) readContents, err := ioutil.ReadAll(reader) c.Assert(err, check.IsNil) c.Assert(readContents, check.DeepEquals, contents[offset:]) } wg.Add(10) for i := 0; i < 10; i++ { go readContents() } wg.Wait() } // TestConcurrentFileStreams checks that multiple *os.File objects can be passed // in to WriteStream concurrently without hanging. func (suite *DriverSuite) TestConcurrentFileStreams(c *check.C) { // if _, isIPC := suite.StorageDriver.(*ipc.StorageDriverClient); isIPC { // c.Skip("Need to fix out-of-process concurrency") // } numStreams := 32 if testing.Short() { numStreams = 8 c.Log("Reducing number of streams to 8 for short mode") } var wg sync.WaitGroup testStream := func(size int64) { defer wg.Done() suite.testFileStreams(c, size) } wg.Add(numStreams) for i := numStreams; i > 0; i-- { go testStream(int64(numStreams) * 1024 * 1024) } wg.Wait() } // TestEventualConsistency checks that if stat says that a file is a certain size, then // you can freely read from the file (this is the only guarantee that the driver needs to provide) func (suite *DriverSuite) TestEventualConsistency(c *check.C) { if testing.Short() { c.Skip("Skipping test in short mode") } filename := randomPath(32) defer suite.StorageDriver.Delete(firstPart(filename)) var offset int64 var misswrites int var chunkSize int64 = 32 for i := 0; i < 1024; i++ { contents := randomContents(chunkSize) read, err := suite.StorageDriver.WriteStream(filename, offset, bytes.NewReader(contents)) c.Assert(err, check.IsNil) fi, err := suite.StorageDriver.Stat(filename) c.Assert(err, check.IsNil) // We are most concerned with being able to read data as soon as Stat declares // it is uploaded. This is the strongest guarantee that some drivers (that guarantee // at best eventual consistency) absolutely need to provide. if fi.Size() == offset+chunkSize { reader, err := suite.StorageDriver.ReadStream(filename, offset) c.Assert(err, check.IsNil) readContents, err := ioutil.ReadAll(reader) c.Assert(err, check.IsNil) c.Assert(readContents, check.DeepEquals, contents) reader.Close() offset += read } else { misswrites++ } } if misswrites > 0 { c.Log("There were " + string(misswrites) + " occurences of a write not being instantly available.") } c.Assert(misswrites, check.Not(check.Equals), 1024) } // BenchmarkPutGetEmptyFiles benchmarks PutContent/GetContent for 0B files func (suite *DriverSuite) BenchmarkPutGetEmptyFiles(c *check.C) { suite.benchmarkPutGetFiles(c, 0) } // BenchmarkPutGet1KBFiles benchmarks PutContent/GetContent for 1KB files func (suite *DriverSuite) BenchmarkPutGet1KBFiles(c *check.C) { suite.benchmarkPutGetFiles(c, 1024) } // BenchmarkPutGet1MBFiles benchmarks PutContent/GetContent for 1MB files func (suite *DriverSuite) BenchmarkPutGet1MBFiles(c *check.C) { suite.benchmarkPutGetFiles(c, 1024*1024) } // BenchmarkPutGet1GBFiles benchmarks PutContent/GetContent for 1GB files func (suite *DriverSuite) BenchmarkPutGet1GBFiles(c *check.C) { suite.benchmarkPutGetFiles(c, 1024*1024*1024) } func (suite *DriverSuite) benchmarkPutGetFiles(c *check.C, size int64) { c.SetBytes(size) parentDir := randomPath(8) defer func() { c.StopTimer() suite.StorageDriver.Delete(firstPart(parentDir)) }() for i := 0; i < c.N; i++ { filename := path.Join(parentDir, randomPath(32)) err := suite.StorageDriver.PutContent(filename, randomContents(size)) c.Assert(err, check.IsNil) _, err = suite.StorageDriver.GetContent(filename) c.Assert(err, check.IsNil) } } // BenchmarkStreamEmptyFiles benchmarks WriteStream/ReadStream for 0B files func (suite *DriverSuite) BenchmarkStreamEmptyFiles(c *check.C) { suite.benchmarkStreamFiles(c, 0) } // BenchmarkStream1KBFiles benchmarks WriteStream/ReadStream for 1KB files func (suite *DriverSuite) BenchmarkStream1KBFiles(c *check.C) { suite.benchmarkStreamFiles(c, 1024) } // BenchmarkStream1MBFiles benchmarks WriteStream/ReadStream for 1MB files func (suite *DriverSuite) BenchmarkStream1MBFiles(c *check.C) { suite.benchmarkStreamFiles(c, 1024*1024) } // BenchmarkStream1GBFiles benchmarks WriteStream/ReadStream for 1GB files func (suite *DriverSuite) BenchmarkStream1GBFiles(c *check.C) { suite.benchmarkStreamFiles(c, 1024*1024*1024) } func (suite *DriverSuite) benchmarkStreamFiles(c *check.C, size int64) { c.SetBytes(size) parentDir := randomPath(8) defer func() { c.StopTimer() suite.StorageDriver.Delete(firstPart(parentDir)) }() for i := 0; i < c.N; i++ { filename := path.Join(parentDir, randomPath(32)) written, err := suite.StorageDriver.WriteStream(filename, 0, bytes.NewReader(randomContents(size))) c.Assert(err, check.IsNil) c.Assert(written, check.Equals, size) rc, err := suite.StorageDriver.ReadStream(filename, 0) c.Assert(err, check.IsNil) rc.Close() } } // BenchmarkList5Files benchmarks List for 5 small files func (suite *DriverSuite) BenchmarkList5Files(c *check.C) { suite.benchmarkListFiles(c, 5) } // BenchmarkList50Files benchmarks List for 50 small files func (suite *DriverSuite) BenchmarkList50Files(c *check.C) { suite.benchmarkListFiles(c, 50) } func (suite *DriverSuite) benchmarkListFiles(c *check.C, numFiles int64) { parentDir := randomPath(8) defer func() { c.StopTimer() suite.StorageDriver.Delete(firstPart(parentDir)) }() for i := int64(0); i < numFiles; i++ { err := suite.StorageDriver.PutContent(path.Join(parentDir, randomPath(32)), nil) c.Assert(err, check.IsNil) } c.ResetTimer() for i := 0; i < c.N; i++ { files, err := suite.StorageDriver.List(parentDir) c.Assert(err, check.IsNil) c.Assert(int64(len(files)), check.Equals, numFiles) } } // BenchmarkDelete5Files benchmarks Delete for 5 small files func (suite *DriverSuite) BenchmarkDelete5Files(c *check.C) { suite.benchmarkDeleteFiles(c, 5) } // BenchmarkDelete50Files benchmarks Delete for 50 small files func (suite *DriverSuite) BenchmarkDelete50Files(c *check.C) { suite.benchmarkDeleteFiles(c, 50) } func (suite *DriverSuite) benchmarkDeleteFiles(c *check.C, numFiles int64) { for i := 0; i < c.N; i++ { parentDir := randomPath(8) defer suite.StorageDriver.Delete(firstPart(parentDir)) c.StopTimer() for j := int64(0); j < numFiles; j++ { err := suite.StorageDriver.PutContent(path.Join(parentDir, randomPath(32)), nil) c.Assert(err, check.IsNil) } c.StartTimer() // This is the operation we're benchmarking err := suite.StorageDriver.Delete(firstPart(parentDir)) c.Assert(err, check.IsNil) } } func (suite *DriverSuite) testFileStreams(c *check.C, size int64) { tf, err := ioutil.TempFile("", "tf") c.Assert(err, check.IsNil) defer os.Remove(tf.Name()) defer tf.Close() filename := randomPath(32) defer suite.StorageDriver.Delete(firstPart(filename)) contents := randomContents(size) _, err = tf.Write(contents) c.Assert(err, check.IsNil) tf.Sync() tf.Seek(0, os.SEEK_SET) nn, err := suite.StorageDriver.WriteStream(filename, 0, tf) c.Assert(err, check.IsNil) c.Assert(nn, check.Equals, size) reader, err := suite.StorageDriver.ReadStream(filename, 0) c.Assert(err, check.IsNil) defer reader.Close() readContents, err := ioutil.ReadAll(reader) c.Assert(err, check.IsNil) c.Assert(readContents, check.DeepEquals, contents) } func (suite *DriverSuite) writeReadCompare(c *check.C, filename string, contents []byte) { defer suite.StorageDriver.Delete(firstPart(filename)) err := suite.StorageDriver.PutContent(filename, contents) c.Assert(err, check.IsNil) readContents, err := suite.StorageDriver.GetContent(filename) c.Assert(err, check.IsNil) c.Assert(readContents, check.DeepEquals, contents) } func (suite *DriverSuite) writeReadCompareStreams(c *check.C, filename string, contents []byte) { defer suite.StorageDriver.Delete(firstPart(filename)) nn, err := suite.StorageDriver.WriteStream(filename, 0, bytes.NewReader(contents)) c.Assert(err, check.IsNil) c.Assert(nn, check.Equals, int64(len(contents))) reader, err := suite.StorageDriver.ReadStream(filename, 0) c.Assert(err, check.IsNil) defer reader.Close() readContents, err := ioutil.ReadAll(reader) c.Assert(err, check.IsNil) c.Assert(readContents, check.DeepEquals, contents) } var filenameChars = []byte("abcdefghijklmnopqrstuvwxyz0123456789") var separatorChars = []byte("._-") func randomPath(length int64) string { path := "/" for int64(len(path)) < length { chunkLength := rand.Int63n(length-int64(len(path))) + 1 chunk := randomFilename(chunkLength) path += chunk remaining := length - int64(len(path)) if remaining == 1 { path += randomFilename(1) } else if remaining > 1 { path += "/" } } return path } func randomFilename(length int64) string { b := make([]byte, length) wasSeparator := true for i := range b { if !wasSeparator && i < len(b)-1 && rand.Intn(4) == 0 { b[i] = separatorChars[rand.Intn(len(separatorChars))] wasSeparator = true } else { b[i] = filenameChars[rand.Intn(len(filenameChars))] wasSeparator = false } } return string(b) } func randomContents(length int64) []byte { b := make([]byte, length) for i := range b { b[i] = byte(rand.Intn(2 << 8)) } return b } type randReader struct { r int64 m sync.Mutex } func (rr *randReader) Read(p []byte) (n int, err error) { rr.m.Lock() defer rr.m.Unlock() for i := 0; i < len(p) && rr.r > 0; i++ { p[i] = byte(rand.Intn(255)) n++ rr.r-- } if rr.r == 0 { err = io.EOF } return } func newRandReader(n int64) *randReader { return &randReader{r: n} } func firstPart(filePath string) string { if filePath == "" { return "/" } for { if filePath[len(filePath)-1] == '/' { filePath = filePath[:len(filePath)-1] } dir, file := path.Split(filePath) if dir == "" && file == "" { return "/" } if dir == "/" || dir == "" { return "/" + file } if file == "" { return dir } filePath = dir } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/azure/0000755000175000017500000000000012502424227026514 5ustar tianontianon././@LongLink0000644000000000000000000000015300000000000011602 Lustar rootrootdistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/azure/zerofillwriter_test.godistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/azure/zerofillwriter_t0000644000175000017500000000721712502424227032054 0ustar tianontianonpackage azure import ( "bytes" "testing" ) func Test_zeroFillWrite_AppendNoGap(t *testing.T) { s := NewStorageSimulator() bw := newRandomBlobWriter(&s, 1024*1) zw := newZeroFillWriter(&bw) if err := s.CreateBlockBlob("a", "b"); err != nil { t.Fatal(err) } firstChunk := randomContents(1024*3 + 512) if nn, err := zw.Write("a", "b", 0, bytes.NewReader(firstChunk)); err != nil { t.Fatal(err) } else if expected := int64(len(firstChunk)); expected != nn { t.Fatalf("wrong written bytes count: %v, expected: %v", nn, expected) } if out, err := s.GetBlob("a", "b"); err != nil { t.Fatal(err) } else { assertBlobContents(t, out, firstChunk) } secondChunk := randomContents(256) if nn, err := zw.Write("a", "b", int64(len(firstChunk)), bytes.NewReader(secondChunk)); err != nil { t.Fatal(err) } else if expected := int64(len(secondChunk)); expected != nn { t.Fatalf("wrong written bytes count: %v, expected: %v", nn, expected) } if out, err := s.GetBlob("a", "b"); err != nil { t.Fatal(err) } else { assertBlobContents(t, out, append(firstChunk, secondChunk...)) } } func Test_zeroFillWrite_StartWithGap(t *testing.T) { s := NewStorageSimulator() bw := newRandomBlobWriter(&s, 1024*2) zw := newZeroFillWriter(&bw) if err := s.CreateBlockBlob("a", "b"); err != nil { t.Fatal(err) } chunk := randomContents(1024 * 5) padding := int64(1024*2 + 256) if nn, err := zw.Write("a", "b", padding, bytes.NewReader(chunk)); err != nil { t.Fatal(err) } else if expected := int64(len(chunk)); expected != nn { t.Fatalf("wrong written bytes count: %v, expected: %v", nn, expected) } if out, err := s.GetBlob("a", "b"); err != nil { t.Fatal(err) } else { assertBlobContents(t, out, append(make([]byte, padding), chunk...)) } } func Test_zeroFillWrite_AppendWithGap(t *testing.T) { s := NewStorageSimulator() bw := newRandomBlobWriter(&s, 1024*2) zw := newZeroFillWriter(&bw) if err := s.CreateBlockBlob("a", "b"); err != nil { t.Fatal(err) } firstChunk := randomContents(1024*3 + 512) if _, err := zw.Write("a", "b", 0, bytes.NewReader(firstChunk)); err != nil { t.Fatal(err) } if out, err := s.GetBlob("a", "b"); err != nil { t.Fatal(err) } else { assertBlobContents(t, out, firstChunk) } secondChunk := randomContents(256) padding := int64(1024 * 4) if nn, err := zw.Write("a", "b", int64(len(firstChunk))+padding, bytes.NewReader(secondChunk)); err != nil { t.Fatal(err) } else if expected := int64(len(secondChunk)); expected != nn { t.Fatalf("wrong written bytes count: %v, expected: %v", nn, expected) } if out, err := s.GetBlob("a", "b"); err != nil { t.Fatal(err) } else { assertBlobContents(t, out, append(firstChunk, append(make([]byte, padding), secondChunk...)...)) } } func Test_zeroFillWrite_LiesWithinSize(t *testing.T) { s := NewStorageSimulator() bw := newRandomBlobWriter(&s, 1024*2) zw := newZeroFillWriter(&bw) if err := s.CreateBlockBlob("a", "b"); err != nil { t.Fatal(err) } firstChunk := randomContents(1024 * 3) if _, err := zw.Write("a", "b", 0, bytes.NewReader(firstChunk)); err != nil { t.Fatal(err) } if out, err := s.GetBlob("a", "b"); err != nil { t.Fatal(err) } else { assertBlobContents(t, out, firstChunk) } // in this case, zerofill won't be used secondChunk := randomContents(256) if nn, err := zw.Write("a", "b", 0, bytes.NewReader(secondChunk)); err != nil { t.Fatal(err) } else if expected := int64(len(secondChunk)); expected != nn { t.Fatalf("wrong written bytes count: %v, expected: %v", nn, expected) } if out, err := s.GetBlob("a", "b"); err != nil { t.Fatal(err) } else { assertBlobContents(t, out, append(secondChunk, firstChunk[len(secondChunk):]...)) } } ././@LongLink0000644000000000000000000000014600000000000011604 Lustar rootrootdistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/azure/zerofillwriter.godistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/azure/zerofillwriter.g0000644000175000017500000000253212502424227031751 0ustar tianontianonpackage azure import ( "bytes" "io" ) type blockBlobWriter interface { GetSize(container, blob string) (int64, error) WriteBlobAt(container, blob string, offset int64, chunk io.Reader) (int64, error) } // zeroFillWriter enables writing to an offset outside a block blob's size // by offering the chunk to the underlying writer as a contiguous data with // the gap in between filled with NUL (zero) bytes. type zeroFillWriter struct { blockBlobWriter } func newZeroFillWriter(b blockBlobWriter) zeroFillWriter { w := zeroFillWriter{} w.blockBlobWriter = b return w } // Write writes the given chunk to the specified existing blob even though // offset is out of blob's size. The gaps are filled with zeros. Returned // written number count does not include zeros written. func (z *zeroFillWriter) Write(container, blob string, offset int64, chunk io.Reader) (int64, error) { size, err := z.blockBlobWriter.GetSize(container, blob) if err != nil { return 0, err } var reader io.Reader var zeroPadding int64 if offset <= size { reader = chunk } else { zeroPadding = offset - size offset = size // adjust offset to be the append index zeros := bytes.NewReader(make([]byte, zeroPadding)) reader = io.MultiReader(zeros, chunk) } nn, err := z.blockBlobWriter.WriteBlobAt(container, blob, offset, reader) nn -= zeroPadding return nn, err } ././@LongLink0000644000000000000000000000015100000000000011600 Lustar rootrootdistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/azure/randomwriter_test.godistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/azure/randomwriter_tes0000644000175000017500000002664712502424227032046 0ustar tianontianonpackage azure import ( "bytes" "io" "io/ioutil" "math/rand" "reflect" "strings" "testing" azure "github.com/MSOpenTech/azure-sdk-for-go/clients/storage" ) func TestRandomWriter_writeChunkToBlocks(t *testing.T) { s := NewStorageSimulator() rw := newRandomBlobWriter(&s, 3) rand := newBlockIDGenerator() c := []byte("AAABBBCCCD") if err := rw.bs.CreateBlockBlob("a", "b"); err != nil { t.Fatal(err) } bw, nn, err := rw.writeChunkToBlocks("a", "b", bytes.NewReader(c), rand) if err != nil { t.Fatal(err) } if expected := int64(len(c)); nn != expected { t.Fatalf("wrong nn:%v, expected:%v", nn, expected) } if expected := 4; len(bw) != expected { t.Fatal("unexpected written block count") } bx, err := s.GetBlockList("a", "b", azure.BlockListTypeAll) if err != nil { t.Fatal(err) } if expected := 0; len(bx.CommittedBlocks) != expected { t.Fatal("unexpected committed block count") } if expected := 4; len(bx.UncommittedBlocks) != expected { t.Fatalf("unexpected uncommitted block count: %d -- %#v", len(bx.UncommittedBlocks), bx) } if err := rw.bs.PutBlockList("a", "b", bw); err != nil { t.Fatal(err) } r, err := rw.bs.GetBlob("a", "b") if err != nil { t.Fatal(err) } assertBlobContents(t, r, c) } func TestRandomWriter_blocksLeftSide(t *testing.T) { blob := "AAAAABBBBBCCC" cases := []struct { offset int64 expectedBlob string expectedPattern []azure.BlockStatus }{ {0, "", []azure.BlockStatus{}}, // write to beginning, discard all {13, blob, []azure.BlockStatus{azure.BlockStatusCommitted, azure.BlockStatusCommitted, azure.BlockStatusCommitted}}, // write to end, no change {1, "A", []azure.BlockStatus{azure.BlockStatusUncommitted}}, // write at 1 {5, "AAAAA", []azure.BlockStatus{azure.BlockStatusCommitted}}, // write just after first block {6, "AAAAAB", []azure.BlockStatus{azure.BlockStatusCommitted, azure.BlockStatusUncommitted}}, // split the second block {9, "AAAAABBBB", []azure.BlockStatus{azure.BlockStatusCommitted, azure.BlockStatusUncommitted}}, // write just after first block } for _, c := range cases { s := NewStorageSimulator() rw := newRandomBlobWriter(&s, 5) rand := newBlockIDGenerator() if err := rw.bs.CreateBlockBlob("a", "b"); err != nil { t.Fatal(err) } bw, _, err := rw.writeChunkToBlocks("a", "b", strings.NewReader(blob), rand) if err != nil { t.Fatal(err) } if err := rw.bs.PutBlockList("a", "b", bw); err != nil { t.Fatal(err) } bx, err := rw.blocksLeftSide("a", "b", c.offset, rand) if err != nil { t.Fatal(err) } bs := []azure.BlockStatus{} for _, v := range bx { bs = append(bs, v.Status) } if !reflect.DeepEqual(bs, c.expectedPattern) { t.Logf("Committed blocks %v", bw) t.Fatalf("For offset %v: Expected pattern: %v, Got: %v\n(Returned: %v)", c.offset, c.expectedPattern, bs, bx) } if rw.bs.PutBlockList("a", "b", bx); err != nil { t.Fatal(err) } r, err := rw.bs.GetBlob("a", "b") if err != nil { t.Fatal(err) } cout, err := ioutil.ReadAll(r) if err != nil { t.Fatal(err) } outBlob := string(cout) if outBlob != c.expectedBlob { t.Fatalf("wrong blob contents: %v, expected: %v", outBlob, c.expectedBlob) } } } func TestRandomWriter_blocksRightSide(t *testing.T) { blob := "AAAAABBBBBCCC" cases := []struct { offset int64 size int64 expectedBlob string expectedPattern []azure.BlockStatus }{ {0, 100, "", []azure.BlockStatus{}}, // overwrite the entire blob {0, 3, "AABBBBBCCC", []azure.BlockStatus{azure.BlockStatusUncommitted, azure.BlockStatusCommitted, azure.BlockStatusCommitted}}, // split first block {4, 1, "BBBBBCCC", []azure.BlockStatus{azure.BlockStatusCommitted, azure.BlockStatusCommitted}}, // write to last char of first block {1, 6, "BBBCCC", []azure.BlockStatus{azure.BlockStatusUncommitted, azure.BlockStatusCommitted}}, // overwrite splits first and second block, last block remains {3, 8, "CC", []azure.BlockStatus{azure.BlockStatusUncommitted}}, // overwrite a block in middle block, split end block {10, 1, "CC", []azure.BlockStatus{azure.BlockStatusUncommitted}}, // overwrite first byte of rightmost block {11, 2, "", []azure.BlockStatus{}}, // overwrite the rightmost index {13, 20, "", []azure.BlockStatus{}}, // append to the end } for _, c := range cases { s := NewStorageSimulator() rw := newRandomBlobWriter(&s, 5) rand := newBlockIDGenerator() if err := rw.bs.CreateBlockBlob("a", "b"); err != nil { t.Fatal(err) } bw, _, err := rw.writeChunkToBlocks("a", "b", strings.NewReader(blob), rand) if err != nil { t.Fatal(err) } if err := rw.bs.PutBlockList("a", "b", bw); err != nil { t.Fatal(err) } bx, err := rw.blocksRightSide("a", "b", c.offset, c.size, rand) if err != nil { t.Fatal(err) } bs := []azure.BlockStatus{} for _, v := range bx { bs = append(bs, v.Status) } if !reflect.DeepEqual(bs, c.expectedPattern) { t.Logf("Committed blocks %v", bw) t.Fatalf("For offset %v-size:%v: Expected pattern: %v, Got: %v\n(Returned: %v)", c.offset, c.size, c.expectedPattern, bs, bx) } if rw.bs.PutBlockList("a", "b", bx); err != nil { t.Fatal(err) } r, err := rw.bs.GetBlob("a", "b") if err != nil { t.Fatal(err) } cout, err := ioutil.ReadAll(r) if err != nil { t.Fatal(err) } outBlob := string(cout) if outBlob != c.expectedBlob { t.Fatalf("For offset %v-size:%v: wrong blob contents: %v, expected: %v", c.offset, c.size, outBlob, c.expectedBlob) } } } func TestRandomWriter_Write_NewBlob(t *testing.T) { var ( s = NewStorageSimulator() rw = newRandomBlobWriter(&s, 1024*3) // 3 KB blocks blob = randomContents(1024 * 7) // 7 KB blob ) if err := rw.bs.CreateBlockBlob("a", "b"); err != nil { t.Fatal(err) } if _, err := rw.WriteBlobAt("a", "b", 10, bytes.NewReader(blob)); err == nil { t.Fatal("expected error, got nil") } if _, err := rw.WriteBlobAt("a", "b", 100000, bytes.NewReader(blob)); err == nil { t.Fatal("expected error, got nil") } if nn, err := rw.WriteBlobAt("a", "b", 0, bytes.NewReader(blob)); err != nil { t.Fatal(err) } else if expected := int64(len(blob)); expected != nn { t.Fatalf("wrong written bytes count: %v, expected: %v", nn, expected) } if out, err := rw.bs.GetBlob("a", "b"); err != nil { t.Fatal(err) } else { assertBlobContents(t, out, blob) } if bx, err := rw.bs.GetBlockList("a", "b", azure.BlockListTypeCommitted); err != nil { t.Fatal(err) } else if len(bx.CommittedBlocks) != 3 { t.Fatalf("got wrong number of committed blocks: %v", len(bx.CommittedBlocks)) } // Replace first 512 bytes leftChunk := randomContents(512) blob = append(leftChunk, blob[512:]...) if nn, err := rw.WriteBlobAt("a", "b", 0, bytes.NewReader(leftChunk)); err != nil { t.Fatal(err) } else if expected := int64(len(leftChunk)); expected != nn { t.Fatalf("wrong written bytes count: %v, expected: %v", nn, expected) } if out, err := rw.bs.GetBlob("a", "b"); err != nil { t.Fatal(err) } else { assertBlobContents(t, out, blob) } if bx, err := rw.bs.GetBlockList("a", "b", azure.BlockListTypeCommitted); err != nil { t.Fatal(err) } else if expected := 4; len(bx.CommittedBlocks) != expected { t.Fatalf("got wrong number of committed blocks: %v, expected: %v", len(bx.CommittedBlocks), expected) } // Replace last 512 bytes with 1024 bytes rightChunk := randomContents(1024) offset := int64(len(blob) - 512) blob = append(blob[:offset], rightChunk...) if nn, err := rw.WriteBlobAt("a", "b", offset, bytes.NewReader(rightChunk)); err != nil { t.Fatal(err) } else if expected := int64(len(rightChunk)); expected != nn { t.Fatalf("wrong written bytes count: %v, expected: %v", nn, expected) } if out, err := rw.bs.GetBlob("a", "b"); err != nil { t.Fatal(err) } else { assertBlobContents(t, out, blob) } if bx, err := rw.bs.GetBlockList("a", "b", azure.BlockListTypeCommitted); err != nil { t.Fatal(err) } else if expected := 5; len(bx.CommittedBlocks) != expected { t.Fatalf("got wrong number of committed blocks: %v, expected: %v", len(bx.CommittedBlocks), expected) } // Replace 2K-4K (overlaps 2 blocks from L/R) newChunk := randomContents(1024 * 2) offset = 1024 * 2 blob = append(append(blob[:offset], newChunk...), blob[offset+int64(len(newChunk)):]...) if nn, err := rw.WriteBlobAt("a", "b", offset, bytes.NewReader(newChunk)); err != nil { t.Fatal(err) } else if expected := int64(len(newChunk)); expected != nn { t.Fatalf("wrong written bytes count: %v, expected: %v", nn, expected) } if out, err := rw.bs.GetBlob("a", "b"); err != nil { t.Fatal(err) } else { assertBlobContents(t, out, blob) } if bx, err := rw.bs.GetBlockList("a", "b", azure.BlockListTypeCommitted); err != nil { t.Fatal(err) } else if expected := 6; len(bx.CommittedBlocks) != expected { t.Fatalf("got wrong number of committed blocks: %v, expected: %v\n%v", len(bx.CommittedBlocks), expected, bx.CommittedBlocks) } // Replace the entire blob newBlob := randomContents(1024 * 30) if nn, err := rw.WriteBlobAt("a", "b", 0, bytes.NewReader(newBlob)); err != nil { t.Fatal(err) } else if expected := int64(len(newBlob)); expected != nn { t.Fatalf("wrong written bytes count: %v, expected: %v", nn, expected) } if out, err := rw.bs.GetBlob("a", "b"); err != nil { t.Fatal(err) } else { assertBlobContents(t, out, newBlob) } if bx, err := rw.bs.GetBlockList("a", "b", azure.BlockListTypeCommitted); err != nil { t.Fatal(err) } else if expected := 10; len(bx.CommittedBlocks) != expected { t.Fatalf("got wrong number of committed blocks: %v, expected: %v\n%v", len(bx.CommittedBlocks), expected, bx.CommittedBlocks) } else if expected, size := int64(1024*30), getBlobSize(bx); size != expected { t.Fatalf("committed block size does not indicate blob size") } } func Test_getBlobSize(t *testing.T) { // with some committed blocks if expected, size := int64(151), getBlobSize(azure.BlockListResponse{ CommittedBlocks: []azure.BlockResponse{ {"A", 100}, {"B", 50}, {"C", 1}, }, UncommittedBlocks: []azure.BlockResponse{ {"D", 200}, }}); expected != size { t.Fatalf("wrong blob size: %v, expected: %v", size, expected) } // with no committed blocks if expected, size := int64(0), getBlobSize(azure.BlockListResponse{ UncommittedBlocks: []azure.BlockResponse{ {"A", 100}, {"B", 50}, {"C", 1}, {"D", 200}, }}); expected != size { t.Fatalf("wrong blob size: %v, expected: %v", size, expected) } } func assertBlobContents(t *testing.T, r io.Reader, expected []byte) { out, err := ioutil.ReadAll(r) if err != nil { t.Fatal(err) } if !reflect.DeepEqual(out, expected) { t.Fatalf("wrong blob contents. size: %v, expected: %v", len(out), len(expected)) } } func randomContents(length int64) []byte { b := make([]byte, length) for i := range b { b[i] = byte(rand.Intn(2 << 8)) } return b } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/azure/azure.go0000644000175000017500000002231612502424227030175 0ustar tianontianon// Package azure provides a storagedriver.StorageDriver implementation to // store blobs in Microsoft Azure Blob Storage Service. package azure import ( "bytes" "fmt" "io" "io/ioutil" "net/http" "strings" "time" storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/distribution/registry/storage/driver/base" "github.com/docker/distribution/registry/storage/driver/factory" azure "github.com/MSOpenTech/azure-sdk-for-go/clients/storage" ) const driverName = "azure" const ( paramAccountName = "accountname" paramAccountKey = "accountkey" paramContainer = "container" ) type driver struct { client azure.BlobStorageClient container string } type baseEmbed struct{ base.Base } // Driver is a storagedriver.StorageDriver implementation backed by // Microsoft Azure Blob Storage Service. type Driver struct{ baseEmbed } func init() { factory.Register(driverName, &azureDriverFactory{}) } type azureDriverFactory struct{} func (factory *azureDriverFactory) Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error) { return FromParameters(parameters) } // FromParameters constructs a new Driver with a given parameters map. func FromParameters(parameters map[string]interface{}) (*Driver, error) { accountName, ok := parameters[paramAccountName] if !ok || fmt.Sprint(accountName) == "" { return nil, fmt.Errorf("No %s parameter provided", paramAccountName) } accountKey, ok := parameters[paramAccountKey] if !ok || fmt.Sprint(accountKey) == "" { return nil, fmt.Errorf("No %s parameter provided", paramAccountKey) } container, ok := parameters[paramContainer] if !ok || fmt.Sprint(container) == "" { return nil, fmt.Errorf("No %s parameter provided", paramContainer) } return New(fmt.Sprint(accountName), fmt.Sprint(accountKey), fmt.Sprint(container)) } // New constructs a new Driver with the given Azure Storage Account credentials func New(accountName, accountKey, container string) (*Driver, error) { api, err := azure.NewBasicClient(accountName, accountKey) if err != nil { return nil, err } blobClient := api.GetBlobService() // Create registry container if _, err = blobClient.CreateContainerIfNotExists(container, azure.ContainerAccessTypePrivate); err != nil { return nil, err } d := &driver{ client: *blobClient, container: container} return &Driver{baseEmbed: baseEmbed{Base: base.Base{StorageDriver: d}}}, nil } // Implement the storagedriver.StorageDriver interface. // GetContent retrieves the content stored at "path" as a []byte. func (d *driver) GetContent(path string) ([]byte, error) { blob, err := d.client.GetBlob(d.container, path) if err != nil { if is404(err) { return nil, storagedriver.PathNotFoundError{Path: path} } return nil, err } return ioutil.ReadAll(blob) } // PutContent stores the []byte content at a location designated by "path". func (d *driver) PutContent(path string, contents []byte) error { return d.client.PutBlockBlob(d.container, path, ioutil.NopCloser(bytes.NewReader(contents))) } // ReadStream retrieves an io.ReadCloser for the content stored at "path" with a // given byte offset. func (d *driver) ReadStream(path string, offset int64) (io.ReadCloser, error) { if ok, err := d.client.BlobExists(d.container, path); err != nil { return nil, err } else if !ok { return nil, storagedriver.PathNotFoundError{Path: path} } info, err := d.client.GetBlobProperties(d.container, path) if err != nil { return nil, err } size := int64(info.ContentLength) if offset >= size { return ioutil.NopCloser(bytes.NewReader(nil)), nil } bytesRange := fmt.Sprintf("%v-", offset) resp, err := d.client.GetBlobRange(d.container, path, bytesRange) if err != nil { return nil, err } return resp, nil } // WriteStream stores the contents of the provided io.ReadCloser at a location // designated by the given path. func (d *driver) WriteStream(path string, offset int64, reader io.Reader) (int64, error) { if blobExists, err := d.client.BlobExists(d.container, path); err != nil { return 0, err } else if !blobExists { err := d.client.CreateBlockBlob(d.container, path) if err != nil { return 0, err } } if offset < 0 { return 0, storagedriver.InvalidOffsetError{Path: path, Offset: offset} } bs := newAzureBlockStorage(d.client) bw := newRandomBlobWriter(&bs, azure.MaxBlobBlockSize) zw := newZeroFillWriter(&bw) return zw.Write(d.container, path, offset, reader) } // Stat retrieves the FileInfo for the given path, including the current size // in bytes and the creation time. func (d *driver) Stat(path string) (storagedriver.FileInfo, error) { // Check if the path is a blob if ok, err := d.client.BlobExists(d.container, path); err != nil { return nil, err } else if ok { blob, err := d.client.GetBlobProperties(d.container, path) if err != nil { return nil, err } mtim, err := time.Parse(http.TimeFormat, blob.LastModified) if err != nil { return nil, err } return storagedriver.FileInfoInternal{FileInfoFields: storagedriver.FileInfoFields{ Path: path, Size: int64(blob.ContentLength), ModTime: mtim, IsDir: false, }}, nil } // Check if path is a virtual container virtContainerPath := path if !strings.HasSuffix(virtContainerPath, "/") { virtContainerPath += "/" } blobs, err := d.client.ListBlobs(d.container, azure.ListBlobsParameters{ Prefix: virtContainerPath, MaxResults: 1, }) if err != nil { return nil, err } if len(blobs.Blobs) > 0 { // path is a virtual container return storagedriver.FileInfoInternal{FileInfoFields: storagedriver.FileInfoFields{ Path: path, IsDir: true, }}, nil } // path is not a blob or virtual container return nil, storagedriver.PathNotFoundError{Path: path} } // List returns a list of the objects that are direct descendants of the given // path. func (d *driver) List(path string) ([]string, error) { if path == "/" { path = "" } blobs, err := d.listBlobs(d.container, path) if err != nil { return blobs, err } list := directDescendants(blobs, path) return list, nil } // Move moves an object stored at sourcePath to destPath, removing the original // object. func (d *driver) Move(sourcePath string, destPath string) error { sourceBlobURL := d.client.GetBlobUrl(d.container, sourcePath) err := d.client.CopyBlob(d.container, destPath, sourceBlobURL) if err != nil { if is404(err) { return storagedriver.PathNotFoundError{Path: sourcePath} } return err } return d.client.DeleteBlob(d.container, sourcePath) } // Delete recursively deletes all objects stored at "path" and its subpaths. func (d *driver) Delete(path string) error { ok, err := d.client.DeleteBlobIfExists(d.container, path) if err != nil { return err } if ok { return nil // was a blob and deleted, return } // Not a blob, see if path is a virtual container with blobs blobs, err := d.listBlobs(d.container, path) if err != nil { return err } for _, b := range blobs { if err = d.client.DeleteBlob(d.container, b); err != nil { return err } } if len(blobs) == 0 { return storagedriver.PathNotFoundError{Path: path} } return nil } // URLFor returns a publicly accessible URL for the blob stored at given path // for specified duration by making use of Azure Storage Shared Access Signatures (SAS). // See https://msdn.microsoft.com/en-us/library/azure/ee395415.aspx for more info. func (d *driver) URLFor(path string, options map[string]interface{}) (string, error) { expiresTime := time.Now().UTC().Add(20 * time.Minute) // default expiration expires, ok := options["expiry"] if ok { t, ok := expires.(time.Time) if ok { expiresTime = t } } return d.client.GetBlobSASURI(d.container, path, expiresTime, "r") } // directDescendants will find direct descendants (blobs or virtual containers) // of from list of blob paths and will return their full paths. Elements in blobs // list must be prefixed with a "/" and // // Example: direct descendants of "/" in {"/foo", "/bar/1", "/bar/2"} is // {"/foo", "/bar"} and direct descendants of "bar" is {"/bar/1", "/bar/2"} func directDescendants(blobs []string, prefix string) []string { if !strings.HasPrefix(prefix, "/") { // add trailing '/' prefix = "/" + prefix } if !strings.HasSuffix(prefix, "/") { // containerify the path prefix += "/" } out := make(map[string]bool) for _, b := range blobs { if strings.HasPrefix(b, prefix) { rel := b[len(prefix):] c := strings.Count(rel, "/") if c == 0 { out[b] = true } else { out[prefix+rel[:strings.Index(rel, "/")]] = true } } } var keys []string for k := range out { keys = append(keys, k) } return keys } func (d *driver) listBlobs(container, virtPath string) ([]string, error) { if virtPath != "" && !strings.HasSuffix(virtPath, "/") { // containerify the path virtPath += "/" } out := []string{} marker := "" for { resp, err := d.client.ListBlobs(d.container, azure.ListBlobsParameters{ Marker: marker, Prefix: virtPath, }) if err != nil { return out, err } for _, b := range resp.Blobs { out = append(out, b.Name) } if len(resp.Blobs) == 0 || resp.NextMarker == "" { break } marker = resp.NextMarker } return out, nil } func is404(err error) bool { e, ok := err.(azure.StorageServiceError) return ok && e.StatusCode == 404 } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/azure/blockid.go0000644000175000017500000000241412502424227030453 0ustar tianontianonpackage azure import ( "encoding/base64" "fmt" "math/rand" "sync" "time" azure "github.com/MSOpenTech/azure-sdk-for-go/clients/storage" ) type blockIDGenerator struct { pool map[string]bool r *rand.Rand m sync.Mutex } // Generate returns an unused random block id and adds the generated ID // to list of used IDs so that the same block name is not used again. func (b *blockIDGenerator) Generate() string { b.m.Lock() defer b.m.Unlock() var id string for { id = toBlockID(int(b.r.Int())) if !b.exists(id) { break } } b.pool[id] = true return id } func (b *blockIDGenerator) exists(id string) bool { _, used := b.pool[id] return used } func (b *blockIDGenerator) Feed(blocks azure.BlockListResponse) { b.m.Lock() defer b.m.Unlock() for _, bl := range append(blocks.CommittedBlocks, blocks.UncommittedBlocks...) { b.pool[bl.Name] = true } } func newBlockIDGenerator() *blockIDGenerator { return &blockIDGenerator{ pool: make(map[string]bool), r: rand.New(rand.NewSource(time.Now().UnixNano()))} } // toBlockId converts given integer to base64-encoded block ID of a fixed length. func toBlockID(i int) string { s := fmt.Sprintf("%029d", i) // add zero padding for same length-blobs return base64.StdEncoding.EncodeToString([]byte(s)) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/azure/randomwriter.go0000644000175000017500000001541212502424227031563 0ustar tianontianonpackage azure import ( "fmt" "io" "io/ioutil" azure "github.com/MSOpenTech/azure-sdk-for-go/clients/storage" ) // blockStorage is the interface required from a block storage service // client implementation type blockStorage interface { CreateBlockBlob(container, blob string) error GetBlob(container, blob string) (io.ReadCloser, error) GetSectionReader(container, blob string, start, length int64) (io.ReadCloser, error) PutBlock(container, blob, blockID string, chunk []byte) error GetBlockList(container, blob string, blockType azure.BlockListType) (azure.BlockListResponse, error) PutBlockList(container, blob string, blocks []azure.Block) error } // randomBlobWriter enables random access semantics on Azure block blobs // by enabling writing arbitrary length of chunks to arbitrary write offsets // within the blob. Normally, Azure Blob Storage does not support random // access semantics on block blobs; however, this writer can download, split and // reupload the overlapping blocks and discards those being overwritten entirely. type randomBlobWriter struct { bs blockStorage blockSize int } func newRandomBlobWriter(bs blockStorage, blockSize int) randomBlobWriter { return randomBlobWriter{bs: bs, blockSize: blockSize} } // WriteBlobAt writes the given chunk to the specified position of an existing blob. // The offset must be equals to size of the blob or smaller than it. func (r *randomBlobWriter) WriteBlobAt(container, blob string, offset int64, chunk io.Reader) (int64, error) { rand := newBlockIDGenerator() blocks, err := r.bs.GetBlockList(container, blob, azure.BlockListTypeCommitted) if err != nil { return 0, err } rand.Feed(blocks) // load existing block IDs // Check for write offset for existing blob size := getBlobSize(blocks) if offset < 0 || offset > size { return 0, fmt.Errorf("wrong offset for Write: %v", offset) } // Upload the new chunk as blocks blockList, nn, err := r.writeChunkToBlocks(container, blob, chunk, rand) if err != nil { return 0, err } // For non-append operations, existing blocks may need to be splitted if offset != size { // Split the block on the left end (if any) leftBlocks, err := r.blocksLeftSide(container, blob, offset, rand) if err != nil { return 0, err } blockList = append(leftBlocks, blockList...) // Split the block on the right end (if any) rightBlocks, err := r.blocksRightSide(container, blob, offset, nn, rand) if err != nil { return 0, err } blockList = append(blockList, rightBlocks...) } else { // Use existing block list var existingBlocks []azure.Block for _, v := range blocks.CommittedBlocks { existingBlocks = append(existingBlocks, azure.Block{Id: v.Name, Status: azure.BlockStatusCommitted}) } blockList = append(existingBlocks, blockList...) } // Put block list return nn, r.bs.PutBlockList(container, blob, blockList) } func (r *randomBlobWriter) GetSize(container, blob string) (int64, error) { blocks, err := r.bs.GetBlockList(container, blob, azure.BlockListTypeCommitted) if err != nil { return 0, err } return getBlobSize(blocks), nil } // writeChunkToBlocks writes given chunk to one or multiple blocks within specified // blob and returns their block representations. Those blocks are not committed, yet func (r *randomBlobWriter) writeChunkToBlocks(container, blob string, chunk io.Reader, rand *blockIDGenerator) ([]azure.Block, int64, error) { var newBlocks []azure.Block var nn int64 // Read chunks of at most size N except the last chunk to // maximize block size and minimize block count. buf := make([]byte, r.blockSize) for { n, err := io.ReadFull(chunk, buf) if err == io.EOF { break } nn += int64(n) data := buf[:n] blockID := rand.Generate() if err := r.bs.PutBlock(container, blob, blockID, data); err != nil { return newBlocks, nn, err } newBlocks = append(newBlocks, azure.Block{Id: blockID, Status: azure.BlockStatusUncommitted}) } return newBlocks, nn, nil } // blocksLeftSide returns the blocks that are going to be at the left side of // the writeOffset: [0, writeOffset) by identifying blocks that will remain // the same and splitting blocks and reuploading them as needed. func (r *randomBlobWriter) blocksLeftSide(container, blob string, writeOffset int64, rand *blockIDGenerator) ([]azure.Block, error) { var left []azure.Block bx, err := r.bs.GetBlockList(container, blob, azure.BlockListTypeAll) if err != nil { return left, err } o := writeOffset elapsed := int64(0) for _, v := range bx.CommittedBlocks { blkSize := int64(v.Size) if o >= blkSize { // use existing block left = append(left, azure.Block{Id: v.Name, Status: azure.BlockStatusCommitted}) o -= blkSize elapsed += blkSize } else if o > 0 { // current block needs to be splitted start := elapsed size := o part, err := r.bs.GetSectionReader(container, blob, start, size) if err != nil { return left, err } newBlockID := rand.Generate() data, err := ioutil.ReadAll(part) if err != nil { return left, err } if err = r.bs.PutBlock(container, blob, newBlockID, data); err != nil { return left, err } left = append(left, azure.Block{Id: newBlockID, Status: azure.BlockStatusUncommitted}) break } } return left, nil } // blocksRightSide returns the blocks that are going to be at the right side of // the written chunk: [writeOffset+size, +inf) by identifying blocks that will remain // the same and splitting blocks and reuploading them as needed. func (r *randomBlobWriter) blocksRightSide(container, blob string, writeOffset int64, chunkSize int64, rand *blockIDGenerator) ([]azure.Block, error) { var right []azure.Block bx, err := r.bs.GetBlockList(container, blob, azure.BlockListTypeAll) if err != nil { return nil, err } re := writeOffset + chunkSize - 1 // right end of written chunk var elapsed int64 for _, v := range bx.CommittedBlocks { var ( bs = elapsed // left end of current block be = elapsed + int64(v.Size) - 1 // right end of current block ) if bs > re { // take the block as is right = append(right, azure.Block{Id: v.Name, Status: azure.BlockStatusCommitted}) } else if be > re { // current block needs to be splitted part, err := r.bs.GetSectionReader(container, blob, re+1, be-(re+1)+1) if err != nil { return right, err } newBlockID := rand.Generate() data, err := ioutil.ReadAll(part) if err != nil { return right, err } if err = r.bs.PutBlock(container, blob, newBlockID, data); err != nil { return right, err } right = append(right, azure.Block{Id: newBlockID, Status: azure.BlockStatusUncommitted}) } elapsed += int64(v.Size) } return right, nil } func getBlobSize(blocks azure.BlockListResponse) int64 { var n int64 for _, v := range blocks.CommittedBlocks { n += int64(v.Size) } return n } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/azure/azure_test.go0000644000175000017500000000274312502424227031236 0ustar tianontianonpackage azure import ( "fmt" "os" "strings" "testing" storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/distribution/registry/storage/driver/testsuites" . "gopkg.in/check.v1" ) const ( envAccountName = "AZURE_STORAGE_ACCOUNT_NAME" envAccountKey = "AZURE_STORAGE_ACCOUNT_KEY" envContainer = "AZURE_STORAGE_CONTAINER" ) // Hook up gocheck into the "go test" runner. func Test(t *testing.T) { TestingT(t) } func init() { var ( accountName string accountKey string container string ) config := []struct { env string value *string }{ {envAccountName, &accountName}, {envAccountKey, &accountKey}, {envContainer, &container}, } missing := []string{} for _, v := range config { *v.value = os.Getenv(v.env) if *v.value == "" { missing = append(missing, v.env) } } azureDriverConstructor := func() (storagedriver.StorageDriver, error) { return New(accountName, accountKey, container) } // Skip Azure storage driver tests if environment variable parameters are not provided skipCheck := func() string { if len(missing) > 0 { return fmt.Sprintf("Must set %s environment variables to run Azure tests", strings.Join(missing, ", ")) } return "" } testsuites.RegisterInProcessSuite(azureDriverConstructor, skipCheck) // testsuites.RegisterIPCSuite(driverName, map[string]string{ // paramAccountName: accountName, // paramAccountKey: accountKey, // paramContainer: container, // }, skipCheck) } ././@LongLink0000644000000000000000000000014600000000000011604 Lustar rootrootdistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/azure/blockblob_test.godistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/azure/blockblob_test.g0000644000175000017500000000713212502424227031657 0ustar tianontianonpackage azure import ( "bytes" "fmt" "io" "io/ioutil" azure "github.com/MSOpenTech/azure-sdk-for-go/clients/storage" ) type StorageSimulator struct { blobs map[string]*BlockBlob } type BlockBlob struct { blocks map[string]*DataBlock blockList []string } type DataBlock struct { data []byte committed bool } func (s *StorageSimulator) path(container, blob string) string { return fmt.Sprintf("%s/%s", container, blob) } func (s *StorageSimulator) BlobExists(container, blob string) (bool, error) { _, ok := s.blobs[s.path(container, blob)] return ok, nil } func (s *StorageSimulator) GetBlob(container, blob string) (io.ReadCloser, error) { bb, ok := s.blobs[s.path(container, blob)] if !ok { return nil, fmt.Errorf("blob not found") } var readers []io.Reader for _, bID := range bb.blockList { readers = append(readers, bytes.NewReader(bb.blocks[bID].data)) } return ioutil.NopCloser(io.MultiReader(readers...)), nil } func (s *StorageSimulator) GetSectionReader(container, blob string, start, length int64) (io.ReadCloser, error) { r, err := s.GetBlob(container, blob) if err != nil { return nil, err } b, err := ioutil.ReadAll(r) if err != nil { return nil, err } return ioutil.NopCloser(bytes.NewReader(b[start : start+length])), nil } func (s *StorageSimulator) CreateBlockBlob(container, blob string) error { path := s.path(container, blob) bb := &BlockBlob{ blocks: make(map[string]*DataBlock), blockList: []string{}, } s.blobs[path] = bb return nil } func (s *StorageSimulator) PutBlock(container, blob, blockID string, chunk []byte) error { path := s.path(container, blob) bb, ok := s.blobs[path] if !ok { return fmt.Errorf("blob not found") } data := make([]byte, len(chunk)) copy(data, chunk) bb.blocks[blockID] = &DataBlock{data: data, committed: false} // add block to blob return nil } func (s *StorageSimulator) GetBlockList(container, blob string, blockType azure.BlockListType) (azure.BlockListResponse, error) { resp := azure.BlockListResponse{} bb, ok := s.blobs[s.path(container, blob)] if !ok { return resp, fmt.Errorf("blob not found") } // Iterate committed blocks (in order) if blockType == azure.BlockListTypeAll || blockType == azure.BlockListTypeCommitted { for _, blockID := range bb.blockList { b := bb.blocks[blockID] block := azure.BlockResponse{ Name: blockID, Size: int64(len(b.data)), } resp.CommittedBlocks = append(resp.CommittedBlocks, block) } } // Iterate uncommitted blocks (in no order) if blockType == azure.BlockListTypeAll || blockType == azure.BlockListTypeCommitted { for blockID, b := range bb.blocks { block := azure.BlockResponse{ Name: blockID, Size: int64(len(b.data)), } if !b.committed { resp.UncommittedBlocks = append(resp.UncommittedBlocks, block) } } } return resp, nil } func (s *StorageSimulator) PutBlockList(container, blob string, blocks []azure.Block) error { bb, ok := s.blobs[s.path(container, blob)] if !ok { return fmt.Errorf("blob not found") } var blockIDs []string for _, v := range blocks { bl, ok := bb.blocks[v.Id] if !ok { // check if block ID exists return fmt.Errorf("Block id '%s' not found", v.Id) } bl.committed = true blockIDs = append(blockIDs, v.Id) } // Mark all other blocks uncommitted for k, b := range bb.blocks { inList := false for _, v := range blockIDs { if k == v { inList = true break } } if !inList { b.committed = false } } bb.blockList = blockIDs return nil } func NewStorageSimulator() StorageSimulator { return StorageSimulator{ blobs: make(map[string]*BlockBlob), } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/azure/blockblob.go0000644000175000017500000000115112502424227030772 0ustar tianontianonpackage azure import ( "fmt" "io" azure "github.com/MSOpenTech/azure-sdk-for-go/clients/storage" ) // azureBlockStorage is adaptor between azure.BlobStorageClient and // blockStorage interface. type azureBlockStorage struct { azure.BlobStorageClient } func (b *azureBlockStorage) GetSectionReader(container, blob string, start, length int64) (io.ReadCloser, error) { return b.BlobStorageClient.GetBlobRange(container, blob, fmt.Sprintf("%v-%v", start, start+length-1)) } func newAzureBlockStorage(b azure.BlobStorageClient) azureBlockStorage { a := azureBlockStorage{} a.BlobStorageClient = b return a } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/azure/blockid_test.go0000644000175000017500000000355112502424227031515 0ustar tianontianonpackage azure import ( "math" "testing" azure "github.com/MSOpenTech/azure-sdk-for-go/clients/storage" ) func Test_blockIdGenerator(t *testing.T) { r := newBlockIDGenerator() for i := 1; i <= 10; i++ { if expected := i - 1; len(r.pool) != expected { t.Fatalf("rand pool had wrong number of items: %d, expected:%d", len(r.pool), expected) } if id := r.Generate(); id == "" { t.Fatal("returned empty id") } if expected := i; len(r.pool) != expected { t.Fatalf("rand pool has wrong number of items: %d, expected:%d", len(r.pool), expected) } } } func Test_blockIdGenerator_Feed(t *testing.T) { r := newBlockIDGenerator() if expected := 0; len(r.pool) != expected { t.Fatalf("rand pool had wrong number of items: %d, expected:%d", len(r.pool), expected) } // feed empty list blocks := azure.BlockListResponse{} r.Feed(blocks) if expected := 0; len(r.pool) != expected { t.Fatalf("rand pool had wrong number of items: %d, expected:%d", len(r.pool), expected) } // feed blocks blocks = azure.BlockListResponse{ CommittedBlocks: []azure.BlockResponse{ {"1", 1}, {"2", 2}, }, UncommittedBlocks: []azure.BlockResponse{ {"3", 3}, }} r.Feed(blocks) if expected := 3; len(r.pool) != expected { t.Fatalf("rand pool had wrong number of items: %d, expected:%d", len(r.pool), expected) } // feed same block IDs with committed/uncommitted place changed blocks = azure.BlockListResponse{ CommittedBlocks: []azure.BlockResponse{ {"3", 3}, }, UncommittedBlocks: []azure.BlockResponse{ {"1", 1}, }} r.Feed(blocks) if expected := 3; len(r.pool) != expected { t.Fatalf("rand pool had wrong number of items: %d, expected:%d", len(r.pool), expected) } } func Test_toBlockId(t *testing.T) { min := 0 max := math.MaxInt64 if len(toBlockID(min)) != len(toBlockID(max)) { t.Fatalf("different-sized blockIDs are returned") } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/storagedriver.go0000644000175000017500000001011512502424227030573 0ustar tianontianonpackage driver import ( "errors" "fmt" "io" "regexp" "strconv" "strings" ) // Version is a string representing the storage driver version, of the form // Major.Minor. // The registry must accept storage drivers with equal major version and greater // minor version, but may not be compatible with older storage driver versions. type Version string // Major returns the major (primary) component of a version. func (version Version) Major() uint { majorPart := strings.Split(string(version), ".")[0] major, _ := strconv.ParseUint(majorPart, 10, 0) return uint(major) } // Minor returns the minor (secondary) component of a version. func (version Version) Minor() uint { minorPart := strings.Split(string(version), ".")[1] minor, _ := strconv.ParseUint(minorPart, 10, 0) return uint(minor) } // CurrentVersion is the current storage driver Version. const CurrentVersion Version = "0.1" // StorageDriver defines methods that a Storage Driver must implement for a // filesystem-like key/value object storage. type StorageDriver interface { // GetContent retrieves the content stored at "path" as a []byte. // This should primarily be used for small objects. GetContent(path string) ([]byte, error) // PutContent stores the []byte content at a location designated by "path". // This should primarily be used for small objects. PutContent(path string, content []byte) error // ReadStream retrieves an io.ReadCloser for the content stored at "path" // with a given byte offset. // May be used to resume reading a stream by providing a nonzero offset. ReadStream(path string, offset int64) (io.ReadCloser, error) // WriteStream stores the contents of the provided io.ReadCloser at a // location designated by the given path. // May be used to resume writing a stream by providing a nonzero offset. // The offset must be no larger than the CurrentSize for this path. WriteStream(path string, offset int64, reader io.Reader) (nn int64, err error) // Stat retrieves the FileInfo for the given path, including the current // size in bytes and the creation time. Stat(path string) (FileInfo, error) // List returns a list of the objects that are direct descendants of the //given path. List(path string) ([]string, error) // Move moves an object stored at sourcePath to destPath, removing the // original object. // Note: This may be no more efficient than a copy followed by a delete for // many implementations. Move(sourcePath string, destPath string) error // Delete recursively deletes all objects stored at "path" and its subpaths. Delete(path string) error // URLFor returns a URL which may be used to retrieve the content stored at // the given path, possibly using the given options. // May return an ErrUnsupportedMethod in certain StorageDriver // implementations. URLFor(path string, options map[string]interface{}) (string, error) } // PathRegexp is the regular expression which each file path must match. A // file path is absolute, beginning with a slash and containing a positive // number of path components separated by slashes, where each component is // restricted to lowercase alphanumeric characters or a period, underscore, or // hyphen. var PathRegexp = regexp.MustCompile(`^(/[a-z0-9._-]+)+$`) // ErrUnsupportedMethod may be returned in the case where a StorageDriver implementation does not support an optional method. var ErrUnsupportedMethod = errors.New("unsupported method") // PathNotFoundError is returned when operating on a nonexistent path. type PathNotFoundError struct { Path string } func (err PathNotFoundError) Error() string { return fmt.Sprintf("Path not found: %s", err.Path) } // InvalidPathError is returned when the provided path is malformed. type InvalidPathError struct { Path string } func (err InvalidPathError) Error() string { return fmt.Sprintf("Invalid path: %s", err.Path) } // InvalidOffsetError is returned when attempting to read or write from an // invalid offset. type InvalidOffsetError struct { Path string Offset int64 } func (err InvalidOffsetError) Error() string { return fmt.Sprintf("Invalid offset: %d for path: %s", err.Offset, err.Path) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/ipc/0000755000175000017500000000000012502424227026141 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/ipc/ipc.go0000644000175000017500000001023412502424227027243 0ustar tianontianon// +build ignore package ipc import ( "fmt" "io" "reflect" storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/libchan" ) // StorageDriver is the interface which IPC storage drivers must implement. As external storage // drivers may be defined to use a different version of the storagedriver.StorageDriver interface, // we use an additional version check to determine compatiblity. type StorageDriver interface { // Version returns the storagedriver.StorageDriver interface version which this storage driver // implements, which is used to determine driver compatibility Version() (storagedriver.Version, error) } // IncompatibleVersionError is returned when a storage driver is using an incompatible version of // the storagedriver.StorageDriver api type IncompatibleVersionError struct { version storagedriver.Version } func (e IncompatibleVersionError) Error() string { return fmt.Sprintf("Incompatible storage driver version: %s", e.version) } // Request defines a remote method call request // A return value struct is to be sent over the ResponseChannel type Request struct { Type string `codec:",omitempty"` Parameters map[string]interface{} `codec:",omitempty"` ResponseChannel libchan.Sender `codec:",omitempty"` } // ResponseError is a serializable error type. // The Type and Parameters may be used to reconstruct the same error on the // client side, falling back to using the Type and Message if this cannot be // done. type ResponseError struct { Type string `codec:",omitempty"` Message string `codec:",omitempty"` Parameters map[string]interface{} `codec:",omitempty"` } // WrapError wraps an error in a serializable struct containing the error's type // and message. func WrapError(err error) *ResponseError { if err == nil { return nil } v := reflect.ValueOf(err) re := ResponseError{ Type: v.Type().String(), Message: err.Error(), } if v.Kind() == reflect.Struct { re.Parameters = make(map[string]interface{}) for i := 0; i < v.NumField(); i++ { field := v.Type().Field(i) re.Parameters[field.Name] = v.Field(i).Interface() } } return &re } // Unwrap returns the underlying error if it can be reconstructed, or the // original ResponseError otherwise. func (err *ResponseError) Unwrap() error { var errVal reflect.Value var zeroVal reflect.Value switch err.Type { case "storagedriver.PathNotFoundError": errVal = reflect.ValueOf(&storagedriver.PathNotFoundError{}) case "storagedriver.InvalidOffsetError": errVal = reflect.ValueOf(&storagedriver.InvalidOffsetError{}) } if errVal == zeroVal { return err } for k, v := range err.Parameters { fieldVal := errVal.Elem().FieldByName(k) if fieldVal == zeroVal { return err } fieldVal.Set(reflect.ValueOf(v)) } if unwrapped, ok := errVal.Elem().Interface().(error); ok { return unwrapped } return err } func (err *ResponseError) Error() string { return fmt.Sprintf("%s: %s", err.Type, err.Message) } // IPC method call response object definitions // VersionResponse is a response for a Version request type VersionResponse struct { Version storagedriver.Version `codec:",omitempty"` Error *ResponseError `codec:",omitempty"` } // ReadStreamResponse is a response for a ReadStream request type ReadStreamResponse struct { Reader io.ReadCloser `codec:",omitempty"` Error *ResponseError `codec:",omitempty"` } // WriteStreamResponse is a response for a WriteStream request type WriteStreamResponse struct { Error *ResponseError `codec:",omitempty"` } // CurrentSizeResponse is a response for a CurrentSize request type CurrentSizeResponse struct { Position uint64 `codec:",omitempty"` Error *ResponseError `codec:",omitempty"` } // ListResponse is a response for a List request type ListResponse struct { Keys []string `codec:",omitempty"` Error *ResponseError `codec:",omitempty"` } // MoveResponse is a response for a Move request type MoveResponse struct { Error *ResponseError `codec:",omitempty"` } // DeleteResponse is a response for a Delete request type DeleteResponse struct { Error *ResponseError `codec:",omitempty"` } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/ipc/client.go0000644000175000017500000002657012502424227027760 0ustar tianontianon// +build ignore package ipc import ( "bytes" "encoding/json" "fmt" "io" "io/ioutil" "net" "os" "os/exec" "syscall" storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/libchan" "github.com/docker/libchan/spdy" ) // StorageDriverExecutablePrefix is the prefix which the IPC storage driver // loader expects driver executables to begin with. For example, the s3 driver // should be named "registry-storagedriver-s3". const StorageDriverExecutablePrefix = "registry-storagedriver-" // StorageDriverClient is a storagedriver.StorageDriver implementation using a // managed child process communicating over IPC using libchan with a unix domain // socket type StorageDriverClient struct { subprocess *exec.Cmd exitChan chan error exitErr error stopChan chan struct{} socket *os.File transport *spdy.Transport sender libchan.Sender version storagedriver.Version } // NewDriverClient constructs a new out-of-process storage driver using the // driver name and configuration parameters // A user must call Start on this driver client before remote method calls can // be made // // Looks for drivers in the following locations in order: // - Storage drivers directory (to be determined, yet not implemented) // - $GOPATH/bin // - $PATH func NewDriverClient(name string, parameters map[string]string) (*StorageDriverClient, error) { paramsBytes, err := json.Marshal(parameters) if err != nil { return nil, err } driverExecName := StorageDriverExecutablePrefix + name driverPath, err := exec.LookPath(driverExecName) if err != nil { return nil, err } command := exec.Command(driverPath, string(paramsBytes)) return &StorageDriverClient{ subprocess: command, }, nil } // Start starts the designated child process storage driver and binds a socket // to this process for IPC method calls func (driver *StorageDriverClient) Start() error { driver.exitErr = nil driver.exitChan = make(chan error) driver.stopChan = make(chan struct{}) fileDescriptors, err := syscall.Socketpair(syscall.AF_LOCAL, syscall.SOCK_STREAM, 0) if err != nil { return err } childSocket := os.NewFile(uintptr(fileDescriptors[0]), "childSocket") driver.socket = os.NewFile(uintptr(fileDescriptors[1]), "parentSocket") driver.subprocess.Stdout = os.Stdout driver.subprocess.Stderr = os.Stderr driver.subprocess.ExtraFiles = []*os.File{childSocket} if err = driver.subprocess.Start(); err != nil { driver.Stop() return err } go driver.handleSubprocessExit() if err = childSocket.Close(); err != nil { driver.Stop() return err } connection, err := net.FileConn(driver.socket) if err != nil { driver.Stop() return err } driver.transport, err = spdy.NewClientTransport(connection) if err != nil { driver.Stop() return err } driver.sender, err = driver.transport.NewSendChannel() if err != nil { driver.Stop() return err } // Check the driver's version to determine compatibility receiver, remoteSender := libchan.Pipe() err = driver.sender.Send(&Request{Type: "Version", ResponseChannel: remoteSender}) if err != nil { driver.Stop() return err } var response VersionResponse err = receiver.Receive(&response) if err != nil { driver.Stop() return err } if response.Error != nil { return response.Error.Unwrap() } driver.version = response.Version if driver.version.Major() != storagedriver.CurrentVersion.Major() || driver.version.Minor() > storagedriver.CurrentVersion.Minor() { return IncompatibleVersionError{driver.version} } return nil } // Stop stops the child process storage driver // storagedriver.StorageDriver methods called after Stop will fail func (driver *StorageDriverClient) Stop() error { var closeSenderErr, closeTransportErr, closeSocketErr, killErr error if driver.sender != nil { closeSenderErr = driver.sender.Close() } if driver.transport != nil { closeTransportErr = driver.transport.Close() } if driver.socket != nil { closeSocketErr = driver.socket.Close() } if driver.subprocess != nil { killErr = driver.subprocess.Process.Kill() } if driver.stopChan != nil { close(driver.stopChan) } if closeSenderErr != nil { return closeSenderErr } else if closeTransportErr != nil { return closeTransportErr } else if closeSocketErr != nil { return closeSocketErr } return killErr } // Implement the storagedriver.StorageDriver interface over IPC // GetContent retrieves the content stored at "path" as a []byte. func (driver *StorageDriverClient) GetContent(path string) ([]byte, error) { if err := driver.exited(); err != nil { return nil, err } receiver, remoteSender := libchan.Pipe() params := map[string]interface{}{"Path": path} err := driver.sender.Send(&Request{Type: "GetContent", Parameters: params, ResponseChannel: remoteSender}) if err != nil { return nil, err } response := new(ReadStreamResponse) err = driver.receiveResponse(receiver, response) if err != nil { return nil, err } if response.Error != nil { return nil, response.Error.Unwrap() } defer response.Reader.Close() contents, err := ioutil.ReadAll(response.Reader) if err != nil { return nil, err } return contents, nil } // PutContent stores the []byte content at a location designated by "path". func (driver *StorageDriverClient) PutContent(path string, contents []byte) error { if err := driver.exited(); err != nil { return err } receiver, remoteSender := libchan.Pipe() params := map[string]interface{}{"Path": path, "Reader": ioutil.NopCloser(bytes.NewReader(contents))} err := driver.sender.Send(&Request{Type: "PutContent", Parameters: params, ResponseChannel: remoteSender}) if err != nil { return err } response := new(WriteStreamResponse) err = driver.receiveResponse(receiver, response) if err != nil { return err } if response.Error != nil { return response.Error.Unwrap() } return nil } // ReadStream retrieves an io.ReadCloser for the content stored at "path" with a // given byte offset. func (driver *StorageDriverClient) ReadStream(path string, offset int64) (io.ReadCloser, error) { if err := driver.exited(); err != nil { return nil, err } receiver, remoteSender := libchan.Pipe() params := map[string]interface{}{"Path": path, "Offset": offset} err := driver.sender.Send(&Request{Type: "ReadStream", Parameters: params, ResponseChannel: remoteSender}) if err != nil { return nil, err } response := new(ReadStreamResponse) err = driver.receiveResponse(receiver, response) if err != nil { return nil, err } if response.Error != nil { return nil, response.Error.Unwrap() } return response.Reader, nil } // WriteStream stores the contents of the provided io.ReadCloser at a location // designated by the given path. func (driver *StorageDriverClient) WriteStream(path string, offset, size int64, reader io.ReadCloser) error { if err := driver.exited(); err != nil { return err } receiver, remoteSender := libchan.Pipe() params := map[string]interface{}{"Path": path, "Offset": offset, "Size": size, "Reader": reader} err := driver.sender.Send(&Request{Type: "WriteStream", Parameters: params, ResponseChannel: remoteSender}) if err != nil { return err } response := new(WriteStreamResponse) err = driver.receiveResponse(receiver, response) if err != nil { return err } if response.Error != nil { return response.Error.Unwrap() } return nil } // CurrentSize retrieves the curernt size in bytes of the object at the given // path. func (driver *StorageDriverClient) CurrentSize(path string) (uint64, error) { if err := driver.exited(); err != nil { return 0, err } receiver, remoteSender := libchan.Pipe() params := map[string]interface{}{"Path": path} err := driver.sender.Send(&Request{Type: "CurrentSize", Parameters: params, ResponseChannel: remoteSender}) if err != nil { return 0, err } response := new(CurrentSizeResponse) err = driver.receiveResponse(receiver, response) if err != nil { return 0, err } if response.Error != nil { return 0, response.Error.Unwrap() } return response.Position, nil } // List returns a list of the objects that are direct descendants of the given // path. func (driver *StorageDriverClient) List(path string) ([]string, error) { if err := driver.exited(); err != nil { return nil, err } receiver, remoteSender := libchan.Pipe() params := map[string]interface{}{"Path": path} err := driver.sender.Send(&Request{Type: "List", Parameters: params, ResponseChannel: remoteSender}) if err != nil { return nil, err } response := new(ListResponse) err = driver.receiveResponse(receiver, response) if err != nil { return nil, err } if response.Error != nil { return nil, response.Error.Unwrap() } return response.Keys, nil } // Move moves an object stored at sourcePath to destPath, removing the original // object. func (driver *StorageDriverClient) Move(sourcePath string, destPath string) error { if err := driver.exited(); err != nil { return err } receiver, remoteSender := libchan.Pipe() params := map[string]interface{}{"SourcePath": sourcePath, "DestPath": destPath} err := driver.sender.Send(&Request{Type: "Move", Parameters: params, ResponseChannel: remoteSender}) if err != nil { return err } response := new(MoveResponse) err = driver.receiveResponse(receiver, response) if err != nil { return err } if response.Error != nil { return response.Error.Unwrap() } return nil } // Delete recursively deletes all objects stored at "path" and its subpaths. func (driver *StorageDriverClient) Delete(path string) error { if err := driver.exited(); err != nil { return err } receiver, remoteSender := libchan.Pipe() params := map[string]interface{}{"Path": path} err := driver.sender.Send(&Request{Type: "Delete", Parameters: params, ResponseChannel: remoteSender}) if err != nil { return err } response := new(DeleteResponse) err = driver.receiveResponse(receiver, response) if err != nil { return err } if response.Error != nil { return response.Error.Unwrap() } return nil } // handleSubprocessExit populates the exit channel until we have explicitly // stopped the storage driver subprocess // Requests can select on driver.exitChan and response receiving and not hang if // the process exits func (driver *StorageDriverClient) handleSubprocessExit() { exitErr := driver.subprocess.Wait() if exitErr == nil { exitErr = fmt.Errorf("Storage driver subprocess already exited cleanly") } else { exitErr = fmt.Errorf("Storage driver subprocess exited with error: %s", exitErr) } driver.exitErr = exitErr for { select { case driver.exitChan <- exitErr: case <-driver.stopChan: close(driver.exitChan) return } } } // receiveResponse populates the response value with the next result from the // given receiver, or returns an error if receiving failed or the driver has // stopped func (driver *StorageDriverClient) receiveResponse(receiver libchan.Receiver, response interface{}) error { receiveChan := make(chan error, 1) go func(receiver libchan.Receiver, receiveChan chan<- error) { receiveChan <- receiver.Receive(response) }(receiver, receiveChan) var err error var ok bool select { case err = <-receiveChan: case err, ok = <-driver.exitChan: if !ok { err = driver.exitErr } } return err } // exited returns an exit error if the driver has exited or nil otherwise func (driver *StorageDriverClient) exited() error { select { case err, ok := <-driver.exitChan: if !ok { return driver.exitErr } return err default: return nil } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/ipc/server.go0000644000175000017500000001210712502424227027777 0ustar tianontianon// +build ignore package ipc import ( "bytes" "io" "io/ioutil" "net" "os" "reflect" storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/libchan" "github.com/docker/libchan/spdy" ) // StorageDriverServer runs a new IPC server handling requests for the given // storagedriver.StorageDriver // This explicitly uses file descriptor 3 for IPC communication, as storage drivers are spawned in // client.go // // To create a new out-of-process driver, create a main package which calls StorageDriverServer with // a storagedriver.StorageDriver func StorageDriverServer(driver storagedriver.StorageDriver) error { childSocket := os.NewFile(3, "childSocket") defer childSocket.Close() conn, err := net.FileConn(childSocket) if err != nil { panic(err) } defer conn.Close() if transport, err := spdy.NewServerTransport(conn); err != nil { panic(err) } else { for { receiver, err := transport.WaitReceiveChannel() if err == io.EOF { return nil } else if err != nil { panic(err) } go receive(driver, receiver) } } } // receive receives new storagedriver.StorageDriver method requests and creates a new goroutine to // handle each request // Requests are expected to be of type ipc.Request as the parameters are unknown until the request // type is deserialized func receive(driver storagedriver.StorageDriver, receiver libchan.Receiver) { for { var request Request err := receiver.Receive(&request) if err == io.EOF { return } else if err != nil { panic(err) } go handleRequest(driver, request) } } // handleRequest handles storagedriver.StorageDriver method requests as defined in client.go // Responds to requests using the Request.ResponseChannel func handleRequest(driver storagedriver.StorageDriver, request Request) { switch request.Type { case "Version": err := request.ResponseChannel.Send(&VersionResponse{Version: storagedriver.CurrentVersion}) if err != nil { panic(err) } case "GetContent": path, _ := request.Parameters["Path"].(string) content, err := driver.GetContent(path) var response ReadStreamResponse if err != nil { response = ReadStreamResponse{Error: WrapError(err)} } else { response = ReadStreamResponse{Reader: ioutil.NopCloser(bytes.NewReader(content))} } err = request.ResponseChannel.Send(&response) if err != nil { panic(err) } case "PutContent": path, _ := request.Parameters["Path"].(string) reader, _ := request.Parameters["Reader"].(io.ReadCloser) contents, err := ioutil.ReadAll(reader) defer reader.Close() if err == nil { err = driver.PutContent(path, contents) } response := WriteStreamResponse{ Error: WrapError(err), } err = request.ResponseChannel.Send(&response) if err != nil { panic(err) } case "ReadStream": path, _ := request.Parameters["Path"].(string) // Depending on serialization method, Offset may be convereted to any int/uint type offset := reflect.ValueOf(request.Parameters["Offset"]).Convert(reflect.TypeOf(int64(0))).Int() reader, err := driver.ReadStream(path, offset) var response ReadStreamResponse if err != nil { response = ReadStreamResponse{Error: WrapError(err)} } else { response = ReadStreamResponse{Reader: reader} } err = request.ResponseChannel.Send(&response) if err != nil { panic(err) } case "WriteStream": path, _ := request.Parameters["Path"].(string) // Depending on serialization method, Offset may be convereted to any int/uint type offset := reflect.ValueOf(request.Parameters["Offset"]).Convert(reflect.TypeOf(int64(0))).Int() // Depending on serialization method, Size may be convereted to any int/uint type size := reflect.ValueOf(request.Parameters["Size"]).Convert(reflect.TypeOf(int64(0))).Int() reader, _ := request.Parameters["Reader"].(io.ReadCloser) err := driver.WriteStream(path, offset, size, reader) response := WriteStreamResponse{ Error: WrapError(err), } err = request.ResponseChannel.Send(&response) if err != nil { panic(err) } case "CurrentSize": path, _ := request.Parameters["Path"].(string) position, err := driver.CurrentSize(path) response := CurrentSizeResponse{ Position: position, Error: WrapError(err), } err = request.ResponseChannel.Send(&response) if err != nil { panic(err) } case "List": path, _ := request.Parameters["Path"].(string) keys, err := driver.List(path) response := ListResponse{ Keys: keys, Error: WrapError(err), } err = request.ResponseChannel.Send(&response) if err != nil { panic(err) } case "Move": sourcePath, _ := request.Parameters["SourcePath"].(string) destPath, _ := request.Parameters["DestPath"].(string) err := driver.Move(sourcePath, destPath) response := MoveResponse{ Error: WrapError(err), } err = request.ResponseChannel.Send(&response) if err != nil { panic(err) } case "Delete": path, _ := request.Parameters["Path"].(string) err := driver.Delete(path) response := DeleteResponse{ Error: WrapError(err), } err = request.ResponseChannel.Send(&response) if err != nil { panic(err) } default: panic(request) } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/middleware/0000755000175000017500000000000012502424227027503 5ustar tianontianon././@LongLink0000644000000000000000000000015600000000000011605 Lustar rootrootdistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/middleware/storagemiddleware.godistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/middleware/storagemidd0000644000175000017500000000241712502424227031734 0ustar tianontianonpackage storagemiddleware import ( "fmt" storagedriver "github.com/docker/distribution/registry/storage/driver" ) // InitFunc is the type of a StorageMiddleware factory function and is // used to register the constructor for different StorageMiddleware backends. type InitFunc func(storageDriver storagedriver.StorageDriver, options map[string]interface{}) (storagedriver.StorageDriver, error) var storageMiddlewares map[string]InitFunc // Register is used to register an InitFunc for // a StorageMiddleware backend with the given name. func Register(name string, initFunc InitFunc) error { if storageMiddlewares == nil { storageMiddlewares = make(map[string]InitFunc) } if _, exists := storageMiddlewares[name]; exists { return fmt.Errorf("name already registered: %s", name) } storageMiddlewares[name] = initFunc return nil } // Get constructs a StorageMiddleware with the given options using the named backend. func Get(name string, options map[string]interface{}, storageDriver storagedriver.StorageDriver) (storagedriver.StorageDriver, error) { if storageMiddlewares != nil { if initFunc, exists := storageMiddlewares[name]; exists { return initFunc(storageDriver, options) } } return nil, fmt.Errorf("no storage middleware registered with name: %s", name) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/middleware/cloudfront/0000755000175000017500000000000012502424227031662 5ustar tianontianon././@LongLink0000644000000000000000000000016200000000000011602 Lustar rootrootdistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/middleware/cloudfront/middleware.godistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/middleware/cloudfront/0000644000175000017500000000653312502424227031673 0ustar tianontianon// Package middleware - cloudfront wrapper for storage libs // N.B. currently only works with S3, not arbitrary sites // package middleware import ( "crypto/x509" "encoding/pem" "fmt" "io/ioutil" "net/url" "time" "github.com/AdRoll/goamz/cloudfront" storagedriver "github.com/docker/distribution/registry/storage/driver" storagemiddleware "github.com/docker/distribution/registry/storage/driver/middleware" ) // cloudFrontStorageMiddleware provides an simple implementation of layerHandler that // constructs temporary signed CloudFront URLs from the storagedriver layer URL, // then issues HTTP Temporary Redirects to this CloudFront content URL. type cloudFrontStorageMiddleware struct { storagedriver.StorageDriver cloudfront *cloudfront.CloudFront duration time.Duration } var _ storagedriver.StorageDriver = &cloudFrontStorageMiddleware{} // newCloudFrontLayerHandler constructs and returns a new CloudFront // LayerHandler implementation. // Required options: baseurl, privatekey, keypairid func newCloudFrontStorageMiddleware(storageDriver storagedriver.StorageDriver, options map[string]interface{}) (storagedriver.StorageDriver, error) { base, ok := options["baseurl"] if !ok { return nil, fmt.Errorf("No baseurl provided") } baseURL, ok := base.(string) if !ok { return nil, fmt.Errorf("baseurl must be a string") } pk, ok := options["privatekey"] if !ok { return nil, fmt.Errorf("No privatekey provided") } pkPath, ok := pk.(string) if !ok { return nil, fmt.Errorf("privatekey must be a string") } kpid, ok := options["keypairid"] if !ok { return nil, fmt.Errorf("No keypairid provided") } keypairID, ok := kpid.(string) if !ok { return nil, fmt.Errorf("keypairid must be a string") } pkBytes, err := ioutil.ReadFile(pkPath) if err != nil { return nil, fmt.Errorf("Failed to read privatekey file: %s", err) } block, _ := pem.Decode([]byte(pkBytes)) if block == nil { return nil, fmt.Errorf("Failed to decode private key as an rsa private key") } privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) if err != nil { return nil, err } cf := cloudfront.New(baseURL, privateKey, keypairID) duration := 20 * time.Minute d, ok := options["duration"] if ok { switch d := d.(type) { case time.Duration: duration = d case string: dur, err := time.ParseDuration(d) if err != nil { return nil, fmt.Errorf("Invalid duration: %s", err) } duration = dur } } return &cloudFrontStorageMiddleware{StorageDriver: storageDriver, cloudfront: cf, duration: duration}, nil } // Resolve returns an http.Handler which can serve the contents of the given // Layer, or an error if not supported by the storagedriver. func (lh *cloudFrontStorageMiddleware) URLFor(path string, options map[string]interface{}) (string, error) { // TODO(endophage): currently only supports S3 options["expiry"] = time.Now().Add(lh.duration) layerURLStr, err := lh.StorageDriver.URLFor(path, options) if err != nil { return "", err } layerURL, err := url.Parse(layerURLStr) if err != nil { return "", err } cfURL, err := lh.cloudfront.CannedSignedURL(layerURL.Path, "", time.Now().Add(lh.duration)) if err != nil { return "", err } return cfURL, nil } // init registers the cloudfront layerHandler backend. func init() { storagemiddleware.Register("cloudfront", storagemiddleware.InitFunc(newCloudFrontStorageMiddleware)) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/base/0000755000175000017500000000000012502424227026300 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/base/base.go0000644000175000017500000001055612502424227027550 0ustar tianontianon// Package base provides a base implementation of the storage driver that can // be used to implement common checks. The goal is to increase the amount of // code sharing. // // The canonical approach to use this class is to embed in the exported driver // struct such that calls are proxied through this implementation. First, // declare the internal driver, as follows: // // type driver struct { ... internal ...} // // The resulting type should implement StorageDriver such that it can be the // target of a Base struct. The exported type can then be declared as follows: // // type Driver struct { // Base // } // // Because Driver embeds Base, it effectively implements Base. If the driver // needs to intercept a call, before going to base, Driver should implement // that method. Effectively, Driver can intercept calls before coming in and // driver implements the actual logic. // // To further shield the embed from other packages, it is recommended to // employ a private embed struct: // // type baseEmbed struct { // base.Base // } // // Then, declare driver to embed baseEmbed, rather than Base directly: // // type Driver struct { // baseEmbed // } // // The type now implements StorageDriver, proxying through Base, without // exporting an unnessecary field. package base import ( "io" storagedriver "github.com/docker/distribution/registry/storage/driver" ) // Base provides a wrapper around a storagedriver implementation that provides // common path and bounds checking. type Base struct { storagedriver.StorageDriver } // GetContent wraps GetContent of underlying storage driver. func (base *Base) GetContent(path string) ([]byte, error) { if !storagedriver.PathRegexp.MatchString(path) { return nil, storagedriver.InvalidPathError{Path: path} } return base.StorageDriver.GetContent(path) } // PutContent wraps PutContent of underlying storage driver. func (base *Base) PutContent(path string, content []byte) error { if !storagedriver.PathRegexp.MatchString(path) { return storagedriver.InvalidPathError{Path: path} } return base.StorageDriver.PutContent(path, content) } // ReadStream wraps ReadStream of underlying storage driver. func (base *Base) ReadStream(path string, offset int64) (io.ReadCloser, error) { if offset < 0 { return nil, storagedriver.InvalidOffsetError{Path: path, Offset: offset} } if !storagedriver.PathRegexp.MatchString(path) { return nil, storagedriver.InvalidPathError{Path: path} } return base.StorageDriver.ReadStream(path, offset) } // WriteStream wraps WriteStream of underlying storage driver. func (base *Base) WriteStream(path string, offset int64, reader io.Reader) (nn int64, err error) { if offset < 0 { return 0, storagedriver.InvalidOffsetError{Path: path, Offset: offset} } if !storagedriver.PathRegexp.MatchString(path) { return 0, storagedriver.InvalidPathError{Path: path} } return base.StorageDriver.WriteStream(path, offset, reader) } // Stat wraps Stat of underlying storage driver. func (base *Base) Stat(path string) (storagedriver.FileInfo, error) { if !storagedriver.PathRegexp.MatchString(path) { return nil, storagedriver.InvalidPathError{Path: path} } return base.StorageDriver.Stat(path) } // List wraps List of underlying storage driver. func (base *Base) List(path string) ([]string, error) { if !storagedriver.PathRegexp.MatchString(path) && path != "/" { return nil, storagedriver.InvalidPathError{Path: path} } return base.StorageDriver.List(path) } // Move wraps Move of underlying storage driver. func (base *Base) Move(sourcePath string, destPath string) error { if !storagedriver.PathRegexp.MatchString(sourcePath) { return storagedriver.InvalidPathError{Path: sourcePath} } else if !storagedriver.PathRegexp.MatchString(destPath) { return storagedriver.InvalidPathError{Path: destPath} } return base.StorageDriver.Move(sourcePath, destPath) } // Delete wraps Delete of underlying storage driver. func (base *Base) Delete(path string) error { if !storagedriver.PathRegexp.MatchString(path) { return storagedriver.InvalidPathError{Path: path} } return base.StorageDriver.Delete(path) } // URLFor wraps URLFor of underlying storage driver. func (base *Base) URLFor(path string, options map[string]interface{}) (string, error) { if !storagedriver.PathRegexp.MatchString(path) { return "", storagedriver.InvalidPathError{Path: path} } return base.StorageDriver.URLFor(path, options) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/s3/0000755000175000017500000000000012502424227025713 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/s3/s3.go0000644000175000017500000004520012502424227026570 0ustar tianontianon// Package s3 provides a storagedriver.StorageDriver implementation to // store blobs in Amazon S3 cloud storage. // // This package leverages the AdRoll/goamz client library for interfacing with // s3. // // Because s3 is a key, value store the Stat call does not support last modification // time for directories (directories are an abstraction for key, value stores) // // Keep in mind that s3 guarantees only eventual consistency, so do not assume // that a successful write will mean immediate access to the data written (although // in most regions a new object put has guaranteed read after write). The only true // guarantee is that once you call Stat and receive a certain file size, that much of // the file is already accessible. package s3 import ( "bytes" "fmt" "io" "io/ioutil" "net/http" "strconv" "strings" "time" "github.com/AdRoll/goamz/aws" "github.com/AdRoll/goamz/s3" storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/distribution/registry/storage/driver/base" "github.com/docker/distribution/registry/storage/driver/factory" ) const driverName = "s3" // minChunkSize defines the minimum multipart upload chunk size // S3 API requires multipart upload chunks to be at least 5MB const minChunkSize = 5 << 20 const defaultChunkSize = 2 * minChunkSize // listMax is the largest amount of objects you can request from S3 in a list call const listMax = 1000 //DriverParameters A struct that encapsulates all of the driver parameters after all values have been set type DriverParameters struct { AccessKey string SecretKey string Bucket string Region aws.Region Encrypt bool Secure bool V4Auth bool ChunkSize int64 RootDirectory string } func init() { factory.Register(driverName, &s3DriverFactory{}) } // s3DriverFactory implements the factory.StorageDriverFactory interface type s3DriverFactory struct{} func (factory *s3DriverFactory) Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error) { return FromParameters(parameters) } type driver struct { S3 *s3.S3 Bucket *s3.Bucket ChunkSize int64 Encrypt bool RootDirectory string } type baseEmbed struct { base.Base } // Driver is a storagedriver.StorageDriver implementation backed by Amazon S3 // Objects are stored at absolute keys in the provided bucket. type Driver struct { baseEmbed } // FromParameters constructs a new Driver with a given parameters map // Required parameters: // - accesskey // - secretkey // - region // - bucket // - encrypt func FromParameters(parameters map[string]interface{}) (*Driver, error) { // Providing no values for these is valid in case the user is authenticating // with an IAM on an ec2 instance (in which case the instance credentials will // be summoned when GetAuth is called) accessKey, ok := parameters["accesskey"] if !ok { accessKey = "" } secretKey, ok := parameters["secretkey"] if !ok { secretKey = "" } regionName, ok := parameters["region"] if !ok || fmt.Sprint(regionName) == "" { return nil, fmt.Errorf("No region parameter provided") } region := aws.GetRegion(fmt.Sprint(regionName)) if region.Name == "" { return nil, fmt.Errorf("Invalid region provided: %v", region) } bucket, ok := parameters["bucket"] if !ok || fmt.Sprint(bucket) == "" { return nil, fmt.Errorf("No bucket parameter provided") } encryptBool := false encrypt, ok := parameters["encrypt"] if ok { encryptBool, ok = encrypt.(bool) if !ok { return nil, fmt.Errorf("The encrypt parameter should be a boolean") } } secureBool := true secure, ok := parameters["secure"] if ok { secureBool, ok = secure.(bool) if !ok { return nil, fmt.Errorf("The secure parameter should be a boolean") } } v4AuthBool := false v4Auth, ok := parameters["v4auth"] if ok { v4AuthBool, ok = v4Auth.(bool) if !ok { return nil, fmt.Errorf("The v4auth parameter should be a boolean") } } chunkSize := int64(defaultChunkSize) chunkSizeParam, ok := parameters["chunksize"] if ok { chunkSize, ok = chunkSizeParam.(int64) if !ok || chunkSize < minChunkSize { return nil, fmt.Errorf("The chunksize parameter should be a number that is larger than 5*1024*1024") } } rootDirectory, ok := parameters["rootdirectory"] if !ok { rootDirectory = "" } params := DriverParameters{ fmt.Sprint(accessKey), fmt.Sprint(secretKey), fmt.Sprint(bucket), region, encryptBool, secureBool, v4AuthBool, chunkSize, fmt.Sprint(rootDirectory), } return New(params) } // New constructs a new Driver with the given AWS credentials, region, encryption flag, and // bucketName func New(params DriverParameters) (*Driver, error) { auth, err := aws.GetAuth(params.AccessKey, params.SecretKey, "", time.Time{}) if err != nil { return nil, err } if !params.Secure { params.Region.S3Endpoint = strings.Replace(params.Region.S3Endpoint, "https", "http", 1) } s3obj := s3.New(auth, params.Region) bucket := s3obj.Bucket(params.Bucket) if params.V4Auth { s3obj.Signature = aws.V4Signature } else { if params.Region.Name == "eu-central-1" { return nil, fmt.Errorf("The eu-central-1 region only works with v4 authentication") } } // Validate that the given credentials have at least read permissions in the // given bucket scope. if _, err := bucket.List(strings.TrimRight(params.RootDirectory, "/"), "", "", 1); err != nil { return nil, err } // TODO Currently multipart uploads have no timestamps, so this would be unwise // if you initiated a new s3driver while another one is running on the same bucket. // multis, _, err := bucket.ListMulti("", "") // if err != nil { // return nil, err // } // for _, multi := range multis { // err := multi.Abort() // //TODO appropriate to do this error checking? // if err != nil { // return nil, err // } // } d := &driver{ S3: s3obj, Bucket: bucket, ChunkSize: params.ChunkSize, Encrypt: params.Encrypt, RootDirectory: params.RootDirectory, } return &Driver{ baseEmbed: baseEmbed{ Base: base.Base{ StorageDriver: d, }, }, }, nil } // Implement the storagedriver.StorageDriver interface // GetContent retrieves the content stored at "path" as a []byte. func (d *driver) GetContent(path string) ([]byte, error) { content, err := d.Bucket.Get(d.s3Path(path)) if err != nil { return nil, parseError(path, err) } return content, nil } // PutContent stores the []byte content at a location designated by "path". func (d *driver) PutContent(path string, contents []byte) error { return parseError(path, d.Bucket.Put(d.s3Path(path), contents, d.getContentType(), getPermissions(), d.getOptions())) } // ReadStream retrieves an io.ReadCloser for the content stored at "path" with a // given byte offset. func (d *driver) ReadStream(path string, offset int64) (io.ReadCloser, error) { headers := make(http.Header) headers.Add("Range", "bytes="+strconv.FormatInt(offset, 10)+"-") resp, err := d.Bucket.GetResponseWithHeaders(d.s3Path(path), headers) if err != nil { if s3Err, ok := err.(*s3.Error); ok && s3Err.Code == "InvalidRange" { return ioutil.NopCloser(bytes.NewReader(nil)), nil } return nil, parseError(path, err) } return resp.Body, nil } // WriteStream stores the contents of the provided io.Reader at a // location designated by the given path. The driver will know it has // received the full contents when the reader returns io.EOF. The number // of successfully READ bytes will be returned, even if an error is // returned. May be used to resume writing a stream by providing a nonzero // offset. Offsets past the current size will write from the position // beyond the end of the file. func (d *driver) WriteStream(path string, offset int64, reader io.Reader) (totalRead int64, err error) { partNumber := 1 bytesRead := 0 var putErrChan chan error parts := []s3.Part{} var part s3.Part multi, err := d.Bucket.InitMulti(d.s3Path(path), d.getContentType(), getPermissions(), d.getOptions()) if err != nil { return 0, err } buf := make([]byte, d.ChunkSize) zeroBuf := make([]byte, d.ChunkSize) // We never want to leave a dangling multipart upload, our only consistent state is // when there is a whole object at path. This is in order to remain consistent with // the stat call. // // Note that if the machine dies before executing the defer, we will be left with a dangling // multipart upload, which will eventually be cleaned up, but we will lose all of the progress // made prior to the machine crashing. defer func() { if putErrChan != nil { if putErr := <-putErrChan; putErr != nil { err = putErr } } if len(parts) > 0 { if multi == nil { // Parts should be empty if the multi is not initialized panic("Unreachable") } else { if multi.Complete(parts) != nil { multi.Abort() } } } }() // Fills from 0 to total from current fromSmallCurrent := func(total int64) error { current, err := d.ReadStream(path, 0) if err != nil { return err } bytesRead = 0 for int64(bytesRead) < total { //The loop should very rarely enter a second iteration nn, err := current.Read(buf[bytesRead:total]) bytesRead += nn if err != nil { if err != io.EOF { return err } break } } return nil } // Fills from parameter to chunkSize from reader fromReader := func(from int64) error { bytesRead = 0 for from+int64(bytesRead) < d.ChunkSize { nn, err := reader.Read(buf[from+int64(bytesRead):]) totalRead += int64(nn) bytesRead += nn if err != nil { if err != io.EOF { return err } break } } if putErrChan == nil { putErrChan = make(chan error) } else { if putErr := <-putErrChan; putErr != nil { putErrChan = nil return putErr } } go func(bytesRead int, from int64, buf []byte) { // parts and partNumber are safe, because this function is the only one modifying them and we // force it to be executed serially. if bytesRead > 0 { part, putErr := multi.PutPart(int(partNumber), bytes.NewReader(buf[0:int64(bytesRead)+from])) if putErr != nil { putErrChan <- putErr } parts = append(parts, part) partNumber++ } putErrChan <- nil }(bytesRead, from, buf) buf = make([]byte, d.ChunkSize) return nil } if offset > 0 { resp, err := d.Bucket.Head(d.s3Path(path), nil) if err != nil { if s3Err, ok := err.(*s3.Error); !ok || s3Err.Code != "NoSuchKey" { return 0, err } } currentLength := int64(0) if err == nil { currentLength = resp.ContentLength } if currentLength >= offset { if offset < d.ChunkSize { // chunkSize > currentLength >= offset if err = fromSmallCurrent(offset); err != nil { return totalRead, err } if err = fromReader(offset); err != nil { return totalRead, err } if totalRead+offset < d.ChunkSize { return totalRead, nil } } else { // currentLength >= offset >= chunkSize _, part, err = multi.PutPartCopy(partNumber, s3.CopyOptions{CopySourceOptions: "bytes=0-" + strconv.FormatInt(offset-1, 10)}, d.Bucket.Name+"/"+d.s3Path(path)) if err != nil { return 0, err } parts = append(parts, part) partNumber++ } } else { // Fills between parameters with 0s but only when to - from <= chunkSize fromZeroFillSmall := func(from, to int64) error { bytesRead = 0 for from+int64(bytesRead) < to { nn, err := bytes.NewReader(zeroBuf).Read(buf[from+int64(bytesRead) : to]) bytesRead += nn if err != nil { return err } } return nil } // Fills between parameters with 0s, making new parts fromZeroFillLarge := func(from, to int64) error { bytesRead64 := int64(0) for to-(from+bytesRead64) >= d.ChunkSize { part, err := multi.PutPart(int(partNumber), bytes.NewReader(zeroBuf)) if err != nil { return err } bytesRead64 += d.ChunkSize parts = append(parts, part) partNumber++ } return fromZeroFillSmall(0, (to-from)%d.ChunkSize) } // currentLength < offset if currentLength < d.ChunkSize { if offset < d.ChunkSize { // chunkSize > offset > currentLength if err = fromSmallCurrent(currentLength); err != nil { return totalRead, err } if err = fromZeroFillSmall(currentLength, offset); err != nil { return totalRead, err } if err = fromReader(offset); err != nil { return totalRead, err } if totalRead+offset < d.ChunkSize { return totalRead, nil } } else { // offset >= chunkSize > currentLength if err = fromSmallCurrent(currentLength); err != nil { return totalRead, err } if err = fromZeroFillSmall(currentLength, d.ChunkSize); err != nil { return totalRead, err } part, err = multi.PutPart(int(partNumber), bytes.NewReader(buf)) if err != nil { return totalRead, err } parts = append(parts, part) partNumber++ //Zero fill from chunkSize up to offset, then some reader if err = fromZeroFillLarge(d.ChunkSize, offset); err != nil { return totalRead, err } if err = fromReader(offset % d.ChunkSize); err != nil { return totalRead, err } if totalRead+(offset%d.ChunkSize) < d.ChunkSize { return totalRead, nil } } } else { // offset > currentLength >= chunkSize _, part, err = multi.PutPartCopy(partNumber, s3.CopyOptions{}, d.Bucket.Name+"/"+d.s3Path(path)) if err != nil { return 0, err } parts = append(parts, part) partNumber++ //Zero fill from currentLength up to offset, then some reader if err = fromZeroFillLarge(currentLength, offset); err != nil { return totalRead, err } if err = fromReader((offset - currentLength) % d.ChunkSize); err != nil { return totalRead, err } if totalRead+((offset-currentLength)%d.ChunkSize) < d.ChunkSize { return totalRead, nil } } } } for { if err = fromReader(0); err != nil { return totalRead, err } if int64(bytesRead) < d.ChunkSize { break } } return totalRead, nil } // Stat retrieves the FileInfo for the given path, including the current size // in bytes and the creation time. func (d *driver) Stat(path string) (storagedriver.FileInfo, error) { listResponse, err := d.Bucket.List(d.s3Path(path), "", "", 1) if err != nil { return nil, err } fi := storagedriver.FileInfoFields{ Path: path, } if len(listResponse.Contents) == 1 { if listResponse.Contents[0].Key != d.s3Path(path) { fi.IsDir = true } else { fi.IsDir = false fi.Size = listResponse.Contents[0].Size timestamp, err := time.Parse(time.RFC3339Nano, listResponse.Contents[0].LastModified) if err != nil { return nil, err } fi.ModTime = timestamp } } else if len(listResponse.CommonPrefixes) == 1 { fi.IsDir = true } else { return nil, storagedriver.PathNotFoundError{Path: path} } return storagedriver.FileInfoInternal{FileInfoFields: fi}, nil } // List returns a list of the objects that are direct descendants of the given path. func (d *driver) List(path string) ([]string, error) { if path != "/" && path[len(path)-1] != '/' { path = path + "/" } // This is to cover for the cases when the rootDirectory of the driver is either "" or "/". // In those cases, there is no root prefix to replace and we must actually add a "/" to all // results in order to keep them as valid paths as recognized by storagedriver.PathRegexp prefix := "" if d.s3Path("") == "" { prefix = "/" } listResponse, err := d.Bucket.List(d.s3Path(path), "/", "", listMax) if err != nil { return nil, err } files := []string{} directories := []string{} for { for _, key := range listResponse.Contents { files = append(files, strings.Replace(key.Key, d.s3Path(""), prefix, 1)) } for _, commonPrefix := range listResponse.CommonPrefixes { directories = append(directories, strings.Replace(commonPrefix[0:len(commonPrefix)-1], d.s3Path(""), prefix, 1)) } if listResponse.IsTruncated { listResponse, err = d.Bucket.List(d.s3Path(path), "/", listResponse.NextMarker, listMax) if err != nil { return nil, err } } else { break } } return append(files, directories...), nil } // Move moves an object stored at sourcePath to destPath, removing the original // object. func (d *driver) Move(sourcePath string, destPath string) error { /* This is terrible, but aws doesn't have an actual move. */ _, err := d.Bucket.PutCopy(d.s3Path(destPath), getPermissions(), s3.CopyOptions{Options: d.getOptions(), ContentType: d.getContentType()}, d.Bucket.Name+"/"+d.s3Path(sourcePath)) if err != nil { return parseError(sourcePath, err) } return d.Delete(sourcePath) } // Delete recursively deletes all objects stored at "path" and its subpaths. func (d *driver) Delete(path string) error { listResponse, err := d.Bucket.List(d.s3Path(path), "", "", listMax) if err != nil || len(listResponse.Contents) == 0 { return storagedriver.PathNotFoundError{Path: path} } s3Objects := make([]s3.Object, listMax) for len(listResponse.Contents) > 0 { for index, key := range listResponse.Contents { s3Objects[index].Key = key.Key } err := d.Bucket.DelMulti(s3.Delete{Quiet: false, Objects: s3Objects[0:len(listResponse.Contents)]}) if err != nil { return nil } listResponse, err = d.Bucket.List(d.s3Path(path), "", "", listMax) if err != nil { return err } } return nil } // URLFor returns a URL which may be used to retrieve the content stored at the given path. // May return an UnsupportedMethodErr in certain StorageDriver implementations. func (d *driver) URLFor(path string, options map[string]interface{}) (string, error) { methodString := "GET" method, ok := options["method"] if ok { methodString, ok = method.(string) if !ok || (methodString != "GET" && methodString != "HEAD") { return "", storagedriver.ErrUnsupportedMethod } } expiresTime := time.Now().Add(20 * time.Minute) expires, ok := options["expiry"] if ok { et, ok := expires.(time.Time) if ok { expiresTime = et } } return d.Bucket.SignedURLWithMethod(methodString, d.s3Path(path), expiresTime, nil, nil), nil } func (d *driver) s3Path(path string) string { return strings.TrimLeft(strings.TrimRight(d.RootDirectory, "/")+path, "/") } func parseError(path string, err error) error { if s3Err, ok := err.(*s3.Error); ok && s3Err.Code == "NoSuchKey" { return storagedriver.PathNotFoundError{Path: path} } return err } func hasCode(err error, code string) bool { s3err, ok := err.(*aws.Error) return ok && s3err.Code == code } func (d *driver) getOptions() s3.Options { return s3.Options{SSE: d.Encrypt} } func getPermissions() s3.ACL { return s3.Private } func (d *driver) getContentType() string { return "application/octet-stream" } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/s3/s3_test.go0000644000175000017500000000713512502424227027634 0ustar tianontianonpackage s3 import ( "io/ioutil" "os" "strconv" "testing" "github.com/AdRoll/goamz/aws" storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/distribution/registry/storage/driver/testsuites" "gopkg.in/check.v1" ) // Hook up gocheck into the "go test" runner. func Test(t *testing.T) { check.TestingT(t) } type S3DriverConstructor func(rootDirectory string) (*Driver, error) func init() { accessKey := os.Getenv("AWS_ACCESS_KEY") secretKey := os.Getenv("AWS_SECRET_KEY") bucket := os.Getenv("S3_BUCKET") encrypt := os.Getenv("S3_ENCRYPT") secure := os.Getenv("S3_SECURE") v4auth := os.Getenv("S3_USE_V4_AUTH") region := os.Getenv("AWS_REGION") root, err := ioutil.TempDir("", "driver-") if err != nil { panic(err) } defer os.Remove(root) s3DriverConstructor := func(rootDirectory string) (*Driver, error) { encryptBool := false if encrypt != "" { encryptBool, err = strconv.ParseBool(encrypt) if err != nil { return nil, err } } secureBool := true if secure != "" { secureBool, err = strconv.ParseBool(secure) if err != nil { return nil, err } } v4AuthBool := false if v4auth != "" { v4AuthBool, err = strconv.ParseBool(v4auth) if err != nil { return nil, err } } parameters := DriverParameters{ accessKey, secretKey, bucket, aws.GetRegion(region), encryptBool, secureBool, v4AuthBool, minChunkSize, rootDirectory, } return New(parameters) } // Skip S3 storage driver tests if environment variable parameters are not provided skipCheck := func() string { if accessKey == "" || secretKey == "" || region == "" || bucket == "" || encrypt == "" { return "Must set AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION, S3_BUCKET, and S3_ENCRYPT to run S3 tests" } return "" } driverConstructor := func() (storagedriver.StorageDriver, error) { return s3DriverConstructor(root) } testsuites.RegisterInProcessSuite(driverConstructor, skipCheck) // s3Constructor := func() (*Driver, error) { // return s3DriverConstructor(aws.GetRegion(region)) // } RegisterS3DriverSuite(s3DriverConstructor, skipCheck) // testsuites.RegisterIPCSuite(driverName, map[string]string{ // "accesskey": accessKey, // "secretkey": secretKey, // "region": region.Name, // "bucket": bucket, // "encrypt": encrypt, // }, skipCheck) // } } func RegisterS3DriverSuite(s3DriverConstructor S3DriverConstructor, skipCheck testsuites.SkipCheck) { check.Suite(&S3DriverSuite{ Constructor: s3DriverConstructor, SkipCheck: skipCheck, }) } type S3DriverSuite struct { Constructor S3DriverConstructor testsuites.SkipCheck } func (suite *S3DriverSuite) SetUpSuite(c *check.C) { if reason := suite.SkipCheck(); reason != "" { c.Skip(reason) } } func (suite *S3DriverSuite) TestEmptyRootList(c *check.C) { validRoot, err := ioutil.TempDir("", "driver-") c.Assert(err, check.IsNil) defer os.Remove(validRoot) rootedDriver, err := suite.Constructor(validRoot) c.Assert(err, check.IsNil) emptyRootDriver, err := suite.Constructor("") c.Assert(err, check.IsNil) slashRootDriver, err := suite.Constructor("/") c.Assert(err, check.IsNil) filename := "/test" contents := []byte("contents") err = rootedDriver.PutContent(filename, contents) c.Assert(err, check.IsNil) defer rootedDriver.Delete(filename) keys, err := emptyRootDriver.List("/") for _, path := range keys { c.Assert(storagedriver.PathRegexp.MatchString(path), check.Equals, true) } keys, err = slashRootDriver.List("/") for _, path := range keys { c.Assert(storagedriver.PathRegexp.MatchString(path), check.Equals, true) } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/fileinfo.go0000644000175000017500000000513412502424227027513 0ustar tianontianonpackage driver import "time" // FileInfo returns information about a given path. Inspired by os.FileInfo, // it elides the base name method for a full path instead. type FileInfo interface { // Path provides the full path of the target of this file info. Path() string // Size returns current length in bytes of the file. The return value can // be used to write to the end of the file at path. The value is // meaningless if IsDir returns true. Size() int64 // ModTime returns the modification time for the file. For backends that // don't have a modification time, the creation time should be returned. ModTime() time.Time // IsDir returns true if the path is a directory. IsDir() bool } // NOTE(stevvooe): The next two types, FileInfoFields and FileInfoInternal // should only be used by storagedriver implementations. They should moved to // a "driver" package, similar to database/sql. // FileInfoFields provides the exported fields for implementing FileInfo // interface in storagedriver implementations. It should be used with // InternalFileInfo. type FileInfoFields struct { // Path provides the full path of the target of this file info. Path string // Size is current length in bytes of the file. The value of this field // can be used to write to the end of the file at path. The value is // meaningless if IsDir is set to true. Size int64 // ModTime returns the modification time for the file. For backends that // don't have a modification time, the creation time should be returned. ModTime time.Time // IsDir returns true if the path is a directory. IsDir bool } // FileInfoInternal implements the FileInfo interface. This should only be // used by storagedriver implementations that don't have a specialized // FileInfo type. type FileInfoInternal struct { FileInfoFields } var _ FileInfo = FileInfoInternal{} var _ FileInfo = &FileInfoInternal{} // Path provides the full path of the target of this file info. func (fi FileInfoInternal) Path() string { return fi.FileInfoFields.Path } // Size returns current length in bytes of the file. The return value can // be used to write to the end of the file at path. The value is // meaningless if IsDir returns true. func (fi FileInfoInternal) Size() int64 { return fi.FileInfoFields.Size } // ModTime returns the modification time for the file. For backends that // don't have a modification time, the creation time should be returned. func (fi FileInfoInternal) ModTime() time.Time { return fi.FileInfoFields.ModTime } // IsDir returns true if the path is a directory. func (fi FileInfoInternal) IsDir() bool { return fi.FileInfoFields.IsDir } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/inmemory/0000755000175000017500000000000012502424227027225 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/inmemory/mfs.go0000644000175000017500000001365612502424227030354 0ustar tianontianonpackage inmemory import ( "fmt" "io" "path" "sort" "strings" "time" ) var ( errExists = fmt.Errorf("exists") errNotExists = fmt.Errorf("notexists") errIsNotDir = fmt.Errorf("notdir") errIsDir = fmt.Errorf("isdir") ) type node interface { name() string path() string isdir() bool modtime() time.Time } // dir is the central type for the memory-based storagedriver. All operations // are dispatched from a root dir. type dir struct { common // TODO(stevvooe): Use sorted slice + search. children map[string]node } var _ node = &dir{} func (d *dir) isdir() bool { return true } // add places the node n into dir d. func (d *dir) add(n node) { if d.children == nil { d.children = make(map[string]node) } d.children[n.name()] = n d.mod = time.Now() } // find searches for the node, given path q in dir. If the node is found, it // will be returned. If the node is not found, the closet existing parent. If // the node is found, the returned (node).path() will match q. func (d *dir) find(q string) node { q = strings.Trim(q, "/") i := strings.Index(q, "/") if q == "" { return d } if i == 0 { panic("shouldn't happen, no root paths") } var component string if i < 0 { // No more path components component = q } else { component = q[:i] } child, ok := d.children[component] if !ok { // Node was not found. Return p and the current node. return d } if child.isdir() { // traverse down! q = q[i+1:] return child.(*dir).find(q) } return child } func (d *dir) list(p string) ([]string, error) { n := d.find(p) if n.path() != p { return nil, errNotExists } if !n.isdir() { return nil, errIsNotDir } var children []string for _, child := range n.(*dir).children { children = append(children, child.path()) } sort.Strings(children) return children, nil } // mkfile or return the existing one. returns an error if it exists and is a // directory. Essentially, this is open or create. func (d *dir) mkfile(p string) (*file, error) { n := d.find(p) if n.path() == p { if n.isdir() { return nil, errIsDir } return n.(*file), nil } dirpath, filename := path.Split(p) // Make any non-existent directories n, err := d.mkdirs(dirpath) if err != nil { return nil, err } dd := n.(*dir) n = &file{ common: common{ p: path.Join(dd.path(), filename), mod: time.Now(), }, } dd.add(n) return n.(*file), nil } // mkdirs creates any missing directory entries in p and returns the result. func (d *dir) mkdirs(p string) (*dir, error) { p = normalize(p) n := d.find(p) if !n.isdir() { // Found something there return nil, errIsNotDir } if n.path() == p { return n.(*dir), nil } dd := n.(*dir) relative := strings.Trim(strings.TrimPrefix(p, n.path()), "/") if relative == "" { return dd, nil } components := strings.Split(relative, "/") for _, component := range components { d, err := dd.mkdir(component) if err != nil { // This should actually never happen, since there are no children. return nil, err } dd = d } return dd, nil } // mkdir creates a child directory under d with the given name. func (d *dir) mkdir(name string) (*dir, error) { if name == "" { return nil, fmt.Errorf("invalid dirname") } _, ok := d.children[name] if ok { return nil, errExists } child := &dir{ common: common{ p: path.Join(d.path(), name), mod: time.Now(), }, } d.add(child) d.mod = time.Now() return child, nil } func (d *dir) move(src, dst string) error { dstDirname, _ := path.Split(dst) dp, err := d.mkdirs(dstDirname) if err != nil { return err } srcDirname, srcFilename := path.Split(src) sp := d.find(srcDirname) if normalize(srcDirname) != normalize(sp.path()) { return errNotExists } s, ok := sp.(*dir).children[srcFilename] if !ok { return errNotExists } delete(sp.(*dir).children, srcFilename) switch n := s.(type) { case *dir: n.p = dst case *file: n.p = dst } dp.add(s) return nil } func (d *dir) delete(p string) error { dirname, filename := path.Split(p) parent := d.find(dirname) if normalize(dirname) != normalize(parent.path()) { return errNotExists } if _, ok := parent.(*dir).children[filename]; !ok { return errNotExists } delete(parent.(*dir).children, filename) return nil } // dump outputs a primitive directory structure to stdout. func (d *dir) dump(indent string) { fmt.Println(indent, d.name()+"/") for _, child := range d.children { if child.isdir() { child.(*dir).dump(indent + "\t") } else { fmt.Println(indent, child.name()) } } } func (d *dir) String() string { return fmt.Sprintf("&dir{path: %v, children: %v}", d.p, d.children) } // file stores actual data in the fs tree. It acts like an open, seekable file // where operations are conducted through ReadAt and WriteAt. Use it with // SectionReader for the best effect. type file struct { common data []byte } var _ node = &file{} func (f *file) isdir() bool { return false } func (f *file) truncate() { f.data = f.data[:0] } func (f *file) sectionReader(offset int64) io.Reader { return io.NewSectionReader(f, offset, int64(len(f.data))-offset) } func (f *file) ReadAt(p []byte, offset int64) (n int, err error) { return copy(p, f.data[offset:]), nil } func (f *file) WriteAt(p []byte, offset int64) (n int, err error) { off := int(offset) if cap(f.data) < off+len(p) { data := make([]byte, len(f.data), off+len(p)) copy(data, f.data) f.data = data } f.mod = time.Now() f.data = f.data[:off+len(p)] return copy(f.data[off:off+len(p)], p), nil } func (f *file) String() string { return fmt.Sprintf("&file{path: %q}", f.p) } // common provides shared fields and methods for node implementations. type common struct { p string mod time.Time } func (c *common) name() string { _, name := path.Split(c.p) return name } func (c *common) path() string { return c.p } func (c *common) modtime() time.Time { return c.mod } func normalize(p string) string { return "/" + strings.Trim(p, "/") } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/inmemory/driver.go0000644000175000017500000001411512502424227031051 0ustar tianontianonpackage inmemory import ( "bytes" "fmt" "io" "io/ioutil" "sync" "time" storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/distribution/registry/storage/driver/base" "github.com/docker/distribution/registry/storage/driver/factory" ) const driverName = "inmemory" func init() { factory.Register(driverName, &inMemoryDriverFactory{}) } // inMemoryDriverFacotry implements the factory.StorageDriverFactory interface. type inMemoryDriverFactory struct{} func (factory *inMemoryDriverFactory) Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error) { return New(), nil } type driver struct { root *dir mutex sync.RWMutex } // baseEmbed allows us to hide the Base embed. type baseEmbed struct { base.Base } // Driver is a storagedriver.StorageDriver implementation backed by a local map. // Intended solely for example and testing purposes. type Driver struct { baseEmbed // embedded, hidden base driver. } var _ storagedriver.StorageDriver = &Driver{} // New constructs a new Driver. func New() *Driver { return &Driver{ baseEmbed: baseEmbed{ Base: base.Base{ StorageDriver: &driver{ root: &dir{ common: common{ p: "/", mod: time.Now(), }, }, }, }, }, } } // Implement the storagedriver.StorageDriver interface. // GetContent retrieves the content stored at "path" as a []byte. func (d *driver) GetContent(path string) ([]byte, error) { d.mutex.RLock() defer d.mutex.RUnlock() rc, err := d.ReadStream(path, 0) if err != nil { return nil, err } defer rc.Close() return ioutil.ReadAll(rc) } // PutContent stores the []byte content at a location designated by "path". func (d *driver) PutContent(p string, contents []byte) error { d.mutex.Lock() defer d.mutex.Unlock() f, err := d.root.mkfile(p) if err != nil { // TODO(stevvooe): Again, we need to clarify when this is not a // directory in StorageDriver API. return fmt.Errorf("not a file") } f.truncate() f.WriteAt(contents, 0) return nil } // ReadStream retrieves an io.ReadCloser for the content stored at "path" with a // given byte offset. func (d *driver) ReadStream(path string, offset int64) (io.ReadCloser, error) { d.mutex.RLock() defer d.mutex.RUnlock() if offset < 0 { return nil, storagedriver.InvalidOffsetError{Path: path, Offset: offset} } path = normalize(path) found := d.root.find(path) if found.path() != path { return nil, storagedriver.PathNotFoundError{Path: path} } if found.isdir() { return nil, fmt.Errorf("%q is a directory", path) } return ioutil.NopCloser(found.(*file).sectionReader(offset)), nil } // WriteStream stores the contents of the provided io.ReadCloser at a location // designated by the given path. func (d *driver) WriteStream(path string, offset int64, reader io.Reader) (nn int64, err error) { d.mutex.Lock() defer d.mutex.Unlock() if offset < 0 { return 0, storagedriver.InvalidOffsetError{Path: path, Offset: offset} } normalized := normalize(path) f, err := d.root.mkfile(normalized) if err != nil { return 0, fmt.Errorf("not a file") } // Unlock while we are reading from the source, in case we are reading // from the same mfs instance. This can be fixed by a more granular // locking model. d.mutex.Unlock() d.mutex.RLock() // Take the readlock to block other writers. var buf bytes.Buffer nn, err = buf.ReadFrom(reader) if err != nil { // TODO(stevvooe): This condition is odd and we may need to clarify: // we've read nn bytes from reader but have written nothing to the // backend. What is the correct return value? Really, the caller needs // to know that the reader has been advanced and reattempting the // operation is incorrect. d.mutex.RUnlock() d.mutex.Lock() return nn, err } d.mutex.RUnlock() d.mutex.Lock() f.WriteAt(buf.Bytes(), offset) return nn, err } // Stat returns info about the provided path. func (d *driver) Stat(path string) (storagedriver.FileInfo, error) { d.mutex.RLock() defer d.mutex.RUnlock() normalized := normalize(path) found := d.root.find(path) if found.path() != normalized { return nil, storagedriver.PathNotFoundError{Path: path} } fi := storagedriver.FileInfoFields{ Path: path, IsDir: found.isdir(), ModTime: found.modtime(), } if !fi.IsDir { fi.Size = int64(len(found.(*file).data)) } return storagedriver.FileInfoInternal{FileInfoFields: fi}, nil } // List returns a list of the objects that are direct descendants of the given // path. func (d *driver) List(path string) ([]string, error) { d.mutex.RLock() defer d.mutex.RUnlock() normalized := normalize(path) found := d.root.find(normalized) if !found.isdir() { return nil, fmt.Errorf("not a directory") // TODO(stevvooe): Need error type for this... } entries, err := found.(*dir).list(normalized) if err != nil { switch err { case errNotExists: return nil, storagedriver.PathNotFoundError{Path: path} case errIsNotDir: return nil, fmt.Errorf("not a directory") default: return nil, err } } return entries, nil } // Move moves an object stored at sourcePath to destPath, removing the original // object. func (d *driver) Move(sourcePath string, destPath string) error { d.mutex.Lock() defer d.mutex.Unlock() normalizedSrc, normalizedDst := normalize(sourcePath), normalize(destPath) err := d.root.move(normalizedSrc, normalizedDst) switch err { case errNotExists: return storagedriver.PathNotFoundError{Path: destPath} default: return err } } // Delete recursively deletes all objects stored at "path" and its subpaths. func (d *driver) Delete(path string) error { d.mutex.Lock() defer d.mutex.Unlock() normalized := normalize(path) err := d.root.delete(normalized) switch err { case errNotExists: return storagedriver.PathNotFoundError{Path: path} default: return err } } // URLFor returns a URL which may be used to retrieve the content stored at the given path. // May return an UnsupportedMethodErr in certain StorageDriver implementations. func (d *driver) URLFor(path string, options map[string]interface{}) (string, error) { return "", storagedriver.ErrUnsupportedMethod } ././@LongLink0000644000000000000000000000014600000000000011604 Lustar rootrootdistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/inmemory/driver_test.godistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/storage/driver/inmemory/driver_test.g0000644000175000017500000000124412502424227031730 0ustar tianontianonpackage inmemory import ( "testing" storagedriver "github.com/docker/distribution/registry/storage/driver" "github.com/docker/distribution/registry/storage/driver/testsuites" "gopkg.in/check.v1" ) // Hook up gocheck into the "go test" runner. func Test(t *testing.T) { check.TestingT(t) } func init() { inmemoryDriverConstructor := func() (storagedriver.StorageDriver, error) { return New(), nil } testsuites.RegisterInProcessSuite(inmemoryDriverConstructor, testsuites.NeverSkip) // BUG(stevvooe): Disable flaky IPC tests for now when we can troubleshoot // the problems with libchan. // testsuites.RegisterIPCSuite(driverName, nil, testsuites.NeverSkip) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/api/0000755000175000017500000000000012502424227023200 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/api/v2/0000755000175000017500000000000012502424227023527 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/api/v2/errors_test.go0000644000175000017500000001101712502424227026431 0ustar tianontianonpackage v2 import ( "encoding/json" "reflect" "testing" "github.com/docker/distribution/digest" ) // TestErrorCodes ensures that error code format, mappings and // marshaling/unmarshaling. round trips are stable. func TestErrorCodes(t *testing.T) { for _, desc := range errorDescriptors { if desc.Code.String() != desc.Value { t.Fatalf("error code string incorrect: %q != %q", desc.Code.String(), desc.Value) } if desc.Code.Message() != desc.Message { t.Fatalf("incorrect message for error code %v: %q != %q", desc.Code, desc.Code.Message(), desc.Message) } // Serialize the error code using the json library to ensure that we // get a string and it works round trip. p, err := json.Marshal(desc.Code) if err != nil { t.Fatalf("error marshaling error code %v: %v", desc.Code, err) } if len(p) <= 0 { t.Fatalf("expected content in marshaled before for error code %v", desc.Code) } // First, unmarshal to interface and ensure we have a string. var ecUnspecified interface{} if err := json.Unmarshal(p, &ecUnspecified); err != nil { t.Fatalf("error unmarshaling error code %v: %v", desc.Code, err) } if _, ok := ecUnspecified.(string); !ok { t.Fatalf("expected a string for error code %v on unmarshal got a %T", desc.Code, ecUnspecified) } // Now, unmarshal with the error code type and ensure they are equal var ecUnmarshaled ErrorCode if err := json.Unmarshal(p, &ecUnmarshaled); err != nil { t.Fatalf("error unmarshaling error code %v: %v", desc.Code, err) } if ecUnmarshaled != desc.Code { t.Fatalf("unexpected error code during error code marshal/unmarshal: %v != %v", ecUnmarshaled, desc.Code) } } } // TestErrorsManagement does a quick check of the Errors type to ensure that // members are properly pushed and marshaled. func TestErrorsManagement(t *testing.T) { var errs Errors errs.Push(ErrorCodeDigestInvalid) errs.Push(ErrorCodeBlobUnknown, map[string]digest.Digest{"digest": "sometestblobsumdoesntmatter"}) p, err := json.Marshal(errs) if err != nil { t.Fatalf("error marashaling errors: %v", err) } expectedJSON := "{\"errors\":[{\"code\":\"DIGEST_INVALID\",\"message\":\"provided digest did not match uploaded content\"},{\"code\":\"BLOB_UNKNOWN\",\"message\":\"blob unknown to registry\",\"detail\":{\"digest\":\"sometestblobsumdoesntmatter\"}}]}" if string(p) != expectedJSON { t.Fatalf("unexpected json: %q != %q", string(p), expectedJSON) } errs.Clear() errs.Push(ErrorCodeUnknown) expectedJSON = "{\"errors\":[{\"code\":\"UNKNOWN\",\"message\":\"unknown error\"}]}" p, err = json.Marshal(errs) if err != nil { t.Fatalf("error marashaling errors: %v", err) } if string(p) != expectedJSON { t.Fatalf("unexpected json: %q != %q", string(p), expectedJSON) } } // TestMarshalUnmarshal ensures that api errors can round trip through json // without losing information. func TestMarshalUnmarshal(t *testing.T) { var errors Errors for _, testcase := range []struct { description string err Error }{ { description: "unknown error", err: Error{ Code: ErrorCodeUnknown, Message: ErrorCodeUnknown.Descriptor().Message, }, }, { description: "unknown manifest", err: Error{ Code: ErrorCodeManifestUnknown, Message: ErrorCodeManifestUnknown.Descriptor().Message, }, }, { description: "unknown manifest", err: Error{ Code: ErrorCodeBlobUnknown, Message: ErrorCodeBlobUnknown.Descriptor().Message, Detail: map[string]interface{}{"digest": "asdfqwerqwerqwerqwer"}, }, }, } { fatalf := func(format string, args ...interface{}) { t.Fatalf(testcase.description+": "+format, args...) } unexpectedErr := func(err error) { fatalf("unexpected error: %v", err) } p, err := json.Marshal(testcase.err) if err != nil { unexpectedErr(err) } var unmarshaled Error if err := json.Unmarshal(p, &unmarshaled); err != nil { unexpectedErr(err) } if !reflect.DeepEqual(unmarshaled, testcase.err) { fatalf("errors not equal after round trip: %#v != %#v", unmarshaled, testcase.err) } // Roll everything up into an error response envelope. errors.PushErr(testcase.err) } p, err := json.Marshal(errors) if err != nil { t.Fatalf("unexpected error marshaling error envelope: %v", err) } var unmarshaled Errors if err := json.Unmarshal(p, &unmarshaled); err != nil { t.Fatalf("unexpected error unmarshaling error envelope: %v", err) } if !reflect.DeepEqual(unmarshaled, errors) { t.Fatalf("errors not equal after round trip: %#v != %#v", unmarshaled, errors) } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/api/v2/errors.go0000644000175000017500000001171212502424227025374 0ustar tianontianonpackage v2 import ( "fmt" "strings" ) // ErrorCode represents the error type. The errors are serialized via strings // and the integer format may change and should *never* be exported. type ErrorCode int const ( // ErrorCodeUnknown is a catch-all for errors not defined below. ErrorCodeUnknown ErrorCode = iota // ErrorCodeUnsupported is returned when an operation is not supported. ErrorCodeUnsupported // ErrorCodeUnauthorized is returned if a request is not authorized. ErrorCodeUnauthorized // ErrorCodeDigestInvalid is returned when uploading a blob if the // provided digest does not match the blob contents. ErrorCodeDigestInvalid // ErrorCodeSizeInvalid is returned when uploading a blob if the provided // size does not match the content length. ErrorCodeSizeInvalid // ErrorCodeNameInvalid is returned when the name in the manifest does not // match the provided name. ErrorCodeNameInvalid // ErrorCodeTagInvalid is returned when the tag in the manifest does not // match the provided tag. ErrorCodeTagInvalid // ErrorCodeNameUnknown when the repository name is not known. ErrorCodeNameUnknown // ErrorCodeManifestUnknown returned when image manifest is unknown. ErrorCodeManifestUnknown // ErrorCodeManifestInvalid returned when an image manifest is invalid, // typically during a PUT operation. This error encompasses all errors // encountered during manifest validation that aren't signature errors. ErrorCodeManifestInvalid // ErrorCodeManifestUnverified is returned when the manifest fails // signature verfication. ErrorCodeManifestUnverified // ErrorCodeBlobUnknown is returned when a blob is unknown to the // registry. This can happen when the manifest references a nonexistent // layer or the result is not found by a blob fetch. ErrorCodeBlobUnknown // ErrorCodeBlobUploadUnknown is returned when an upload is unknown. ErrorCodeBlobUploadUnknown // ErrorCodeBlobUploadInvalid is returned when an upload is invalid. ErrorCodeBlobUploadInvalid ) // ParseErrorCode attempts to parse the error code string, returning // ErrorCodeUnknown if the error is not known. func ParseErrorCode(s string) ErrorCode { desc, ok := idToDescriptors[s] if !ok { return ErrorCodeUnknown } return desc.Code } // Descriptor returns the descriptor for the error code. func (ec ErrorCode) Descriptor() ErrorDescriptor { d, ok := errorCodeToDescriptors[ec] if !ok { return ErrorCodeUnknown.Descriptor() } return d } // String returns the canonical identifier for this error code. func (ec ErrorCode) String() string { return ec.Descriptor().Value } // Message returned the human-readable error message for this error code. func (ec ErrorCode) Message() string { return ec.Descriptor().Message } // MarshalText encodes the receiver into UTF-8-encoded text and returns the // result. func (ec ErrorCode) MarshalText() (text []byte, err error) { return []byte(ec.String()), nil } // UnmarshalText decodes the form generated by MarshalText. func (ec *ErrorCode) UnmarshalText(text []byte) error { desc, ok := idToDescriptors[string(text)] if !ok { desc = ErrorCodeUnknown.Descriptor() } *ec = desc.Code return nil } // Error provides a wrapper around ErrorCode with extra Details provided. type Error struct { Code ErrorCode `json:"code"` Message string `json:"message,omitempty"` Detail interface{} `json:"detail,omitempty"` } // Error returns a human readable representation of the error. func (e Error) Error() string { return fmt.Sprintf("%s: %s", strings.ToLower(strings.Replace(e.Code.String(), "_", " ", -1)), e.Message) } // Errors provides the envelope for multiple errors and a few sugar methods // for use within the application. type Errors struct { Errors []Error `json:"errors,omitempty"` } // Push pushes an error on to the error stack, with the optional detail // argument. It is a programming error (ie panic) to push more than one // detail at a time. func (errs *Errors) Push(code ErrorCode, details ...interface{}) { if len(details) > 1 { panic("please specify zero or one detail items for this error") } var detail interface{} if len(details) > 0 { detail = details[0] } if err, ok := detail.(error); ok { detail = err.Error() } errs.PushErr(Error{ Code: code, Message: code.Message(), Detail: detail, }) } // PushErr pushes an error interface onto the error stack. func (errs *Errors) PushErr(err error) { switch err.(type) { case Error: errs.Errors = append(errs.Errors, err.(Error)) default: errs.Errors = append(errs.Errors, Error{Message: err.Error()}) } } func (errs *Errors) Error() string { switch errs.Len() { case 0: return "" case 1: return errs.Errors[0].Error() default: msg := "errors:\n" for _, err := range errs.Errors { msg += err.Error() + "\n" } return msg } } // Clear clears the errors. func (errs *Errors) Clear() { errs.Errors = errs.Errors[:0] } // Len returns the current number of errors. func (errs *Errors) Len() int { return len(errs.Errors) } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/api/v2/doc.go0000644000175000017500000000073512502424227024630 0ustar tianontianon// Package v2 describes routes, urls and the error codes used in the Docker // Registry JSON HTTP API V2. In addition to declarations, descriptors are // provided for routes and error codes that can be used for implementation and // automatically generating documentation. // // Definitions here are considered to be locked down for the V2 registry api. // Any changes must be considered carefully and should not proceed without a // change proposal in docker core. package v2 distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/api/v2/urls_test.go0000644000175000017500000001323412502424227026105 0ustar tianontianonpackage v2 import ( "net/http" "net/url" "testing" ) type urlBuilderTestCase struct { description string expectedPath string build func() (string, error) } func makeURLBuilderTestCases(urlBuilder *URLBuilder) []urlBuilderTestCase { return []urlBuilderTestCase{ { description: "test base url", expectedPath: "/v2/", build: urlBuilder.BuildBaseURL, }, { description: "test tags url", expectedPath: "/v2/foo/bar/tags/list", build: func() (string, error) { return urlBuilder.BuildTagsURL("foo/bar") }, }, { description: "test manifest url", expectedPath: "/v2/foo/bar/manifests/tag", build: func() (string, error) { return urlBuilder.BuildManifestURL("foo/bar", "tag") }, }, { description: "build blob url", expectedPath: "/v2/foo/bar/blobs/tarsum.v1+sha256:abcdef0123456789", build: func() (string, error) { return urlBuilder.BuildBlobURL("foo/bar", "tarsum.v1+sha256:abcdef0123456789") }, }, { description: "build blob upload url", expectedPath: "/v2/foo/bar/blobs/uploads/", build: func() (string, error) { return urlBuilder.BuildBlobUploadURL("foo/bar") }, }, { description: "build blob upload url with digest and size", expectedPath: "/v2/foo/bar/blobs/uploads/?digest=tarsum.v1%2Bsha256%3Aabcdef0123456789&size=10000", build: func() (string, error) { return urlBuilder.BuildBlobUploadURL("foo/bar", url.Values{ "size": []string{"10000"}, "digest": []string{"tarsum.v1+sha256:abcdef0123456789"}, }) }, }, { description: "build blob upload chunk url", expectedPath: "/v2/foo/bar/blobs/uploads/uuid-part", build: func() (string, error) { return urlBuilder.BuildBlobUploadChunkURL("foo/bar", "uuid-part") }, }, { description: "build blob upload chunk url with digest and size", expectedPath: "/v2/foo/bar/blobs/uploads/uuid-part?digest=tarsum.v1%2Bsha256%3Aabcdef0123456789&size=10000", build: func() (string, error) { return urlBuilder.BuildBlobUploadChunkURL("foo/bar", "uuid-part", url.Values{ "size": []string{"10000"}, "digest": []string{"tarsum.v1+sha256:abcdef0123456789"}, }) }, }, } } // TestURLBuilder tests the various url building functions, ensuring they are // returning the expected values. func TestURLBuilder(t *testing.T) { roots := []string{ "http://example.com", "https://example.com", "http://localhost:5000", "https://localhost:5443", } for _, root := range roots { urlBuilder, err := NewURLBuilderFromString(root) if err != nil { t.Fatalf("unexpected error creating urlbuilder: %v", err) } for _, testCase := range makeURLBuilderTestCases(urlBuilder) { url, err := testCase.build() if err != nil { t.Fatalf("%s: error building url: %v", testCase.description, err) } expectedURL := root + testCase.expectedPath if url != expectedURL { t.Fatalf("%s: %q != %q", testCase.description, url, expectedURL) } } } } func TestURLBuilderWithPrefix(t *testing.T) { roots := []string{ "http://example.com/prefix/", "https://example.com/prefix/", "http://localhost:5000/prefix/", "https://localhost:5443/prefix/", } for _, root := range roots { urlBuilder, err := NewURLBuilderFromString(root) if err != nil { t.Fatalf("unexpected error creating urlbuilder: %v", err) } for _, testCase := range makeURLBuilderTestCases(urlBuilder) { url, err := testCase.build() if err != nil { t.Fatalf("%s: error building url: %v", testCase.description, err) } expectedURL := root[0:len(root)-1] + testCase.expectedPath if url != expectedURL { t.Fatalf("%s: %q != %q", testCase.description, url, expectedURL) } } } } type builderFromRequestTestCase struct { request *http.Request base string } func TestBuilderFromRequest(t *testing.T) { u, err := url.Parse("http://example.com") if err != nil { t.Fatal(err) } forwardedProtoHeader := make(http.Header, 1) forwardedProtoHeader.Set("X-Forwarded-Proto", "https") testRequests := []struct { request *http.Request base string }{ { request: &http.Request{URL: u, Host: u.Host}, base: "http://example.com", }, { request: &http.Request{URL: u, Host: u.Host, Header: forwardedProtoHeader}, base: "https://example.com", }, } for _, tr := range testRequests { builder := NewURLBuilderFromRequest(tr.request) for _, testCase := range makeURLBuilderTestCases(builder) { url, err := testCase.build() if err != nil { t.Fatalf("%s: error building url: %v", testCase.description, err) } expectedURL := tr.base + testCase.expectedPath if url != expectedURL { t.Fatalf("%s: %q != %q", testCase.description, url, expectedURL) } } } } func TestBuilderFromRequestWithPrefix(t *testing.T) { u, err := url.Parse("http://example.com/prefix/v2/") if err != nil { t.Fatal(err) } forwardedProtoHeader := make(http.Header, 1) forwardedProtoHeader.Set("X-Forwarded-Proto", "https") testRequests := []struct { request *http.Request base string }{ { request: &http.Request{URL: u, Host: u.Host}, base: "http://example.com/prefix/", }, { request: &http.Request{URL: u, Host: u.Host, Header: forwardedProtoHeader}, base: "https://example.com/prefix/", }, } for _, tr := range testRequests { builder := NewURLBuilderFromRequest(tr.request) for _, testCase := range makeURLBuilderTestCases(builder) { url, err := testCase.build() if err != nil { t.Fatalf("%s: error building url: %v", testCase.description, err) } expectedURL := tr.base[0:len(tr.base)-1] + testCase.expectedPath if url != expectedURL { t.Fatalf("%s: %q != %q", testCase.description, url, expectedURL) } } } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/api/v2/routes.go0000644000175000017500000000227112502424227025401 0ustar tianontianonpackage v2 import "github.com/gorilla/mux" // The following are definitions of the name under which all V2 routes are // registered. These symbols can be used to look up a route based on the name. const ( RouteNameBase = "base" RouteNameManifest = "manifest" RouteNameTags = "tags" RouteNameBlob = "blob" RouteNameBlobUpload = "blob-upload" RouteNameBlobUploadChunk = "blob-upload-chunk" ) var allEndpoints = []string{ RouteNameManifest, RouteNameTags, RouteNameBlob, RouteNameBlobUpload, RouteNameBlobUploadChunk, } // Router builds a gorilla router with named routes for the various API // methods. This can be used directly by both server implementations and // clients. func Router() *mux.Router { return RouterWithPrefix("") } // RouterWithPrefix builds a gorilla router with a configured prefix // on all routes. func RouterWithPrefix(prefix string) *mux.Router { rootRouter := mux.NewRouter() router := rootRouter if prefix != "" { router = router.PathPrefix(prefix).Subrouter() } router.StrictSlash(true) for _, descriptor := range routeDescriptors { router.Path(descriptor.Path).Name(descriptor.Name) } return rootRouter } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/api/v2/descriptors.go0000644000175000017500000013660012502424227026425 0ustar tianontianonpackage v2 import ( "net/http" "regexp" "github.com/docker/distribution/digest" ) var ( nameParameterDescriptor = ParameterDescriptor{ Name: "name", Type: "string", Format: RepositoryNameRegexp.String(), Required: true, Description: `Name of the target repository.`, } tagParameterDescriptor = ParameterDescriptor{ Name: "tag", Type: "string", Format: TagNameRegexp.String(), Required: true, Description: `Tag of the target manifiest.`, } uuidParameterDescriptor = ParameterDescriptor{ Name: "uuid", Type: "opaque", Required: true, Description: `A uuid identifying the upload. This field can accept almost anything.`, } digestPathParameter = ParameterDescriptor{ Name: "digest", Type: "path", Required: true, Format: digest.DigestRegexp.String(), Description: `Digest of desired blob.`, } hostHeader = ParameterDescriptor{ Name: "Host", Type: "string", Description: "Standard HTTP Host Header. Should be set to the registry host.", Format: "", Examples: []string{"registry-1.docker.io"}, } authHeader = ParameterDescriptor{ Name: "Authorization", Type: "string", Description: "An RFC7235 compliant authorization header.", Format: " ", Examples: []string{"Bearer dGhpcyBpcyBhIGZha2UgYmVhcmVyIHRva2VuIQ=="}, } authChallengeHeader = ParameterDescriptor{ Name: "WWW-Authenticate", Type: "string", Description: "An RFC7235 compliant authentication challenge header.", Format: ` realm="", ..."`, Examples: []string{ `Bearer realm="https://auth.docker.com/", service="registry.docker.com", scopes="repository:library/ubuntu:pull"`, }, } contentLengthZeroHeader = ParameterDescriptor{ Name: "Content-Length", Description: "The `Content-Length` header must be zero and the body must be empty.", Type: "integer", Format: "0", } dockerUploadUUIDHeader = ParameterDescriptor{ Name: "Docker-Upload-UUID", Description: "Identifies the docker upload uuid for the current request.", Type: "uuid", Format: "", } digestHeader = ParameterDescriptor{ Name: "Docker-Content-Digest", Description: "Digest of the targeted content for the request.", Type: "digest", Format: "", } unauthorizedResponse = ResponseDescriptor{ Description: "The client does not have access to the repository.", StatusCode: http.StatusUnauthorized, Headers: []ParameterDescriptor{ authChallengeHeader, { Name: "Content-Length", Type: "integer", Description: "Length of the JSON error response body.", Format: "", }, }, ErrorCodes: []ErrorCode{ ErrorCodeUnauthorized, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: unauthorizedErrorsBody, }, } unauthorizedResponsePush = ResponseDescriptor{ Description: "The client does not have access to push to the repository.", StatusCode: http.StatusUnauthorized, Headers: []ParameterDescriptor{ authChallengeHeader, { Name: "Content-Length", Type: "integer", Description: "Length of the JSON error response body.", Format: "", }, }, ErrorCodes: []ErrorCode{ ErrorCodeUnauthorized, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: unauthorizedErrorsBody, }, } ) const ( manifestBody = `{ "name": , "tag": , "fsLayers": [ { "blobSum": }, ... ] ], "history": , "signature": }` errorsBody = `{ "errors:" [ { "code": , "message": "", "detail": ... }, ... ] }` unauthorizedErrorsBody = `{ "errors:" [ { "code": "UNAUTHORIZED", "message": "access to the requested resource is not authorized", "detail": ... }, ... ] }` ) // APIDescriptor exports descriptions of the layout of the v2 registry API. var APIDescriptor = struct { // RouteDescriptors provides a list of the routes available in the API. RouteDescriptors []RouteDescriptor // ErrorDescriptors provides a list of the error codes and their // associated documentation and metadata. ErrorDescriptors []ErrorDescriptor }{ RouteDescriptors: routeDescriptors, ErrorDescriptors: errorDescriptors, } // RouteDescriptor describes a route specified by name. type RouteDescriptor struct { // Name is the name of the route, as specified in RouteNameXXX exports. // These names a should be considered a unique reference for a route. If // the route is registered with gorilla, this is the name that will be // used. Name string // Path is a gorilla/mux-compatible regexp that can be used to match the // route. For any incoming method and path, only one route descriptor // should match. Path string // Entity should be a short, human-readalbe description of the object // targeted by the endpoint. Entity string // Description should provide an accurate overview of the functionality // provided by the route. Description string // Methods should describe the various HTTP methods that may be used on // this route, including request and response formats. Methods []MethodDescriptor } // MethodDescriptor provides a description of the requests that may be // conducted with the target method. type MethodDescriptor struct { // Method is an HTTP method, such as GET, PUT or POST. Method string // Description should provide an overview of the functionality provided by // the covered method, suitable for use in documentation. Use of markdown // here is encouraged. Description string // Requests is a slice of request descriptors enumerating how this // endpoint may be used. Requests []RequestDescriptor } // RequestDescriptor covers a particular set of headers and parameters that // can be carried out with the parent method. Its most helpful to have one // RequestDescriptor per API use case. type RequestDescriptor struct { // Name provides a short identifier for the request, usable as a title or // to provide quick context for the particalar request. Name string // Description should cover the requests purpose, covering any details for // this particular use case. Description string // Headers describes headers that must be used with the HTTP request. Headers []ParameterDescriptor // PathParameters enumerate the parameterized path components for the // given request, as defined in the route's regular expression. PathParameters []ParameterDescriptor // QueryParameters provides a list of query parameters for the given // request. QueryParameters []ParameterDescriptor // Body describes the format of the request body. Body BodyDescriptor // Successes enumerates the possible responses that are considered to be // the result of a successful request. Successes []ResponseDescriptor // Failures covers the possible failures from this particular request. Failures []ResponseDescriptor } // ResponseDescriptor describes the components of an API response. type ResponseDescriptor struct { // Name provides a short identifier for the response, usable as a title or // to provide quick context for the particalar response. Name string // Description should provide a brief overview of the role of the // response. Description string // StatusCode specifies the status recieved by this particular response. StatusCode int // Headers covers any headers that may be returned from the response. Headers []ParameterDescriptor // ErrorCodes enumerates the error codes that may be returned along with // the response. ErrorCodes []ErrorCode // Body describes the body of the response, if any. Body BodyDescriptor } // BodyDescriptor describes a request body and its expected content type. For // the most part, it should be example json or some placeholder for body // data in documentation. type BodyDescriptor struct { ContentType string Format string } // ParameterDescriptor describes the format of a request parameter, which may // be a header, path parameter or query parameter. type ParameterDescriptor struct { // Name is the name of the parameter, either of the path component or // query parameter. Name string // Type specifies the type of the parameter, such as string, integer, etc. Type string // Description provides a human-readable description of the parameter. Description string // Required means the field is required when set. Required bool // Format is a specifying the string format accepted by this parameter. Format string // Regexp is a compiled regular expression that can be used to validate // the contents of the parameter. Regexp *regexp.Regexp // Examples provides multiple examples for the values that might be valid // for this parameter. Examples []string } // ErrorDescriptor provides relevant information about a given error code. type ErrorDescriptor struct { // Code is the error code that this descriptor describes. Code ErrorCode // Value provides a unique, string key, often captilized with // underscores, to identify the error code. This value is used as the // keyed value when serializing api errors. Value string // Message is a short, human readable decription of the error condition // included in API responses. Message string // Description provides a complete account of the errors purpose, suitable // for use in documentation. Description string // HTTPStatusCodes provides a list of status under which this error // condition may arise. If it is empty, the error condition may be seen // for any status code. HTTPStatusCodes []int } var routeDescriptors = []RouteDescriptor{ { Name: RouteNameBase, Path: "/v2/", Entity: "Base", Description: `Base V2 API route. Typically, this can be used for lightweight version checks and to validate registry authorization.`, Methods: []MethodDescriptor{ { Method: "GET", Description: "Check that the endpoint implements Docker Registry API V2.", Requests: []RequestDescriptor{ { Headers: []ParameterDescriptor{ hostHeader, authHeader, }, Successes: []ResponseDescriptor{ { Description: "The API implements V2 protocol and is accessible.", StatusCode: http.StatusOK, }, }, Failures: []ResponseDescriptor{ { Description: "The client is not authorized to access the registry.", StatusCode: http.StatusUnauthorized, Headers: []ParameterDescriptor{ authChallengeHeader, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, ErrorCodes: []ErrorCode{ ErrorCodeUnauthorized, }, }, { Description: "The registry does not implement the V2 API.", StatusCode: http.StatusNotFound, }, }, }, }, }, }, }, { Name: RouteNameTags, Path: "/v2/{name:" + RepositoryNameRegexp.String() + "}/tags/list", Entity: "Tags", Description: "Retrieve information about tags.", Methods: []MethodDescriptor{ { Method: "GET", Description: "Fetch the tags under the repository identified by `name`.", Requests: []RequestDescriptor{ { Headers: []ParameterDescriptor{ hostHeader, authHeader, }, PathParameters: []ParameterDescriptor{ nameParameterDescriptor, }, Successes: []ResponseDescriptor{ { StatusCode: http.StatusOK, Description: "A list of tags for the named repository.", Headers: []ParameterDescriptor{ { Name: "Content-Length", Type: "integer", Description: "Length of the JSON response body.", Format: "", }, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: `{ "name": , "tags": [ , ... ] }`, }, }, }, Failures: []ResponseDescriptor{ { StatusCode: http.StatusNotFound, Description: "The repository is not known to the registry.", Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, ErrorCodes: []ErrorCode{ ErrorCodeNameUnknown, }, }, { StatusCode: http.StatusUnauthorized, Description: "The client does not have access to the repository.", Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, ErrorCodes: []ErrorCode{ ErrorCodeUnauthorized, }, }, }, }, }, }, }, }, { Name: RouteNameManifest, Path: "/v2/{name:" + RepositoryNameRegexp.String() + "}/manifests/{reference:" + TagNameRegexp.String() + "|" + digest.DigestRegexp.String() + "}", Entity: "Manifest", Description: "Create, update and retrieve manifests.", Methods: []MethodDescriptor{ { Method: "GET", Description: "Fetch the manifest identified by `name` and `reference` where `reference` can be a tag or digest.", Requests: []RequestDescriptor{ { Headers: []ParameterDescriptor{ hostHeader, authHeader, }, PathParameters: []ParameterDescriptor{ nameParameterDescriptor, tagParameterDescriptor, }, Successes: []ResponseDescriptor{ { Description: "The manifest idenfied by `name` and `reference`. The contents can be used to identify and resolve resources required to run the specified image.", StatusCode: http.StatusOK, Headers: []ParameterDescriptor{ digestHeader, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: manifestBody, }, }, }, Failures: []ResponseDescriptor{ { Description: "The name or reference was invalid.", StatusCode: http.StatusBadRequest, ErrorCodes: []ErrorCode{ ErrorCodeNameInvalid, ErrorCodeTagInvalid, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, }, { StatusCode: http.StatusUnauthorized, Description: "The client does not have access to the repository.", Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, ErrorCodes: []ErrorCode{ ErrorCodeUnauthorized, }, }, { Description: "The named manifest is not known to the registry.", StatusCode: http.StatusNotFound, ErrorCodes: []ErrorCode{ ErrorCodeNameUnknown, ErrorCodeManifestUnknown, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, }, }, }, }, }, { Method: "PUT", Description: "Put the manifest identified by `name` and `reference` where `reference` can be a tag or digest.", Requests: []RequestDescriptor{ { Headers: []ParameterDescriptor{ hostHeader, authHeader, }, PathParameters: []ParameterDescriptor{ nameParameterDescriptor, tagParameterDescriptor, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: manifestBody, }, Successes: []ResponseDescriptor{ { Description: "The manifest has been accepted by the registry and is stored under the specified `name` and `tag`.", StatusCode: http.StatusAccepted, Headers: []ParameterDescriptor{ { Name: "Location", Type: "url", Description: "The canonical location url of the uploaded manifest.", Format: "", }, contentLengthZeroHeader, digestHeader, }, }, }, Failures: []ResponseDescriptor{ { Name: "Invalid Manifest", Description: "The received manifest was invalid in some way, as described by the error codes. The client should resolve the issue and retry the request.", StatusCode: http.StatusBadRequest, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, ErrorCodes: []ErrorCode{ ErrorCodeNameInvalid, ErrorCodeTagInvalid, ErrorCodeManifestInvalid, ErrorCodeManifestUnverified, ErrorCodeBlobUnknown, }, }, { StatusCode: http.StatusUnauthorized, Description: "The client does not have permission to push to the repository.", Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, ErrorCodes: []ErrorCode{ ErrorCodeUnauthorized, }, }, { Name: "Missing Layer(s)", Description: "One or more layers may be missing during a manifest upload. If so, the missing layers will be enumerated in the error response.", StatusCode: http.StatusBadRequest, ErrorCodes: []ErrorCode{ ErrorCodeBlobUnknown, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: `{ "errors:" [{ "code": "BLOB_UNKNOWN", "message": "blob unknown to registry", "detail": { "digest": } }, ... ] }`, }, }, { StatusCode: http.StatusUnauthorized, Headers: []ParameterDescriptor{ authChallengeHeader, { Name: "Content-Length", Type: "integer", Description: "Length of the JSON error response body.", Format: "", }, }, ErrorCodes: []ErrorCode{ ErrorCodeUnauthorized, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, }, }, }, }, }, { Method: "DELETE", Description: "Delete the manifest identified by `name` and `reference` where `reference` can be a tag or digest.", Requests: []RequestDescriptor{ { Headers: []ParameterDescriptor{ hostHeader, authHeader, }, PathParameters: []ParameterDescriptor{ nameParameterDescriptor, tagParameterDescriptor, }, Successes: []ResponseDescriptor{ { StatusCode: http.StatusAccepted, }, }, Failures: []ResponseDescriptor{ { Name: "Invalid Name or Tag", Description: "The specified `name` or `tag` were invalid and the delete was unable to proceed.", StatusCode: http.StatusBadRequest, ErrorCodes: []ErrorCode{ ErrorCodeNameInvalid, ErrorCodeTagInvalid, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, }, { StatusCode: http.StatusUnauthorized, Headers: []ParameterDescriptor{ authChallengeHeader, { Name: "Content-Length", Type: "integer", Description: "Length of the JSON error response body.", Format: "", }, }, ErrorCodes: []ErrorCode{ ErrorCodeUnauthorized, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, }, { Name: "Unknown Manifest", Description: "The specified `name` or `tag` are unknown to the registry and the delete was unable to proceed. Clients can assume the manifest was already deleted if this response is returned.", StatusCode: http.StatusNotFound, ErrorCodes: []ErrorCode{ ErrorCodeNameUnknown, ErrorCodeManifestUnknown, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, }, }, }, }, }, }, }, { Name: RouteNameBlob, Path: "/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/{digest:" + digest.DigestRegexp.String() + "}", Entity: "Blob", Description: "Fetch the blob identified by `name` and `digest`. Used to fetch layers by tarsum digest.", Methods: []MethodDescriptor{ { Method: "GET", Description: "Retrieve the blob from the registry identified by `digest`. A `HEAD` request can also be issued to this endpoint to obtain resource information without receiving all data.", Requests: []RequestDescriptor{ { Name: "Fetch Blob", Headers: []ParameterDescriptor{ hostHeader, authHeader, }, PathParameters: []ParameterDescriptor{ nameParameterDescriptor, digestPathParameter, }, Successes: []ResponseDescriptor{ { Description: "The blob identified by `digest` is available. The blob content will be present in the body of the request.", StatusCode: http.StatusOK, Headers: []ParameterDescriptor{ { Name: "Content-Length", Type: "integer", Description: "The length of the requested blob content.", Format: "", }, digestHeader, }, Body: BodyDescriptor{ ContentType: "application/octet-stream", Format: "", }, }, { Description: "The blob identified by `digest` is available at the provided location.", StatusCode: http.StatusTemporaryRedirect, Headers: []ParameterDescriptor{ { Name: "Location", Type: "url", Description: "The location where the layer should be accessible.", Format: "", }, digestHeader, }, }, }, Failures: []ResponseDescriptor{ { Description: "There was a problem with the request that needs to be addressed by the client, such as an invalid `name` or `tag`.", StatusCode: http.StatusBadRequest, ErrorCodes: []ErrorCode{ ErrorCodeNameInvalid, ErrorCodeDigestInvalid, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, }, unauthorizedResponse, { Description: "The blob, identified by `name` and `digest`, is unknown to the registry.", StatusCode: http.StatusNotFound, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, ErrorCodes: []ErrorCode{ ErrorCodeNameUnknown, ErrorCodeBlobUnknown, }, }, }, }, { Name: "Fetch Blob Part", Description: "This endpoint may also support RFC7233 compliant range requests. Support can be detected by issuing a HEAD request. If the header `Accept-Range: bytes` is returned, range requests can be used to fetch partial content.", Headers: []ParameterDescriptor{ hostHeader, authHeader, { Name: "Range", Type: "string", Description: "HTTP Range header specifying blob chunk.", Format: "bytes=-", }, }, PathParameters: []ParameterDescriptor{ nameParameterDescriptor, digestPathParameter, }, Successes: []ResponseDescriptor{ { Description: "The blob identified by `digest` is available. The specified chunk of blob content will be present in the body of the request.", StatusCode: http.StatusPartialContent, Headers: []ParameterDescriptor{ { Name: "Content-Length", Type: "integer", Description: "The length of the requested blob chunk.", Format: "", }, { Name: "Content-Range", Type: "byte range", Description: "Content range of blob chunk.", Format: "bytes -/", }, }, Body: BodyDescriptor{ ContentType: "application/octet-stream", Format: "", }, }, }, Failures: []ResponseDescriptor{ { Description: "There was a problem with the request that needs to be addressed by the client, such as an invalid `name` or `tag`.", StatusCode: http.StatusBadRequest, ErrorCodes: []ErrorCode{ ErrorCodeNameInvalid, ErrorCodeDigestInvalid, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, }, unauthorizedResponse, { StatusCode: http.StatusNotFound, ErrorCodes: []ErrorCode{ ErrorCodeNameUnknown, ErrorCodeBlobUnknown, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, }, { Description: "The range specification cannot be satisfied for the requested content. This can happen when the range is not formatted correctly or if the range is outside of the valid size of the content.", StatusCode: http.StatusRequestedRangeNotSatisfiable, }, }, }, }, }, // TODO(stevvooe): We may want to add a PUT request here to // kickoff an upload of a blob, integrated with the blob upload // API. }, }, { Name: RouteNameBlobUpload, Path: "/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/uploads/", Entity: "Intiate Blob Upload", Description: "Initiate a blob upload. This endpoint can be used to create resumable uploads or monolithic uploads.", Methods: []MethodDescriptor{ { Method: "POST", Description: "Initiate a resumable blob upload. If successful, an upload location will be provided to complete the upload. Optionally, if the `digest` parameter is present, the request body will be used to complete the upload in a single request.", Requests: []RequestDescriptor{ { Name: "Initiate Monolithic Blob Upload", Description: "Upload a blob identified by the `digest` parameter in single request. This upload will not be resumable unless a recoverable error is returned.", Headers: []ParameterDescriptor{ hostHeader, authHeader, { Name: "Content-Length", Type: "integer", Format: "", }, }, PathParameters: []ParameterDescriptor{ nameParameterDescriptor, }, QueryParameters: []ParameterDescriptor{ { Name: "digest", Type: "query", Format: "", Regexp: digest.DigestRegexp, Description: `Digest of uploaded blob. If present, the upload will be completed, in a single request, with contents of the request body as the resulting blob.`, }, }, Body: BodyDescriptor{ ContentType: "application/octect-stream", Format: "", }, Successes: []ResponseDescriptor{ { Description: "The blob has been created in the registry and is available at the provided location.", StatusCode: http.StatusCreated, Headers: []ParameterDescriptor{ { Name: "Location", Type: "url", Format: "", }, contentLengthZeroHeader, dockerUploadUUIDHeader, }, }, }, Failures: []ResponseDescriptor{ { Name: "Invalid Name or Digest", StatusCode: http.StatusBadRequest, ErrorCodes: []ErrorCode{ ErrorCodeDigestInvalid, ErrorCodeNameInvalid, }, }, unauthorizedResponsePush, }, }, { Name: "Initiate Resumable Blob Upload", Description: "Initiate a resumable blob upload with an empty request body.", Headers: []ParameterDescriptor{ hostHeader, authHeader, contentLengthZeroHeader, }, PathParameters: []ParameterDescriptor{ nameParameterDescriptor, }, Successes: []ResponseDescriptor{ { Description: "The upload has been created. The `Location` header must be used to complete the upload. The response should be identical to a `GET` request on the contents of the returned `Location` header.", StatusCode: http.StatusAccepted, Headers: []ParameterDescriptor{ contentLengthZeroHeader, { Name: "Location", Type: "url", Format: "/v2//blobs/uploads/", Description: "The location of the created upload. Clients should use the contents verbatim to complete the upload, adding parameters where required.", }, { Name: "Range", Format: "0-0", Description: "Range header indicating the progress of the upload. When starting an upload, it will return an empty range, since no content has been received.", }, dockerUploadUUIDHeader, }, }, }, Failures: []ResponseDescriptor{ { Name: "Invalid Name or Digest", StatusCode: http.StatusBadRequest, ErrorCodes: []ErrorCode{ ErrorCodeDigestInvalid, ErrorCodeNameInvalid, }, }, unauthorizedResponsePush, }, }, }, }, }, }, { Name: RouteNameBlobUploadChunk, Path: "/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/uploads/{uuid}", Entity: "Blob Upload", Description: "Interact with blob uploads. Clients should never assemble URLs for this endpoint and should only take it through the `Location` header on related API requests. The `Location` header and its parameters should be preserved by clients, using the latest value returned via upload related API calls.", Methods: []MethodDescriptor{ { Method: "GET", Description: "Retrieve status of upload identified by `uuid`. The primary purpose of this endpoint is to resolve the current status of a resumable upload.", Requests: []RequestDescriptor{ { Description: "Retrieve the progress of the current upload, as reported by the `Range` header.", Headers: []ParameterDescriptor{ hostHeader, authHeader, }, PathParameters: []ParameterDescriptor{ nameParameterDescriptor, uuidParameterDescriptor, }, Successes: []ResponseDescriptor{ { Name: "Upload Progress", Description: "The upload is known and in progress. The last received offset is available in the `Range` header.", StatusCode: http.StatusNoContent, Headers: []ParameterDescriptor{ { Name: "Range", Type: "header", Format: "0-", Description: "Range indicating the current progress of the upload.", }, contentLengthZeroHeader, dockerUploadUUIDHeader, }, }, }, Failures: []ResponseDescriptor{ { Description: "There was an error processing the upload and it must be restarted.", StatusCode: http.StatusBadRequest, ErrorCodes: []ErrorCode{ ErrorCodeDigestInvalid, ErrorCodeNameInvalid, ErrorCodeBlobUploadInvalid, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, }, unauthorizedResponse, { Description: "The upload is unknown to the registry. The upload must be restarted.", StatusCode: http.StatusNotFound, ErrorCodes: []ErrorCode{ ErrorCodeBlobUploadUnknown, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, }, }, }, }, }, { Method: "PATCH", Description: "Upload a chunk of data for the specified upload.", Requests: []RequestDescriptor{ { Description: "Upload a chunk of data to specified upload without completing the upload.", PathParameters: []ParameterDescriptor{ nameParameterDescriptor, uuidParameterDescriptor, }, Headers: []ParameterDescriptor{ hostHeader, authHeader, { Name: "Content-Range", Type: "header", Format: "-", Required: true, Description: "Range of bytes identifying the desired block of content represented by the body. Start must the end offset retrieved via status check plus one. Note that this is a non-standard use of the `Content-Range` header.", }, { Name: "Content-Length", Type: "integer", Format: "", Description: "Length of the chunk being uploaded, corresponding the length of the request body.", }, }, Body: BodyDescriptor{ ContentType: "application/octet-stream", Format: "", }, Successes: []ResponseDescriptor{ { Name: "Chunk Accepted", Description: "The chunk of data has been accepted and the current progress is available in the range header. The updated upload location is available in the `Location` header.", StatusCode: http.StatusNoContent, Headers: []ParameterDescriptor{ { Name: "Location", Type: "url", Format: "/v2//blobs/uploads/", Description: "The location of the upload. Clients should assume this changes after each request. Clients should use the contents verbatim to complete the upload, adding parameters where required.", }, { Name: "Range", Type: "header", Format: "0-", Description: "Range indicating the current progress of the upload.", }, contentLengthZeroHeader, dockerUploadUUIDHeader, }, }, }, Failures: []ResponseDescriptor{ { Description: "There was an error processing the upload and it must be restarted.", StatusCode: http.StatusBadRequest, ErrorCodes: []ErrorCode{ ErrorCodeDigestInvalid, ErrorCodeNameInvalid, ErrorCodeBlobUploadInvalid, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, }, unauthorizedResponsePush, { Description: "The upload is unknown to the registry. The upload must be restarted.", StatusCode: http.StatusNotFound, ErrorCodes: []ErrorCode{ ErrorCodeBlobUploadUnknown, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, }, { Description: "The `Content-Range` specification cannot be accepted, either because it does not overlap with the current progress or it is invalid.", StatusCode: http.StatusRequestedRangeNotSatisfiable, }, }, }, }, }, { Method: "PUT", Description: "Complete the upload specified by `uuid`, optionally appending the body as the final chunk.", Requests: []RequestDescriptor{ { // TODO(stevvooe): Break this down into three separate requests: // 1. Complete an upload where all data has already been sent. // 2. Complete an upload where the entire body is in the PUT. // 3. Complete an upload where the final, partial chunk is the body. Description: "Complete the upload, providing the _final_ chunk of data, if necessary. This method may take a body with all the data. If the `Content-Range` header is specified, it may include the final chunk. A request without a body will just complete the upload with previously uploaded content.", Headers: []ParameterDescriptor{ hostHeader, authHeader, { Name: "Content-Range", Type: "header", Format: "-", Description: "Range of bytes identifying the block of content represented by the body. Start must the end offset retrieved via status check plus one. Note that this is a non-standard use of the `Content-Range` header. May be omitted if no data is provided.", }, { Name: "Content-Length", Type: "integer", Format: "", Description: "Length of the chunk being uploaded, corresponding to the length of the request body. May be zero if no data is provided.", }, }, PathParameters: []ParameterDescriptor{ nameParameterDescriptor, uuidParameterDescriptor, }, QueryParameters: []ParameterDescriptor{ { Name: "digest", Type: "string", Format: "", Regexp: digest.DigestRegexp, Required: true, Description: `Digest of uploaded blob.`, }, }, Body: BodyDescriptor{ ContentType: "application/octet-stream", Format: "", }, Successes: []ResponseDescriptor{ { Name: "Upload Complete", Description: "The upload has been completed and accepted by the registry. The canonical location will be available in the `Location` header.", StatusCode: http.StatusNoContent, Headers: []ParameterDescriptor{ { Name: "Location", Type: "url", Format: "", }, { Name: "Content-Range", Type: "header", Format: "-", Description: "Range of bytes identifying the desired block of content represented by the body. Start must match the end of offset retrieved via status check. Note that this is a non-standard use of the `Content-Range` header.", }, { Name: "Content-Length", Type: "integer", Format: "", Description: "Length of the chunk being uploaded, corresponding the length of the request body.", }, digestHeader, }, }, }, Failures: []ResponseDescriptor{ { Description: "There was an error processing the upload and it must be restarted.", StatusCode: http.StatusBadRequest, ErrorCodes: []ErrorCode{ ErrorCodeDigestInvalid, ErrorCodeNameInvalid, ErrorCodeBlobUploadInvalid, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, }, unauthorizedResponsePush, { Description: "The upload is unknown to the registry. The upload must be restarted.", StatusCode: http.StatusNotFound, ErrorCodes: []ErrorCode{ ErrorCodeBlobUploadUnknown, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, }, { Description: "The `Content-Range` specification cannot be accepted, either because it does not overlap with the current progress or it is invalid. The contents of the `Range` header may be used to resolve the condition.", StatusCode: http.StatusRequestedRangeNotSatisfiable, Headers: []ParameterDescriptor{ { Name: "Location", Type: "url", Format: "/v2//blobs/uploads/", Description: "The location of the upload. Clients should assume this changes after each request. Clients should use the contents verbatim to complete the upload, adding parameters where required.", }, { Name: "Range", Type: "header", Format: "0-", Description: "Range indicating the current progress of the upload.", }, }, }, }, }, }, }, { Method: "DELETE", Description: "Cancel outstanding upload processes, releasing associated resources. If this is not called, the unfinished uploads will eventually timeout.", Requests: []RequestDescriptor{ { Description: "Cancel the upload specified by `uuid`.", PathParameters: []ParameterDescriptor{ nameParameterDescriptor, uuidParameterDescriptor, }, Headers: []ParameterDescriptor{ hostHeader, authHeader, contentLengthZeroHeader, }, Successes: []ResponseDescriptor{ { Name: "Upload Deleted", Description: "The upload has been successfully deleted.", StatusCode: http.StatusNoContent, Headers: []ParameterDescriptor{ contentLengthZeroHeader, }, }, }, Failures: []ResponseDescriptor{ { Description: "An error was encountered processing the delete. The client may ignore this error.", StatusCode: http.StatusBadRequest, ErrorCodes: []ErrorCode{ ErrorCodeNameInvalid, ErrorCodeBlobUploadInvalid, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, }, unauthorizedResponse, { Description: "The upload is unknown to the registry. The client may ignore this error and assume the upload has been deleted.", StatusCode: http.StatusNotFound, ErrorCodes: []ErrorCode{ ErrorCodeBlobUploadUnknown, }, Body: BodyDescriptor{ ContentType: "application/json; charset=utf-8", Format: errorsBody, }, }, }, }, }, }, }, }, } // ErrorDescriptors provides a list of HTTP API Error codes that may be // encountered when interacting with the registry API. var errorDescriptors = []ErrorDescriptor{ { Code: ErrorCodeUnknown, Value: "UNKNOWN", Message: "unknown error", Description: `Generic error returned when the error does not have an API classification.`, }, { Code: ErrorCodeUnsupported, Value: "UNSUPPORTED", Message: "The operation is unsupported.", Description: `The operation was unsupported due to a missing implementation or invalid set of parameters.`, }, { Code: ErrorCodeUnauthorized, Value: "UNAUTHORIZED", Message: "access to the requested resource is not authorized", Description: `The access controller denied access for the operation on a resource. Often this will be accompanied by a 401 Unauthorized response status.`, }, { Code: ErrorCodeDigestInvalid, Value: "DIGEST_INVALID", Message: "provided digest did not match uploaded content", Description: `When a blob is uploaded, the registry will check that the content matches the digest provided by the client. The error may include a detail structure with the key "digest", including the invalid digest string. This error may also be returned when a manifest includes an invalid layer digest.`, HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, }, { Code: ErrorCodeSizeInvalid, Value: "SIZE_INVALID", Message: "provided length did not match content length", Description: `When a layer is uploaded, the provided size will be checked against the uploaded content. If they do not match, this error will be returned.`, HTTPStatusCodes: []int{http.StatusBadRequest}, }, { Code: ErrorCodeNameInvalid, Value: "NAME_INVALID", Message: "invalid repository name", Description: `Invalid repository name encountered either during manifest validation or any API operation.`, HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, }, { Code: ErrorCodeTagInvalid, Value: "TAG_INVALID", Message: "manifest tag did not match URI", Description: `During a manifest upload, if the tag in the manifest does not match the uri tag, this error will be returned.`, HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, }, { Code: ErrorCodeNameUnknown, Value: "NAME_UNKNOWN", Message: "repository name not known to registry", Description: `This is returned if the name used during an operation is unknown to the registry.`, HTTPStatusCodes: []int{http.StatusNotFound}, }, { Code: ErrorCodeManifestUnknown, Value: "MANIFEST_UNKNOWN", Message: "manifest unknown", Description: `This error is returned when the manifest, identified by name and tag is unknown to the repository.`, HTTPStatusCodes: []int{http.StatusNotFound}, }, { Code: ErrorCodeManifestInvalid, Value: "MANIFEST_INVALID", Message: "manifest invalid", Description: `During upload, manifests undergo several checks ensuring validity. If those checks fail, this error may be returned, unless a more specific error is included. The detail will contain information the failed validation.`, HTTPStatusCodes: []int{http.StatusBadRequest}, }, { Code: ErrorCodeManifestUnverified, Value: "MANIFEST_UNVERIFIED", Message: "manifest failed signature verification", Description: `During manifest upload, if the manifest fails signature verification, this error will be returned.`, HTTPStatusCodes: []int{http.StatusBadRequest}, }, { Code: ErrorCodeBlobUnknown, Value: "BLOB_UNKNOWN", Message: "blob unknown to registry", Description: `This error may be returned when a blob is unknown to the registry in a specified repository. This can be returned with a standard get or if a manifest references an unknown layer during upload.`, HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, }, { Code: ErrorCodeBlobUploadUnknown, Value: "BLOB_UPLOAD_UNKNOWN", Message: "blob upload unknown to registry", Description: `If a blob upload has been cancelled or was never started, this error code may be returned.`, HTTPStatusCodes: []int{http.StatusNotFound}, }, { Code: ErrorCodeBlobUploadInvalid, Value: "BLOB_UPLOAD_INVALID", Message: "blob upload invalid", Description: `The blob upload encountered an error and can no longer proceed.`, HTTPStatusCodes: []int{http.StatusNotFound}, }, } var errorCodeToDescriptors map[ErrorCode]ErrorDescriptor var idToDescriptors map[string]ErrorDescriptor var routeDescriptorsMap map[string]RouteDescriptor func init() { errorCodeToDescriptors = make(map[ErrorCode]ErrorDescriptor, len(errorDescriptors)) idToDescriptors = make(map[string]ErrorDescriptor, len(errorDescriptors)) routeDescriptorsMap = make(map[string]RouteDescriptor, len(routeDescriptors)) for _, descriptor := range errorDescriptors { errorCodeToDescriptors[descriptor.Code] = descriptor idToDescriptors[descriptor.Value] = descriptor } for _, descriptor := range routeDescriptors { routeDescriptorsMap[descriptor.Name] = descriptor } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/api/v2/names.go0000644000175000017500000000757512502424227025177 0ustar tianontianonpackage v2 import ( "fmt" "regexp" "strings" ) // TODO(stevvooe): Move these definitions back to an exported package. While // they are used with v2 definitions, their relevance expands beyond. // "distribution/names" is a candidate package. const ( // RepositoryNameComponentMinLength is the minimum number of characters in a // single repository name slash-delimited component RepositoryNameComponentMinLength = 2 // RepositoryNameMinComponents is the minimum number of slash-delimited // components that a repository name must have RepositoryNameMinComponents = 1 // RepositoryNameTotalLengthMax is the maximum total number of characters in // a repository name RepositoryNameTotalLengthMax = 255 ) // RepositoryNameComponentRegexp restricts registry path component names to // start with at least one letter or number, with following parts able to // be separated by one period, dash or underscore. var RepositoryNameComponentRegexp = regexp.MustCompile(`[a-z0-9]+(?:[._-][a-z0-9]+)*`) // RepositoryNameComponentAnchoredRegexp is the version of // RepositoryNameComponentRegexp which must completely match the content var RepositoryNameComponentAnchoredRegexp = regexp.MustCompile(`^` + RepositoryNameComponentRegexp.String() + `$`) // RepositoryNameRegexp builds on RepositoryNameComponentRegexp to allow // multiple path components, separated by a forward slash. var RepositoryNameRegexp = regexp.MustCompile(`(?:` + RepositoryNameComponentRegexp.String() + `/)*` + RepositoryNameComponentRegexp.String()) // TagNameRegexp matches valid tag names. From docker/docker:graph/tags.go. var TagNameRegexp = regexp.MustCompile(`[\w][\w.-]{0,127}`) // TODO(stevvooe): Contribute these exports back to core, so they are shared. var ( // ErrRepositoryNameComponentShort is returned when a repository name // contains a component which is shorter than // RepositoryNameComponentMinLength ErrRepositoryNameComponentShort = fmt.Errorf("respository name component must be %v or more characters", RepositoryNameComponentMinLength) // ErrRepositoryNameMissingComponents is returned when a repository name // contains fewer than RepositoryNameMinComponents components ErrRepositoryNameMissingComponents = fmt.Errorf("repository name must have at least %v components", RepositoryNameMinComponents) // ErrRepositoryNameLong is returned when a repository name is longer than // RepositoryNameTotalLengthMax ErrRepositoryNameLong = fmt.Errorf("repository name must not be more than %v characters", RepositoryNameTotalLengthMax) // ErrRepositoryNameComponentInvalid is returned when a repository name does // not match RepositoryNameComponentRegexp ErrRepositoryNameComponentInvalid = fmt.Errorf("repository name component must match %q", RepositoryNameComponentRegexp.String()) ) // ValidateRespositoryName ensures the repository name is valid for use in the // registry. This function accepts a superset of what might be accepted by // docker core or docker hub. If the name does not pass validation, an error, // describing the conditions, is returned. // // Effectively, the name should comply with the following grammar: // // alpha-numeric := /[a-z0-9]+/ // separator := /[._-]/ // component := alpha-numeric [separator alpha-numeric]* // namespace := component ['/' component]* // // The result of the production, known as the "namespace", should be limited // to 255 characters. func ValidateRespositoryName(name string) error { if len(name) > RepositoryNameTotalLengthMax { return ErrRepositoryNameLong } components := strings.Split(name, "/") if len(components) < RepositoryNameMinComponents { return ErrRepositoryNameMissingComponents } for _, component := range components { if len(component) < RepositoryNameComponentMinLength { return ErrRepositoryNameComponentShort } if !RepositoryNameComponentAnchoredRegexp.MatchString(component) { return ErrRepositoryNameComponentInvalid } } return nil } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/api/v2/routes_test.go0000644000175000017500000002105012502424227026434 0ustar tianontianonpackage v2 import ( "encoding/json" "fmt" "math/rand" "net/http" "net/http/httptest" "reflect" "strings" "testing" "time" "github.com/gorilla/mux" ) type routeTestCase struct { RequestURI string ExpectedURI string Vars map[string]string RouteName string StatusCode int } // TestRouter registers a test handler with all the routes and ensures that // each route returns the expected path variables. Not method verification is // present. This not meant to be exhaustive but as check to ensure that the // expected variables are extracted. // // This may go away as the application structure comes together. func TestRouter(t *testing.T) { testCases := []routeTestCase{ { RouteName: RouteNameBase, RequestURI: "/v2/", Vars: map[string]string{}, }, { RouteName: RouteNameManifest, RequestURI: "/v2/foo/manifests/bar", Vars: map[string]string{ "name": "foo", "reference": "bar", }, }, { RouteName: RouteNameManifest, RequestURI: "/v2/foo/bar/manifests/tag", Vars: map[string]string{ "name": "foo/bar", "reference": "tag", }, }, { RouteName: RouteNameManifest, RequestURI: "/v2/foo/bar/manifests/sha256:abcdef01234567890", Vars: map[string]string{ "name": "foo/bar", "reference": "sha256:abcdef01234567890", }, }, { RouteName: RouteNameTags, RequestURI: "/v2/foo/bar/tags/list", Vars: map[string]string{ "name": "foo/bar", }, }, { RouteName: RouteNameBlob, RequestURI: "/v2/foo/bar/blobs/tarsum.dev+foo:abcdef0919234", Vars: map[string]string{ "name": "foo/bar", "digest": "tarsum.dev+foo:abcdef0919234", }, }, { RouteName: RouteNameBlob, RequestURI: "/v2/foo/bar/blobs/sha256:abcdef0919234", Vars: map[string]string{ "name": "foo/bar", "digest": "sha256:abcdef0919234", }, }, { RouteName: RouteNameBlobUpload, RequestURI: "/v2/foo/bar/blobs/uploads/", Vars: map[string]string{ "name": "foo/bar", }, }, { RouteName: RouteNameBlobUploadChunk, RequestURI: "/v2/foo/bar/blobs/uploads/uuid", Vars: map[string]string{ "name": "foo/bar", "uuid": "uuid", }, }, { RouteName: RouteNameBlobUploadChunk, RequestURI: "/v2/foo/bar/blobs/uploads/D95306FA-FAD3-4E36-8D41-CF1C93EF8286", Vars: map[string]string{ "name": "foo/bar", "uuid": "D95306FA-FAD3-4E36-8D41-CF1C93EF8286", }, }, { RouteName: RouteNameBlobUploadChunk, RequestURI: "/v2/foo/bar/blobs/uploads/RDk1MzA2RkEtRkFEMy00RTM2LThENDEtQ0YxQzkzRUY4Mjg2IA==", Vars: map[string]string{ "name": "foo/bar", "uuid": "RDk1MzA2RkEtRkFEMy00RTM2LThENDEtQ0YxQzkzRUY4Mjg2IA==", }, }, { // Check ambiguity: ensure we can distinguish between tags for // "foo/bar/image/image" and image for "foo/bar/image" with tag // "tags" RouteName: RouteNameManifest, RequestURI: "/v2/foo/bar/manifests/manifests/tags", Vars: map[string]string{ "name": "foo/bar/manifests", "reference": "tags", }, }, { // This case presents an ambiguity between foo/bar with tag="tags" // and list tags for "foo/bar/manifest" RouteName: RouteNameTags, RequestURI: "/v2/foo/bar/manifests/tags/list", Vars: map[string]string{ "name": "foo/bar/manifests", }, }, } checkTestRouter(t, testCases, "", true) checkTestRouter(t, testCases, "/prefix/", true) } func TestRouterWithPathTraversals(t *testing.T) { testCases := []routeTestCase{ { RouteName: RouteNameBlobUploadChunk, RequestURI: "/v2/foo/../../blob/uploads/D95306FA-FAD3-4E36-8D41-CF1C93EF8286", ExpectedURI: "/blob/uploads/D95306FA-FAD3-4E36-8D41-CF1C93EF8286", StatusCode: http.StatusNotFound, }, { // Testing for path traversal attack handling RouteName: RouteNameTags, RequestURI: "/v2/foo/../bar/baz/tags/list", ExpectedURI: "/v2/bar/baz/tags/list", Vars: map[string]string{ "name": "bar/baz", }, }, } checkTestRouter(t, testCases, "", false) } func TestRouterWithBadCharacters(t *testing.T) { if testing.Short() { testCases := []routeTestCase{ { RouteName: RouteNameBlobUploadChunk, RequestURI: "/v2/foo/blob/uploads/不95306FA-FAD3-4E36-8D41-CF1C93EF8286", StatusCode: http.StatusNotFound, }, { // Testing for path traversal attack handling RouteName: RouteNameTags, RequestURI: "/v2/foo/不bar/tags/list", StatusCode: http.StatusNotFound, }, } checkTestRouter(t, testCases, "", true) } else { // in the long version we're going to fuzz the router // with random UTF8 characters not in the 128 bit ASCII range. // These are not valid characters for the router and we expect // 404s on every test. rand.Seed(time.Now().UTC().UnixNano()) testCases := make([]routeTestCase, 1000) for idx := range testCases { testCases[idx] = routeTestCase{ RouteName: RouteNameTags, RequestURI: fmt.Sprintf("/v2/%v/%v/tags/list", randomString(10), randomString(10)), StatusCode: http.StatusNotFound, } } checkTestRouter(t, testCases, "", true) } } func checkTestRouter(t *testing.T, testCases []routeTestCase, prefix string, deeplyEqual bool) { router := RouterWithPrefix(prefix) testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { testCase := routeTestCase{ RequestURI: r.RequestURI, Vars: mux.Vars(r), RouteName: mux.CurrentRoute(r).GetName(), } enc := json.NewEncoder(w) if err := enc.Encode(testCase); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } }) // Startup test server server := httptest.NewServer(router) for _, testcase := range testCases { testcase.RequestURI = strings.TrimSuffix(prefix, "/") + testcase.RequestURI // Register the endpoint route := router.GetRoute(testcase.RouteName) if route == nil { t.Fatalf("route for name %q not found", testcase.RouteName) } route.Handler(testHandler) u := server.URL + testcase.RequestURI resp, err := http.Get(u) if err != nil { t.Fatalf("error issuing get request: %v", err) } if testcase.StatusCode == 0 { // Override default, zero-value testcase.StatusCode = http.StatusOK } if testcase.ExpectedURI == "" { // Override default, zero-value testcase.ExpectedURI = testcase.RequestURI } if resp.StatusCode != testcase.StatusCode { t.Fatalf("unexpected status for %s: %v %v", u, resp.Status, resp.StatusCode) } if testcase.StatusCode != http.StatusOK { // We don't care about json response. continue } dec := json.NewDecoder(resp.Body) var actualRouteInfo routeTestCase if err := dec.Decode(&actualRouteInfo); err != nil { t.Fatalf("error reading json response: %v", err) } // Needs to be set out of band actualRouteInfo.StatusCode = resp.StatusCode if actualRouteInfo.RequestURI != testcase.ExpectedURI { t.Fatalf("URI %v incorrectly parsed, expected %v", actualRouteInfo.RequestURI, testcase.ExpectedURI) } if actualRouteInfo.RouteName != testcase.RouteName { t.Fatalf("incorrect route %q matched, expected %q", actualRouteInfo.RouteName, testcase.RouteName) } // when testing deep equality, the actualRouteInfo has an empty ExpectedURI, we don't want // that to make the comparison fail. We're otherwise done with the testcase so empty the // testcase.ExpectedURI testcase.ExpectedURI = "" if deeplyEqual && !reflect.DeepEqual(actualRouteInfo, testcase) { t.Fatalf("actual does not equal expected: %#v != %#v", actualRouteInfo, testcase) } } } // -------------- START LICENSED CODE -------------- // The following code is derivative of https://github.com/google/gofuzz // gofuzz is licensed under the Apache License, Version 2.0, January 2004, // a copy of which can be found in the LICENSE file at the root of this // repository. // These functions allow us to generate strings containing only multibyte // characters that are invalid in our URLs. They are used above for fuzzing // to ensure we always get 404s on these invalid strings type charRange struct { first, last rune } // choose returns a random unicode character from the given range, using the // given randomness source. func (r *charRange) choose() rune { count := int64(r.last - r.first) return r.first + rune(rand.Int63n(count)) } var unicodeRanges = []charRange{ {'\u00a0', '\u02af'}, // Multi-byte encoded characters {'\u4e00', '\u9fff'}, // Common CJK (even longer encodings) } func randomString(length int) string { runes := make([]rune, length) for i := range runes { runes[i] = unicodeRanges[rand.Intn(len(unicodeRanges))].choose() } return string(runes) } // -------------- END LICENSED CODE -------------- distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/api/v2/names_test.go0000644000175000017500000000334612502424227026226 0ustar tianontianonpackage v2 import ( "strings" "testing" ) func TestRepositoryNameRegexp(t *testing.T) { for _, testcase := range []struct { input string err error }{ { input: "short", }, { input: "simple/name", }, { input: "library/ubuntu", }, { input: "docker/stevvooe/app", }, { input: "aa/aa/aa/aa/aa/aa/aa/aa/aa/bb/bb/bb/bb/bb/bb", }, { input: "aa/aa/bb/bb/bb", }, { input: "a/a/a/b/b", err: ErrRepositoryNameComponentShort, }, { input: "a/a/a/a/", err: ErrRepositoryNameComponentShort, }, { input: "foo.com/bar/baz", }, { input: "blog.foo.com/bar/baz", }, { input: "asdf", }, { input: "asdf$$^/aa", err: ErrRepositoryNameComponentInvalid, }, { input: "aa-a/aa", }, { input: "aa/aa", }, { input: "a-a/a-a", }, { input: "a", err: ErrRepositoryNameComponentShort, }, { input: "a-/a/a/a", err: ErrRepositoryNameComponentInvalid, }, { input: strings.Repeat("a", 255), }, { input: strings.Repeat("a", 256), err: ErrRepositoryNameLong, }, } { failf := func(format string, v ...interface{}) { t.Logf(testcase.input+": "+format, v...) t.Fail() } if err := ValidateRespositoryName(testcase.input); err != testcase.err { if testcase.err != nil { if err != nil { failf("unexpected error for invalid repository: got %v, expected %v", err, testcase.err) } else { failf("expected invalid repository: %v", testcase.err) } } else { if err != nil { // Wrong error returned. failf("unexpected error validating repository name: %v, expected %v", err, testcase.err) } else { failf("unexpected error validating repository name: %v", err) } } } } } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/registry/api/v2/urls.go0000644000175000017500000001274312502424227025052 0ustar tianontianonpackage v2 import ( "net/http" "net/url" "strings" "github.com/docker/distribution/digest" "github.com/gorilla/mux" ) // URLBuilder creates registry API urls from a single base endpoint. It can be // used to create urls for use in a registry client or server. // // All urls will be created from the given base, including the api version. // For example, if a root of "/foo/" is provided, urls generated will be fall // under "/foo/v2/...". Most application will only provide a schema, host and // port, such as "https://localhost:5000/". type URLBuilder struct { root *url.URL // url root (ie http://localhost/) router *mux.Router } // NewURLBuilder creates a URLBuilder with provided root url object. func NewURLBuilder(root *url.URL) *URLBuilder { return &URLBuilder{ root: root, router: Router(), } } // NewURLBuilderFromString workes identically to NewURLBuilder except it takes // a string argument for the root, returning an error if it is not a valid // url. func NewURLBuilderFromString(root string) (*URLBuilder, error) { u, err := url.Parse(root) if err != nil { return nil, err } return NewURLBuilder(u), nil } // NewURLBuilderFromRequest uses information from an *http.Request to // construct the root url. func NewURLBuilderFromRequest(r *http.Request) *URLBuilder { var scheme string forwardedProto := r.Header.Get("X-Forwarded-Proto") switch { case len(forwardedProto) > 0: scheme = forwardedProto case r.TLS != nil: scheme = "https" case len(r.URL.Scheme) > 0: scheme = r.URL.Scheme default: scheme = "http" } host := r.Host forwardedHost := r.Header.Get("X-Forwarded-Host") if len(forwardedHost) > 0 { host = forwardedHost } basePath := routeDescriptorsMap[RouteNameBase].Path requestPath := r.URL.Path index := strings.Index(requestPath, basePath) u := &url.URL{ Scheme: scheme, Host: host, } if index > 0 { // N.B. index+1 is important because we want to include the trailing / u.Path = requestPath[0 : index+1] } return NewURLBuilder(u) } // BuildBaseURL constructs a base url for the API, typically just "/v2/". func (ub *URLBuilder) BuildBaseURL() (string, error) { route := ub.cloneRoute(RouteNameBase) baseURL, err := route.URL() if err != nil { return "", err } return baseURL.String(), nil } // BuildTagsURL constructs a url to list the tags in the named repository. func (ub *URLBuilder) BuildTagsURL(name string) (string, error) { route := ub.cloneRoute(RouteNameTags) tagsURL, err := route.URL("name", name) if err != nil { return "", err } return tagsURL.String(), nil } // BuildManifestURL constructs a url for the manifest identified by name and // reference. The argument reference may be either a tag or digest. func (ub *URLBuilder) BuildManifestURL(name, reference string) (string, error) { route := ub.cloneRoute(RouteNameManifest) manifestURL, err := route.URL("name", name, "reference", reference) if err != nil { return "", err } return manifestURL.String(), nil } // BuildBlobURL constructs the url for the blob identified by name and dgst. func (ub *URLBuilder) BuildBlobURL(name string, dgst digest.Digest) (string, error) { route := ub.cloneRoute(RouteNameBlob) layerURL, err := route.URL("name", name, "digest", dgst.String()) if err != nil { return "", err } return layerURL.String(), nil } // BuildBlobUploadURL constructs a url to begin a blob upload in the // repository identified by name. func (ub *URLBuilder) BuildBlobUploadURL(name string, values ...url.Values) (string, error) { route := ub.cloneRoute(RouteNameBlobUpload) uploadURL, err := route.URL("name", name) if err != nil { return "", err } return appendValuesURL(uploadURL, values...).String(), nil } // BuildBlobUploadChunkURL constructs a url for the upload identified by uuid, // including any url values. This should generally not be used by clients, as // this url is provided by server implementations during the blob upload // process. func (ub *URLBuilder) BuildBlobUploadChunkURL(name, uuid string, values ...url.Values) (string, error) { route := ub.cloneRoute(RouteNameBlobUploadChunk) uploadURL, err := route.URL("name", name, "uuid", uuid) if err != nil { return "", err } return appendValuesURL(uploadURL, values...).String(), nil } // clondedRoute returns a clone of the named route from the router. Routes // must be cloned to avoid modifying them during url generation. func (ub *URLBuilder) cloneRoute(name string) clonedRoute { route := new(mux.Route) root := new(url.URL) *route = *ub.router.GetRoute(name) // clone the route *root = *ub.root return clonedRoute{Route: route, root: root} } type clonedRoute struct { *mux.Route root *url.URL } func (cr clonedRoute) URL(pairs ...string) (*url.URL, error) { routeURL, err := cr.Route.URL(pairs...) if err != nil { return nil, err } if routeURL.Scheme == "" && routeURL.User == nil && routeURL.Host == "" { routeURL.Path = routeURL.Path[1:] } return cr.root.ResolveReference(routeURL), nil } // appendValuesURL appends the parameters to the url. func appendValuesURL(u *url.URL, values ...url.Values) *url.URL { merged := u.Query() for _, v := range values { for k, vv := range v { merged[k] = append(merged[k], vv...) } } u.RawQuery = merged.Encode() return u } // appendValues appends the parameters to the url. Panics if the string is not // a url. func appendValues(u string, values ...url.Values) string { up, err := url.Parse(u) if err != nil { panic(err) // should never happen } return appendValuesURL(up, values...).String() } distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/Makefile0000644000175000017500000000344312502424227022223 0ustar tianontianon# Set an output prefix, which is the local directory if not specified PREFIX?=$(shell pwd) # Used to populate version variable in main package. GO_LDFLAGS=-ldflags "-X `go list ./version`.Version `git describe --match 'v[0-9]*' --dirty='.m' --always`" .PHONY: clean all fmt vet lint build test binaries .DEFAULT: default all: AUTHORS clean fmt vet fmt lint build test binaries AUTHORS: .mailmap .git/HEAD git log --format='%aN <%aE>' | sort -fu > $@ # This only needs to be generated by hand when cutting full releases. version/version.go: ./version/version.sh > $@ ${PREFIX}/bin/registry: version/version.go $(shell find . -type f -name '*.go') @echo "+ $@" @go build -o $@ ${GO_LDFLAGS} ./cmd/registry ${PREFIX}/bin/registry-api-descriptor-template: version/version.go $(shell find . -type f -name '*.go') @echo "+ $@" @go build -o $@ ${GO_LDFLAGS} ./cmd/registry-api-descriptor-template ${PREFIX}/bin/dist: version/version.go $(shell find . -type f -name '*.go') @echo "+ $@" @go build -o $@ ${GO_LDFLAGS} ./cmd/dist doc/spec/api.md: doc/spec/api.md.tmpl ${PREFIX}/bin/registry-api-descriptor-template ./bin/registry-api-descriptor-template $< > $@ vet: @echo "+ $@" @go vet ./... fmt: @echo "+ $@" @test -z "$$(gofmt -s -l . | grep -v Godeps/_workspace/src/ | tee /dev/stderr)" || \ echo "+ please format Go code with 'gofmt -s'" lint: @echo "+ $@" @test -z "$$(golint ./... | grep -v Godeps/_workspace/src/ | tee /dev/stderr)" build: @echo "+ $@" @go build -v ${GO_LDFLAGS} ./... test: @echo "+ $@" @go test -test.short ./... test-full: @echo "+ $@" @go test ./... binaries: ${PREFIX}/bin/registry ${PREFIX}/bin/registry-api-descriptor-template ${PREFIX}/bin/dist @echo "+ $@" clean: @echo "+ $@" @rm -rf "${PREFIX}/bin/registry" "${PREFIX}/bin/registry-api-descriptor-template" distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/doc/0000755000175000017500000000000012502424227021324 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/doc/overview.md0000644000175000017500000000024612502424227023516 0ustar tianontianon# Overview **TODO(stevvooe):** Table of contents. **TODO(stevvooe):** Include a full overview of each component and dispatch the user to the correct documentation. distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/doc/storagedrivers.md0000644000175000017500000001200312502424227024705 0ustar tianontianonDocker-Registry Storage Driver ============================== This document describes the registry storage driver model, implementation, and explains how to contribute new storage drivers. Provided Drivers ================ This storage driver package comes bundled with several drivers: - [inmemory](storagedriver/inmemory.md): A temporary storage driver using a local inmemory map. This exists solely for reference and testing. - [filesystem](storagedriver/filesystem.md): A local storage driver configured to use a directory tree in the local filesystem. - [s3](storagedriver/s3.md): A driver storing objects in an Amazon Simple Storage Solution (S3) bucket. - [azure](storagedriver/azure.md): A driver storing objects in [Microsoft Azure Blob Storage](http://azure.microsoft.com/en-us/services/storage/). Storage Driver API ================== The storage driver API is designed to model a filesystem-like key/value storage in a manner abstract enough to support a range of drivers from the local filesystem to Amazon S3 or other distributed object storage systems. Storage drivers are required to implement the `storagedriver.StorageDriver` interface provided in `storagedriver.go`, which includes methods for reading, writing, and deleting content, as well as listing child objects of a specified prefix key. Storage drivers are intended (but not required) to be written in go, providing compile-time validation of the `storagedriver.StorageDriver` interface, although an IPC driver wrapper means that it is not required for drivers to be included in the compiled registry. The `storagedriver/ipc` package provides a client/server protocol for running storage drivers provided in external executables as a managed child server process. Driver Selection and Configuration ================================== The preferred method of selecting a storage driver is using the `StorageDriverFactory` interface in the `storagedriver/factory` package. These factories provide a common interface for constructing storage drivers with a parameters map. The factory model is based off of the [Register](http://golang.org/pkg/database/sql/#Register) and [Open](http://golang.org/pkg/database/sql/#Open) methods in the builtin [database/sql](http://golang.org/pkg/database/sql) package. Storage driver factories may be registered by name using the `factory.Register` method, and then later invoked by calling `factory.Create` with a driver name and parameters map. If no driver is registered with the given name, this factory will attempt to find an executable storage driver with the executable name "registry-storage-\" and return an IPC storage driver wrapper managing the driver subprocess. If no such storage driver can be found, `factory.Create` will return an `InvalidStorageDriverError`. Driver Contribution =================== ## Writing new storage drivers To create a valid storage driver, one must implement the `storagedriver.StorageDriver` interface and make sure to expose this driver via the factory system and as a distributable IPC server executable. ### In-process drivers Storage drivers should call `factory.Register` with their driver name in an `init` method, allowing callers of `factory.New` to construct instances of this driver without requiring modification of imports throughout the codebase. ### Out-of-process drivers As many users will run the registry as a pre-constructed docker container, storage drivers should also be distributable as IPC server executables. Drivers written in go should model the main method provided in `storagedriver/filesystem/registry-storage-filesystem/filesystem.go`. Parameters to IPC drivers will be provided as a JSON-serialized map in the first argument to the process. These parameters should be validated and then a blocking call to `ipc.StorageDriverServer` should be made with a new storage driver. Out-of-process drivers must also implement the `ipc.IPCStorageDriver` interface, which exposes a `Version` check for the storage driver. This is used to validate storage driver api compatibility at driver load-time. ## Testing Storage driver test suites are provided in `storagedriver/testsuites/testsuites.go` and may be used for any storage driver written in go. Two methods are provided for registering test suites, `RegisterInProcessSuite` and `RegisterIPCSuite`, which run the same set of tests for the driver imported or managed over IPC respectively. ## Drivers written in other languages Although storage drivers are strongly recommended to be written in go for consistency, compile-time validation, and support, the IPC framework allows for a level of language-agnosticism. Non-go drivers must implement the storage driver protocol by mimicing StorageDriverServer in `storagedriver/ipc/server.go`. As the IPC framework is a layer on top of [docker/libchan](https://github.com/docker/libchan), this currently limits language support to Java via [ndeloof/chan](https://github.com/ndeloof/jchan) and Javascript via [GraftJS/jschan](https://github.com/GraftJS/jschan), although contributions to the libchan project are welcome. distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/doc/spec/0000755000175000017500000000000012502424227022256 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/doc/spec/auth/0000755000175000017500000000000012502424227023217 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/doc/spec/auth/token.md0000644000175000017500000004216312502424227024667 0ustar tianontianon# Docker Registry v2 authentication via central service Today a Docker Registry can run in standalone mode in which there are no authorization checks. While adding your own HTTP authorization requirements in a proxy placed between the client and the registry can give you greater access control, we'd like a native authorization mechanism that's public key based with access control lists managed separately with the ability to have fine granularity in access control on a by-key, by-user, by-namespace, and by-repository basis. In v1 this can be configured by specifying an `index_endpoint` in the registry's config. Clients present tokens generated by the index and tokens are validated on-line by the registry with every request. This results in a complex authentication and authorization loop that occurs with every registry operation. Some people are very familiar with this image: ![index auth](https://docs.docker.com/static_files/docker_pull_chart.png) The above image outlines the 6-step process in accessing the Official Docker Registry. 1. Contact the Docker Hub to know where I should download “samalba/busybox” 2. Docker Hub replies: a. samalba/busybox is on Registry A b. here are the checksums for samalba/busybox (for all layers) c. token 3. Contact Registry A to receive the layers for samalba/busybox (all of them to the base image). Registry A is authoritative for “samalba/busybox” but keeps a copy of all inherited layers and serve them all from the same location. 4. Registry contacts Docker Hub to verify if token/user is allowed to download images. 5. Docker Hub returns true/false lettings registry know if it should proceed or error out. 6. Get the payload for all layers. The goal of this document is to outline a way to eliminate steps 4 and 5 from the above process by using cryptographically signed tokens and no longer require the client to authenticate each request with a username and password stored locally in plain text. The new registry workflow is more like this: ![v2 registry auth](https://docs.google.com/drawings/d/1EHZU9uBLmcH0kytDClBv6jv6WR4xZjE8RKEUw1mARJA/pub?w=480&h=360) 1. Attempt to begin a push/pull operation with the registry. 2. If the registry requires authorization it will return a `401 Unauthorized` HTTP response with information on how to authenticate. 3. The registry client makes a request to the authorization service for a signed JSON Web Token. 4. The authorization service returns a token. 5. The client retries the original request with the token embedded in the request header. 6. The Registry authorizes the client and begins the push/pull session as usual. ## Requirements - Registry Clients capable of generating key pairs which can be used to authenticate to an authorization server. - An authorization server capable of managing user accounts, their public keys, and access controls to their resources hosted by any given service (such as repositories in a Docker Registry). - A Docker Registry capable of trusting the authorization server to sign tokens which clients can use for authorization and the ability to verify these tokens for single use or for use during a sufficiently short period of time. ## Authorization Server Endpoint Descriptions This document borrows heavily from the [JSON Web Token Draft Spec](https://tools.ietf.org/html/draft-ietf-oauth-json-web-token-32) The described server is meant to serve as a user account and key manager and a centralized access control list for resources hosted by other services which wish to authenticate and manage authorizations using this services accounts and their public keys. Such a service could be used by the official docker registry to authenticate clients and verify their authorization to docker image repositories. Docker will need to be updated to interact with an authorization server to get an authorization token. ## How to authenticate Today, registry clients first contact the index to initiate a push or pull. For v2, clients should contact the registry first. If the registry server requires authentication it will return a `401 Unauthorized` response with a `WWW-Authenticate` header detailing how to authenticate to this registry. For example, say I (username `jlhawn`) am attempting to push an image to the repository `samalba/my-app`. For the registry to authorize this, I either need `push` access to the `samalba/my-app` repository or `push` access to the whole `samalba` namespace in general. The registry will first return this response: ``` HTTP/1.1 401 Unauthorized WWW-Authenticate: Bearer realm="https://auth.docker.com/v2/token/",service="registry.docker.com",scope="repository:samalba/my-app:push" ``` This format is documented in [Section 3 of RFC 6750: The OAuth 2.0 Authorization Framework: Bearer Token Usage](https://tools.ietf.org/html/rfc6750#section-3) The client will then know to make a `GET` request to the URL `https://auth.docker.com/v2/token/` using the `service` and `scope` values from the `WWW-Authenticate` header. ## Requesting a Token #### Query Parameters
service
The name of the service which hosts the resource.
scope
The resource in question, formatted as one of the space-delimited entries from the scope parameters from the WWW-Authenticate header shown above. This query parameter should be specified multiple times if there is more than one scope entry from the WWW-Authenticate header. The above example would be specified as: scope=repository:samalba/my-app:push.
account
The name of the account which the client is acting as. Optional if it can be inferred from client authentication.
#### Description Requests an authorization token for access to a specific resource hosted by a specific service provider. Requires the client to authenticate either using a TLS client certificate or using basic authentication (or any other kind of digest/challenge/response authentication scheme if the client doesn't support TLS client certs). If the key in the client certificate is linked to an account then the token is issued for that account key. If the key in the certificate is linked to multiple accounts then the client must specify the `account` query parameter. The returned token is in JWT (JSON Web Token) format, signed using the authorization server's private key. #### Example For this example, the client makes an HTTP request to the following endpoint over TLS using a client certificate with the server being configured to allow a non-verified issuer during the handshake (i.e., a self-signed client cert is okay). ``` GET /v2/token/?service=registry.docker.com&scope=repository:samalba/my-app:push&account=jlhawn HTTP/1.1 Host: auth.docker.com ``` The server first inspects the client certificate to extract the subject key and lookup which account it is associated with. The client is now authenticated using that account. The server next searches its access control list for the account's access to the repository `samalba/my-app` hosted by the service `registry.docker.com`. The server will now construct a JSON Web Token to sign and return. A JSON Web Token has 3 main parts: 1. Headers The header of a JSON Web Token is a standard JOSE header. The "typ" field will be "JWT" and it will also contain the "alg" which identifies the signing algorithm used to produce the signature. It will also usually have a "kid" field, the ID of the key which was used to sign the token. Here is an example JOSE Header for a JSON Web Token (formatted with whitespace for readability): ``` { "typ": "JWT", "alg": "ES256", "kid": "PYYO:TEWU:V7JH:26JV:AQTZ:LJC3:SXVJ:XGHA:34F2:2LAQ:ZRMK:Z7Q6" } ``` It specifies that this object is going to be a JSON Web token signed using the key with the given ID using the Elliptic Curve signature algorithm using a SHA256 hash. 2. Claim Set The Claim Set is a JSON struct containing these standard registered claim name fields:
iss (Issuer)
The issuer of the token, typically the fqdn of the authorization server.
sub (Subject)
The subject of the token; the id of the client which requested it.
aud (Audience)
The intended audience of the token; the id of the service which will verify the token to authorize the client/subject.
exp (Expiration)
The token should only be considered valid up to this specified date and time.
nbf (Not Before)
The token should not be considered valid before this specified date and time.
iat (Issued At)
Specifies the date and time which the Authorization server generated this token.
jti (JWT ID)
A unique identifier for this token. Can be used by the intended audience to prevent replays of the token.
The Claim Set will also contain a private claim name unique to this authorization server specification:
access
An array of access entry objects with the following fields:
type
The type of resource hosted by the service.
name
The name of the recource of the given type hosted by the service.
actions
An array of strings which give the actions authorized on this resource.
Here is an example of such a JWT Claim Set (formatted with whitespace for readability): ``` { "iss": "auth.docker.com", "sub": "jlhawn", "aud": "registry.docker.com", "exp": 1415387315, "nbf": 1415387015, "iat": 1415387015, "jti": "tYJCO1c6cnyy7kAn0c7rKPgbV1H1bFws", "access": [ { "type": "repository", "name": "samalba/my-app", "actions": [ "push" ] } ] } ``` 3. Signature The authorization server will produce a JOSE header and Claim Set with no extraneous whitespace, i.e., the JOSE Header from above would be ``` {"typ":"JWT","alg":"ES256","kid":"PYYO:TEWU:V7JH:26JV:AQTZ:LJC3:SXVJ:XGHA:34F2:2LAQ:ZRMK:Z7Q6"} ``` and the Claim Set from above would be ``` {"iss":"auth.docker.com","sub":"jlhawn","aud":"registry.docker.com","exp":1415387315,"nbf":1415387015,"iat":1415387015,"jti":"tYJCO1c6cnyy7kAn0c7rKPgbV1H1bFws","access":[{"type":"repository","name":"samalba/my-app","actions":["push"]}]} ``` The utf-8 representation of this JOSE header and Claim Set are then url-safe base64 encoded (sans trailing '=' buffer), producing: ``` eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IlBZWU86VEVXVTpWN0pIOjI2SlY6QVFUWjpMSkMzOlNYVko6WEdIQTozNEYyOjJMQVE6WlJNSzpaN1E2In0 ``` for the JOSE Header and ``` eyJpc3MiOiJhdXRoLmRvY2tlci5jb20iLCJzdWIiOiJqbGhhd24iLCJhdWQiOiJyZWdpc3RyeS5kb2NrZXIuY29tIiwiZXhwIjoxNDE1Mzg3MzE1LCJuYmYiOjE0MTUzODcwMTUsImlhdCI6MTQxNTM4NzAxNSwianRpIjoidFlKQ08xYzZjbnl5N2tBbjBjN3JLUGdiVjFIMWJGd3MiLCJhY2Nlc3MiOlt7InR5cGUiOiJyZXBvc2l0b3J5IiwibmFtZSI6InNhbWFsYmEvbXktYXBwIiwiYWN0aW9ucyI6WyJwdXNoIl19XX0 ``` for the Claim Set. These two are concatenated using a '.' character, yielding the string: ``` eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IlBZWU86VEVXVTpWN0pIOjI2SlY6QVFUWjpMSkMzOlNYVko6WEdIQTozNEYyOjJMQVE6WlJNSzpaN1E2In0.eyJpc3MiOiJhdXRoLmRvY2tlci5jb20iLCJzdWIiOiJqbGhhd24iLCJhdWQiOiJyZWdpc3RyeS5kb2NrZXIuY29tIiwiZXhwIjoxNDE1Mzg3MzE1LCJuYmYiOjE0MTUzODcwMTUsImlhdCI6MTQxNTM4NzAxNSwianRpIjoidFlKQ08xYzZjbnl5N2tBbjBjN3JLUGdiVjFIMWJGd3MiLCJhY2Nlc3MiOlt7InR5cGUiOiJyZXBvc2l0b3J5IiwibmFtZSI6InNhbWFsYmEvbXktYXBwIiwiYWN0aW9ucyI6WyJwdXNoIl19XX0 ``` This is then used as the payload to a the `ES256` signature algorithm specified in the JOSE header and specified fully in [Section 3.4 of the JSON Web Algorithms (JWA) draft specification](https://tools.ietf.org/html/draft-ietf-jose-json-web-algorithms-38#section-3.4) This example signature will use the following ECDSA key for the server: ``` { "kty": "EC", "crv": "P-256", "kid": "PYYO:TEWU:V7JH:26JV:AQTZ:LJC3:SXVJ:XGHA:34F2:2LAQ:ZRMK:Z7Q6", "d": "R7OnbfMaD5J2jl7GeE8ESo7CnHSBm_1N2k9IXYFrKJA", "x": "m7zUpx3b-zmVE5cymSs64POG9QcyEpJaYCD82-549_Q", "y": "dU3biz8sZ_8GPB-odm8Wxz3lNDr1xcAQQPQaOcr1fmc" } ``` A resulting signature of the above payload using this key is: ``` QhflHPfbd6eVF4lM9bwYpFZIV0PfikbyXuLx959ykRTBpe3CYnzs6YBK8FToVb5R47920PVLrh8zuLzdCr9t3w ``` Concatenating all of these together with a `.` character gives the resulting JWT: ``` eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IlBZWU86VEVXVTpWN0pIOjI2SlY6QVFUWjpMSkMzOlNYVko6WEdIQTozNEYyOjJMQVE6WlJNSzpaN1E2In0.eyJpc3MiOiJhdXRoLmRvY2tlci5jb20iLCJzdWIiOiJqbGhhd24iLCJhdWQiOiJyZWdpc3RyeS5kb2NrZXIuY29tIiwiZXhwIjoxNDE1Mzg3MzE1LCJuYmYiOjE0MTUzODcwMTUsImlhdCI6MTQxNTM4NzAxNSwianRpIjoidFlKQ08xYzZjbnl5N2tBbjBjN3JLUGdiVjFIMWJGd3MiLCJhY2Nlc3MiOlt7InR5cGUiOiJyZXBvc2l0b3J5IiwibmFtZSI6InNhbWFsYmEvbXktYXBwIiwiYWN0aW9ucyI6WyJwdXNoIl19XX0.QhflHPfbd6eVF4lM9bwYpFZIV0PfikbyXuLx959ykRTBpe3CYnzs6YBK8FToVb5R47920PVLrh8zuLzdCr9t3w ``` This can now be placed in an HTTP response and returned to the client to use to authenticate to the audience service: ``` HTTP/1.1 200 OK Content-Type: application/json {"token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IlBZWU86VEVXVTpWN0pIOjI2SlY6QVFUWjpMSkMzOlNYVko6WEdIQTozNEYyOjJMQVE6WlJNSzpaN1E2In0.eyJpc3MiOiJhdXRoLmRvY2tlci5jb20iLCJzdWIiOiJqbGhhd24iLCJhdWQiOiJyZWdpc3RyeS5kb2NrZXIuY29tIiwiZXhwIjoxNDE1Mzg3MzE1LCJuYmYiOjE0MTUzODcwMTUsImlhdCI6MTQxNTM4NzAxNSwianRpIjoidFlKQ08xYzZjbnl5N2tBbjBjN3JLUGdiVjFIMWJGd3MiLCJhY2Nlc3MiOlt7InR5cGUiOiJyZXBvc2l0b3J5IiwibmFtZSI6InNhbWFsYmEvbXktYXBwIiwiYWN0aW9ucyI6WyJwdXNoIl19XX0.QhflHPfbd6eVF4lM9bwYpFZIV0PfikbyXuLx959ykRTBpe3CYnzs6YBK8FToVb5R47920PVLrh8zuLzdCr9t3w"} ``` ## Using the signed token Once the client has a token, it will try the registry request again with the token placed in the HTTP `Authorization` header like so: ``` Authorization: Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IkJWM0Q6MkFWWjpVQjVaOktJQVA6SU5QTDo1RU42Ok40SjQ6Nk1XTzpEUktFOkJWUUs6M0ZKTDpQT1RMIn0.eyJpc3MiOiJhdXRoLmRvY2tlci5jb20iLCJzdWIiOiJCQ0NZOk9VNlo6UUVKNTpXTjJDOjJBVkM6WTdZRDpBM0xZOjQ1VVc6NE9HRDpLQUxMOkNOSjU6NUlVTCIsImF1ZCI6InJlZ2lzdHJ5LmRvY2tlci5jb20iLCJleHAiOjE0MTUzODczMTUsIm5iZiI6MTQxNTM4NzAxNSwiaWF0IjoxNDE1Mzg3MDE1LCJqdGkiOiJ0WUpDTzFjNmNueXk3a0FuMGM3cktQZ2JWMUgxYkZ3cyIsInNjb3BlIjoiamxoYXduOnJlcG9zaXRvcnk6c2FtYWxiYS9teS1hcHA6cHVzaCxwdWxsIGpsaGF3bjpuYW1lc3BhY2U6c2FtYWxiYTpwdWxsIn0.Y3zZSwaZPqy4y9oRBVRImZyv3m_S9XDHF1tWwN7mL52C_IiA73SJkWVNsvNqpJIn5h7A2F8biv_S2ppQ1lgkbw ``` This is also described in [Section 2.1 of RFC 6750: The OAuth 2.0 Authorization Framework: Bearer Token Usage](https://tools.ietf.org/html/rfc6750#section-2.1) ## Verifying the token The registry must now verify the token presented by the user by inspecting the claim set within. The registry will: - Ensure that the issuer (`iss` claim) is an authority it trusts. - Ensure that the registry identifies as the audience (`aud` claim). - Check that the current time is between the `nbf` and `exp` claim times. - If enforcing single-use tokens, check that the JWT ID (`jti` claim) value has not been seen before. - To enforce this, the registry may keep a record of `jti`s it has seen for up to the `exp` time of the token to prevent token replays. - Check the `access` claim value and use the identified resources and the list of actions authorized to determine whether the token grants the required level of access for the operation the client is attempting to perform. - Verify that the signature of the token is valid. At no point in this process should the registry need to call back to the authorization server. If anything, it would only need to update a list of trusted public keys for verifying token signatures or use a separate API (still to be spec'd) to add/update resource records on the authorization server. distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/doc/spec/json.md0000644000175000017500000000426412502424227023557 0ustar tianontianon# Docker Distribution JSON Canonicalization ## Introduction To provide consistent content hashing of JSON objects throughout Docker Distribution APIs, a canonical JSON format has been defined. Adopting such a canonicalization also aids in caching JSON responses. ## Rules Compliant JSON should conform to the following rules: 1. All generated JSON should comply with [RFC 7159](http://www.ietf.org/rfc/rfc7159.txt). 2. Resulting "JSON text" shall always be encoded in UTF-8. 3. Unless a canonical key order is defined for a particular schema, object keys shall always appear in lexically sorted order. 4. All whitespace between tokens should be removed. 5. No "trailing commas" are allowed in object or array definitions. ## Examples The following is a simple example of a canonicalized JSON string: ```json {"asdf":1,"qwer":[],"zxcv":[{},true,1000000000,"tyui"]} ``` ## Reference ### Other Canonicalizations The OLPC project specifies [Canonical JSON](http://wiki.laptop.org/go/Canonical_JSON). While this is used in [TUF](http://theupdateframework.com/), which may be used with other distribution-related protocols, this alternative format has been proposed in case the original source changes. Specifications complying with either this specification or an alternative should explicitly call out the canonicalization format. Except for key ordering, this specification is mostly compatible. ### Go In Go, the [`encoding/json`](http://golang.org/pkg/encoding/json/) library will emit canonical JSON by default. Simply using `json.Marshal` will suffice in most cases: ```go incoming := map[string]interface{}{ "asdf": 1, "qwer": []interface{}{}, "zxcv": []interface{}{ map[string]interface{}{}, true, int(1e9), "tyui", }, } canonical, err := json.Marshal(incoming) if err != nil { // ... handle error } ``` To apply canonical JSON format spacing to an existing serialized JSON buffer, one can use [`json.Indent`](http://golang.org/src/encoding/json/indent.go?s=1918:1989#L65) with the following arguments: ```go incoming := getBytes() var canonical bytes.Buffer if err := json.Indent(&canonical, incoming, "", ""); err != nil { // ... handle error } ``` distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/doc/spec/api.md0000644000175000017500000023476312502424227023370 0ustar tianontianon# Docker Registry HTTP API V2 ## Introduction The _Docker Registry HTTP API_ is the protocol to facilitate distribution of images to the docker engine. It interacts with instances of the docker registry, which is a service to manage information about docker images and enable their distribution. The specification covers the operation of version 2 of this API, known as _Docker Registry HTTP API V2_. While the V1 registry protocol is usable, there are several problems with the architecture that have led to this new version. The main driver of this specification these changes to the docker the image format, covered in docker/docker#8093. The new, self-contained image manifest simplifies image definition and improves security. This specification will build on that work, leveraging new properties of the manifest format to improve performance, reduce bandwidth usage and decrease the likelihood of backend corruption. For relevant details and history leading up to this specification, please see the following issues: - [docker/docker#8093](https://github.com/docker/docker/issues/8903) - [docker/docker#9015](https://github.com/docker/docker/issues/9015) - [docker/docker-registry#612](https://github.com/docker/docker-registry/issues/612) ### Scope This specification covers the URL layout and protocols of the interaction between docker registry and docker core. This will affect the docker core registry API and the rewrite of docker-registry. Docker registry implementations may implement other API endpoints, but they are not covered by this specification. This includes the following features: - Namespace-oriented URI Layout - PUSH/PULL registry server for V2 image manifest format - Resumable layer PUSH support - V2 Client library implementation While authentication and authorization support will influence this specification, details of the protocol will be left to a future specification. Relevant header definitions and error codes are present to provide an indication of what a client may encounter. #### Future There are features that have been discussed during the process of cutting this specification. The following is an incomplete list: - Immutable image references - Multiple architecture support - Migration from v2compatibility representation These may represent features that are either out of the scope of this specification, the purview of another specification or have been deferred to a future version. ### Use Cases For the most part, the use cases of the former registry API apply to the new version. Differentiating use cases are covered below. #### Image Verification A docker engine instance would like to run verified image named "library/ubuntu", with the tag "latest". The engine contacts the registry, requesting the manifest for "library/ubuntu:latest". An untrusted registry returns a manifest. Before proceeding to download the individual layers, the engine verifies the manifest's signature, ensuring that the content was produced from a trusted source and no tampering has occured. After each layer is downloaded, the engine verifies the digest of the layer, ensuring that the content matches that specified by the manifest. #### Resumable Push Company X's build servers lose connectivity to docker registry before completing an image layer transfer. After connectivity returns, the build server attempts to re-upload the image. The registry notifies the build server that the upload has already been partially attempted. The build server responds by only sending the remaining data to complete the image file. #### Resumable Pull Company X is having more connectivity problems but this time in their deployment datacenter. When downloading an image, the connection is interrupted before completion. The client keeps the partial data and uses http `Range` requests to avoid downloading repeated data. #### Layer Upload De-duplication Company Y's build system creates two identical docker layers from build processes A and B. Build process A completes uploading the layer before B. When process B attempts to upload the layer, the registry indicates that its not necessary because the layer is already known. If process A and B upload the same layer at the same time, both operations will proceed and the first to complete will be stored in the registry (Note: we may modify this to prevent dogpile with some locking mechanism). ### Changes The V2 specification has been written to work as a living document, specifying only what is certain and leaving what is not specified open or to future changes. Only non-conflicting additions should be made to the API and accepted changes should avoid preventing future changes from happening. This section should be updated when changes are made to the specification, indicating what is different. Optionally, we may start marking parts of the specification to correspond with the versions enumerated here.
2.0.1
  • Added support for immutable manifest references in manifest endpoints.
  • Deleting a manifest by tag has been deprecated.
  • Specified `Docker-Content-Digest` header for appropriate entities.
  • Added error code for unsupported operations.
2.0
This is the baseline specification.
## Overview This section covers client flows and details of the API endpoints. The URI layout of the new API is structured to support a rich authentication and authorization model by leveraging namespaces. All endpoints will be prefixed by the API version and the repository name: /v2// For example, an API endpoint that will work with the `library/ubuntu` repository, the URI prefix will be: /v2/library/ubuntu/ This scheme provides rich access control over various operations and methods using the URI prefix and http methods that can be controlled in variety of ways. Classically, repository names have always been two path components where each path component is less than 30 characters. The V2 registry API does not enforce this. The rules for a repository name are as follows: 1. A repository name is broken up into _path components_. A component of a repository name must be at least two lowercase, alpha-numeric characters, optionally separated by periods, dashes or underscores. More strictly, it must match the regular expression `[a-z0-9]+(?:[._-][a-z0-9]+)*` and the matched result must be 2 or more characters in length. 2. The name of a repository must have at least two path components, separated by a forward slash. 3. The total length of a repository name, including slashes, must be less the 256 characters. These name requirements _only_ apply to the registry API and should accept a superset of what is supported by other docker ecosystem components. All endpoints should support aggressive http caching, compression and range headers, where appropriate. The new API attempts to leverage HTTP semantics where possible but may break from standards to implement targeted features. For detail on individual endpoints, please see the [_Detail_](#detail) section. ### Errors Actionable failure conditions, covered in detail in their relevant sections, are reported as part of 4xx responses, in a json response body. One or more errors will be returned in the following format: { "errors:" [{ "code": , "message": , "detail": }, ... ] } The `code` field will be a unique identifier, all caps with underscores by convention. The `message` field will be a human readable string. The optional `detail` field may contain arbitrary json data providing information the client can use to resolve the issue. While the client can take action on certain error codes, the registry may add new error codes over time. All client implementations should treat unknown error codes as `UNKNOWN`, allowing future error codes to be added without breaking API compatibility. For the purposes of the specification error codes will only be added and never removed. For a complete account of all error codes, please see the _Detail_ section. ### API Version Check A minimal endpoint, mounted at `/v2/` will provide version support information based on its response statuses. The request format is as follows: GET /v2/ If a `200 OK` response is returned, the registry implements the V2(.1) registry API and the client may proceed safely with other V2 operations. Optionally, the response may contain information about the supported paths in the response body. The client should be prepared to ignore this data. If a `401 Unauthorized` response is returned, the client should take action based on the contents of the "WWW-Authenticate" header and try the endpoint again. Depending on access control setup, the client may still have to authenticate against different resources, even if this check succeeds. If `404 Not Found` response status, or other unexpected status, is returned, the client should proceed with the assumption that the registry does not implement V2 of the API. ### Pulling An Image An "image" is a combination of a JSON manifest and individual layer files. The process of pulling an image centers around retrieving these two components. The first step in pulling an image is to retrieve the manifest. For reference, the relevant manifest fields for the registry are the following: field | description | ----------|------------------------------------------------| name | The name of the image. | tag | The tag for this version of the image. | fsLayers | A list of layer descriptors (including tarsum) | signature | A JWS used to verify the manifest content | For more information about the manifest format, please see [docker/docker#8093](https://github.com/docker/docker/issues/8093). When the manifest is in hand, the client must verify the signature to ensure the names and layers are valid. Once confirmed, the client will then use the tarsums to download the individual layers. Layers are stored in as blobs in the V2 registry API, keyed by their tarsum digest. #### Pulling an Image Manifest The image manifest can be fetched with the following url: ``` GET /v2//manifests/ ``` The `name` and `reference` parameter identify the image and are required. The reference may include a tag or digest. A `404 Not Found` response will be returned if the image is unknown to the registry. If the image exists and the response is successful, the image manifest will be returned, with the following format (see docker/docker#8093 for details): { "name": , "tag": , "fsLayers": [ { "blobSum": }, ... ] ], "history": , "signature": } The client should verify the returned manifest signature for authenticity before fetching layers. #### Pulling a Layer Layers are stored in the blob portion of the registry, keyed by tarsum digest. Pulling a layer is carried out by a standard http request. The URL is as follows: GET /v2//blobs/ Access to a layer will be gated by the `name` of the repository but is identified uniquely in the registry by `tarsum`. The `tarsum` parameter is an opaque field, to be interpreted by the tarsum library. This endpoint may issue a 307 (302 for /blobs/uploads/ ``` The parameters of this request are the image namespace under which the layer will be linked. Responses to this request are covered below. ##### Existing Layers The existence of a layer can be checked via a `HEAD` request to the blob store API. The request should be formatted as follows: ``` HEAD /v2//blobs/ ``` If the layer with the tarsum specified in `digest` is available, a 200 OK response will be received, with no actual body content (this is according to http specification). The response will look as follows: ``` 200 OK Content-Length: Docker-Content-Digest: ``` When this response is received, the client can assume that the layer is already available in the registry under the given name and should take no further action to upload the layer. Note that the binary digests may differ for the existing registry layer, but the tarsums will be guaranteed to match. ##### Uploading the Layer If the POST request is successful, a `202 Accepted` response will be returned with the upload URL in the `Location` header: ``` 202 Accepted Location: /v2//blobs/uploads/ Range: bytes=0- Content-Length: 0 Docker-Upload-UUID: ``` The rest of the upload process can be carried out with the returned url, called the "Upload URL" from the `Location` header. All responses to the upload url, whether sending data or getting status, will be in this format. Though the URI format (`/v2//blobs/uploads/`) for the `Location` header is specified, clients should treat it as an opaque url and should never try to assemble the it. While the `uuid` parameter may be an actual UUID, this proposal imposes no constraints on the format and clients should never impose any. If clients need to correlate local upload state with remote upload state, the contents of the `Docker-Upload-UUID` header should be used. Such an id can be used to key the last used location header when implementing resumable uploads. ##### Upload Progress The progress and chunk coordination of the upload process will be coordinated through the `Range` header. While this is a non-standard use of the `Range` header, there are examples of [similar approaches](https://developers.google.com/youtube/v3/guides/using_resumable_upload_protocol) in APIs with heavy use. For an upload that just started, for an example with a 1000 byte layer file, the `Range` header would be as follows: ``` Range: bytes=0-0 ``` To get the status of an upload, issue a GET request to the upload URL: ``` GET /v2//blobs/uploads/ Host: ``` The response will be similar to the above, except will return 204 status: ``` 204 No Content Location: /v2//blobs/uploads/ Range: bytes=0- Docker-Upload-UUID: ``` Note that the HTTP `Range` header byte ranges are inclusive and that will be honored, even in non-standard use cases. ##### Monolithic Upload A monolithic upload is simply a chunked upload with a single chunk and may be favored by clients that would like to avoided the complexity of chunking. To carry out a "monolithic" upload, one can simply put the entire content blob to the provided URL: ``` PUT /v2//blobs/uploads/?digest=[&digest=sha256:] Content-Length: Content-Type: application/octet-stream ``` The "digest" parameter must be included with the PUT request. Please see the _Completed Upload_ section for details on the parameters and expected responses. Additionally, the download can be completed with a single `POST` request to the uploads endpoint, including the "size" and "digest" parameters: ``` POST /v2//blobs/uploads/?digest=[&digest=sha256:] Content-Length: Content-Type: application/octet-stream ``` On the registry service, this should allocate a download, accept and verify the data and return the same response as the final chunk of an upload. If the POST request fails collecting the data in any way, the registry should attempt to return an error response to the client with the `Location` header providing a place to continue the download. The single `POST` method is provided for convenience and most clients should implement `POST` + `PUT` to support reliable resume of uploads. ##### Chunked Upload To carry out an upload of a chunk, the client can specify a range header and only include that part of the layer file: ``` PATCH /v2//blobs/uploads/ Content-Length: Content-Range: - Content-Type: application/octet-stream ``` There is no enforcement on layer chunk splits other than that the server must receive them in order. The server may enforce a minimum chunk size. If the server cannot accept the chunk, a `416 Requested Range Not Satisfiable` response will be returned and will include a `Range` header indicating the current status: ``` 416 Requested Range Not Satisfiable Location: /v2//blobs/uploads/ Range: 0- Content-Length: 0 Docker-Upload-UUID: ``` If this response is received, the client should resume from the "last valid range" and upload the subsequent chunk. A 416 will be returned under the following conditions: - Invalid Content-Range header format - Out of order chunk: the range of the next chunk must start immediately after the "last valid range" from the previous response. When a chunk is accepted as part of the upload, a `202 Accepted` response will be returned, including a `Range` header with the current upload status: ``` 202 Accepted Location: /v2//blobs/uploads/ Range: bytes=0- Content-Length: 0 Docker-Upload-UUID: ``` ##### Completed Upload For an upload to be considered complete, the client must submit a `PUT` request on the upload endpoint with a digest parameter. If it is not provided, the download will not be considered complete. The format for the final chunk will be as follows: ``` PUT /v2//blob/uploads/?digest=[&digest=sha256:] Content-Length: Content-Range: - Content-Type: application/octet-stream ``` Optionally, if all chunks have already been uploaded, a `PUT` request with a `digest` parameter and zero-length body may be sent to complete and validated the upload. Multiple "digest" parameters may be provided with different digests. The server may verify none or all of them but _must_ notify the client if the content is rejected. When the last chunk is received and the layer has been validated, the client will receive a `201 Created` response: ``` 201 Created Location: /v2//blobs/ Content-Length: 0 Docker-Content-Digest: ``` The `Location` header will contain the registry URL to access the accepted layer file. The `Docker-Content-Digest` header returns the canonical digest of the uploaded blob which may differ from the provided digest. Most clients may ignore the value but if it is used, the client should verify the value against the uploaded blob data. ###### Digest Parameter The "digest" parameter is designed as an opaque parameter to support verification of a successful transfer. The initial version of the registry API will support a tarsum digest, in the standard tarsum format. For example, a HTTP URI parameter might be as follows: ``` tarsum.v1+sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b ``` Given this parameter, the registry will verify that the provided content does result in this tarsum. Optionally, the registry can support other other digest parameters for non-tarfile content stored as a layer. A regular hash digest might be specified as follows: ``` sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b ``` Such a parameter would be used to verify that the binary content (as opposed to the tar content) would be verified at the end of the upload process. For the initial version, registry servers are only required to support the tarsum format. ##### Canceling an Upload An upload can be cancelled by issuing a DELETE request to the upload endpoint. The format will be as follows: ``` DELETE /v2//blobs/uploads/ ``` After this request is issued, the upload uuid will no longer be valid and the registry server will dump all intermediate data. While uploads will time out if not completed, clients should issue this request if they encounter a fatal error but still have the ability to issue an http request. ##### Errors If an 502, 503 or 504 error is received, the client should assume that the download can proceed due to a temporary condition, honoring the appropriate retry mechanism. Other 5xx errors should be treated as terminal. If there is a problem with the upload, a 4xx error will be returned indicating the problem. After receiving a 4xx response (except 416, as called out above), the upload will be considered failed and the client should take appropriate action. Note that the upload url will not be available forever. If the upload uuid is unknown to the registry, a `404 Not Found` response will be returned and the client must restart the upload process. #### Pushing an Image Manifest Once all of the layers for an image are uploaded, the client can upload the image manifest. An image can be pushed using the following request format: PUT /v2//manifests/ { "name": , "tag": , "fsLayers": [ { "blobSum": }, ... ] ], "history": , "signature": , ... } The `name` and `reference` fields of the response body must match those specified in the URL. The `reference` field may be a "tag" or a "digest". If there is a problem with pushing the manifest, a relevant 4xx response will be returned with a JSON error message. Please see the _PUT Manifest section for details on possible error codes that may be returned. If one or more layers are unknown to the registry, `BLOB_UNKNOWN` errors are returned. The `detail` field of the error response will have a `digest` field identifying the missing blob, which will be a tarsum. An error is returned for each unknown blob. The response format is as follows: { "errors:" [{ "code": "BLOB_UNKNOWN", "message": "blob unknown to registry", "detail": { "digest": } }, ... ] } #### Listing Image Tags It may be necessary to list all of the tags under a given repository. The tags for an image repository can be retrieved with the following request: GET /v2//tags/list The response will be in the following format: 200 OK Content-Type: application/json { "name": , "tags": [ , ... ] } For repositories with a large number of tags, this response may be quite large, so care should be taken by the client when parsing the response to reduce copying. ### Deleting an Image An image may be deleted from the registry via its `name` and `reference`. A delete may be issued with the following request format: DELETE /v2//manifests/ For deletes, `reference` *must* be a digest or the delete will fail. If the image exists and has been successfully deleted, the following response will be issued: 202 Accepted Content-Length: None If the image had already been deleted or did not exist, a `404 Not Found` response will be issued instead. ## Detail > **Note**: This section is still under construction. For the purposes of > implementation, if any details below differ from the described request flows > above, the section below should be corrected. When they match, this note > should be removed. The behavior of the endpoints are covered in detail in this section, organized by route and entity. All aspects of the request and responses are covered, including headers, parameters and body formats. Examples of requests and their corresponding responses, with success and failure, are enumerated. > **Note**: The sections on endpoint detail are arranged with an example > request, a description of the request, followed by information about that > request. A list of methods and URIs are covered in the table below: |Method|Path|Entity|Description| -------|----|------|------------ | GET | `/v2/` | Base | Check that the endpoint implements Docker Registry API V2. | | GET | `/v2//tags/list` | Tags | Fetch the tags under the repository identified by `name`. | | GET | `/v2//manifests/` | Manifest | Fetch the manifest identified by `name` and `reference` where `reference` can be a tag or digest. | | PUT | `/v2//manifests/` | Manifest | Put the manifest identified by `name` and `reference` where `reference` can be a tag or digest. | | DELETE | `/v2//manifests/` | Manifest | Delete the manifest identified by `name` and `reference` where `reference` can be a tag or digest. | | GET | `/v2//blobs/` | Blob | Retrieve the blob from the registry identified by `digest`. A `HEAD` request can also be issued to this endpoint to obtain resource information without receiving all data. | | POST | `/v2//blobs/uploads/` | Intiate Blob Upload | Initiate a resumable blob upload. If successful, an upload location will be provided to complete the upload. Optionally, if the `digest` parameter is present, the request body will be used to complete the upload in a single request. | | GET | `/v2//blobs/uploads/` | Blob Upload | Retrieve status of upload identified by `uuid`. The primary purpose of this endpoint is to resolve the current status of a resumable upload. | | PATCH | `/v2//blobs/uploads/` | Blob Upload | Upload a chunk of data for the specified upload. | | PUT | `/v2//blobs/uploads/` | Blob Upload | Complete the upload specified by `uuid`, optionally appending the body as the final chunk. | | DELETE | `/v2//blobs/uploads/` | Blob Upload | Cancel outstanding upload processes, releasing associated resources. If this is not called, the unfinished uploads will eventually timeout. | The detail for each endpoint is covered in the following sections. ### Errors The error codes encountered via the API are enumerated in the following table: |Code|Message|Description| -------|----|------|------------ `UNKNOWN` | unknown error | Generic error returned when the error does not have an API classification. `UNSUPPORTED` | The operation is unsupported. | The operation was unsupported due to a missing implementation or invalid set of parameters. `UNAUTHORIZED` | access to the requested resource is not authorized | The access controller denied access for the operation on a resource. Often this will be accompanied by a 401 Unauthorized response status. `DIGEST_INVALID` | provided digest did not match uploaded content | When a blob is uploaded, the registry will check that the content matches the digest provided by the client. The error may include a detail structure with the key "digest", including the invalid digest string. This error may also be returned when a manifest includes an invalid layer digest. `SIZE_INVALID` | provided length did not match content length | When a layer is uploaded, the provided size will be checked against the uploaded content. If they do not match, this error will be returned. `NAME_INVALID` | invalid repository name | Invalid repository name encountered either during manifest validation or any API operation. `TAG_INVALID` | manifest tag did not match URI | During a manifest upload, if the tag in the manifest does not match the uri tag, this error will be returned. `NAME_UNKNOWN` | repository name not known to registry | This is returned if the name used during an operation is unknown to the registry. `MANIFEST_UNKNOWN` | manifest unknown | This error is returned when the manifest, identified by name and tag is unknown to the repository. `MANIFEST_INVALID` | manifest invalid | During upload, manifests undergo several checks ensuring validity. If those checks fail, this error may be returned, unless a more specific error is included. The detail will contain information the failed validation. `MANIFEST_UNVERIFIED` | manifest failed signature verification | During manifest upload, if the manifest fails signature verification, this error will be returned. `BLOB_UNKNOWN` | blob unknown to registry | This error may be returned when a blob is unknown to the registry in a specified repository. This can be returned with a standard get or if a manifest references an unknown layer during upload. `BLOB_UPLOAD_UNKNOWN` | blob upload unknown to registry | If a blob upload has been cancelled or was never started, this error code may be returned. `BLOB_UPLOAD_INVALID` | blob upload invalid | The blob upload encountered an error and can no longer proceed. ### Base Base V2 API route. Typically, this can be used for lightweight version checks and to validate registry authorization. #### GET Base Check that the endpoint implements Docker Registry API V2. ``` GET /v2/ Host: Authorization: ``` The following parameters should be specified on the request: |Name|Kind|Description| |----|----|-----------| |`Host`|header|Standard HTTP Host Header. Should be set to the registry host.| |`Authorization`|header|An RFC7235 compliant authorization header.| ###### On Success: OK ``` 200 OK ``` The API implements V2 protocol and is accessible. ###### On Failure: Unauthorized ``` 401 Unauthorized WWW-Authenticate: realm="", ..." Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` The client is not authorized to access the registry. The following headers will be returned on the response: |Name|Description| |----|-----------| |`WWW-Authenticate`|An RFC7235 compliant authentication challenge header.| The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `UNAUTHORIZED` | access to the requested resource is not authorized | The access controller denied access for the operation on a resource. Often this will be accompanied by a 401 Unauthorized response status. | ###### On Failure: Not Found ``` 404 Not Found ``` The registry does not implement the V2 API. ### Tags Retrieve information about tags. #### GET Tags Fetch the tags under the repository identified by `name`. ``` GET /v2//tags/list Host: Authorization: ``` The following parameters should be specified on the request: |Name|Kind|Description| |----|----|-----------| |`Host`|header|Standard HTTP Host Header. Should be set to the registry host.| |`Authorization`|header|An RFC7235 compliant authorization header.| |`name`|path|Name of the target repository.| ###### On Success: OK ``` 200 OK Content-Length: Content-Type: application/json; charset=utf-8 { "name": , "tags": [ , ... ] } ``` A list of tags for the named repository. The following headers will be returned with the response: |Name|Description| |----|-----------| |`Content-Length`|Length of the JSON response body.| ###### On Failure: Not Found ``` 404 Not Found Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` The repository is not known to the registry. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `NAME_UNKNOWN` | repository name not known to registry | This is returned if the name used during an operation is unknown to the registry. | ###### On Failure: Unauthorized ``` 401 Unauthorized Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` The client does not have access to the repository. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `UNAUTHORIZED` | access to the requested resource is not authorized | The access controller denied access for the operation on a resource. Often this will be accompanied by a 401 Unauthorized response status. | ### Manifest Create, update and retrieve manifests. #### GET Manifest Fetch the manifest identified by `name` and `reference` where `reference` can be a tag or digest. ``` GET /v2//manifests/ Host: Authorization: ``` The following parameters should be specified on the request: |Name|Kind|Description| |----|----|-----------| |`Host`|header|Standard HTTP Host Header. Should be set to the registry host.| |`Authorization`|header|An RFC7235 compliant authorization header.| |`name`|path|Name of the target repository.| |`tag`|path|Tag of the target manifiest.| ###### On Success: OK ``` 200 OK Docker-Content-Digest: Content-Type: application/json; charset=utf-8 { "name": , "tag": , "fsLayers": [ { "blobSum": }, ... ] ], "history": , "signature": } ``` The manifest idenfied by `name` and `reference`. The contents can be used to identify and resolve resources required to run the specified image. The following headers will be returned with the response: |Name|Description| |----|-----------| |`Docker-Content-Digest`|Digest of the targeted content for the request.| ###### On Failure: Bad Request ``` 400 Bad Request Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` The name or reference was invalid. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `NAME_INVALID` | invalid repository name | Invalid repository name encountered either during manifest validation or any API operation. | | `TAG_INVALID` | manifest tag did not match URI | During a manifest upload, if the tag in the manifest does not match the uri tag, this error will be returned. | ###### On Failure: Unauthorized ``` 401 Unauthorized Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` The client does not have access to the repository. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `UNAUTHORIZED` | access to the requested resource is not authorized | The access controller denied access for the operation on a resource. Often this will be accompanied by a 401 Unauthorized response status. | ###### On Failure: Not Found ``` 404 Not Found Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` The named manifest is not known to the registry. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `NAME_UNKNOWN` | repository name not known to registry | This is returned if the name used during an operation is unknown to the registry. | | `MANIFEST_UNKNOWN` | manifest unknown | This error is returned when the manifest, identified by name and tag is unknown to the repository. | #### PUT Manifest Put the manifest identified by `name` and `reference` where `reference` can be a tag or digest. ``` PUT /v2//manifests/ Host: Authorization: Content-Type: application/json; charset=utf-8 { "name": , "tag": , "fsLayers": [ { "blobSum": }, ... ] ], "history": , "signature": } ``` The following parameters should be specified on the request: |Name|Kind|Description| |----|----|-----------| |`Host`|header|Standard HTTP Host Header. Should be set to the registry host.| |`Authorization`|header|An RFC7235 compliant authorization header.| |`name`|path|Name of the target repository.| |`tag`|path|Tag of the target manifiest.| ###### On Success: Accepted ``` 202 Accepted Location: Content-Length: 0 Docker-Content-Digest: ``` The manifest has been accepted by the registry and is stored under the specified `name` and `tag`. The following headers will be returned with the response: |Name|Description| |----|-----------| |`Location`|The canonical location url of the uploaded manifest.| |`Content-Length`|The `Content-Length` header must be zero and the body must be empty.| |`Docker-Content-Digest`|Digest of the targeted content for the request.| ###### On Failure: Invalid Manifest ``` 400 Bad Request Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` The received manifest was invalid in some way, as described by the error codes. The client should resolve the issue and retry the request. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `NAME_INVALID` | invalid repository name | Invalid repository name encountered either during manifest validation or any API operation. | | `TAG_INVALID` | manifest tag did not match URI | During a manifest upload, if the tag in the manifest does not match the uri tag, this error will be returned. | | `MANIFEST_INVALID` | manifest invalid | During upload, manifests undergo several checks ensuring validity. If those checks fail, this error may be returned, unless a more specific error is included. The detail will contain information the failed validation. | | `MANIFEST_UNVERIFIED` | manifest failed signature verification | During manifest upload, if the manifest fails signature verification, this error will be returned. | | `BLOB_UNKNOWN` | blob unknown to registry | This error may be returned when a blob is unknown to the registry in a specified repository. This can be returned with a standard get or if a manifest references an unknown layer during upload. | ###### On Failure: Unauthorized ``` 401 Unauthorized Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` The client does not have permission to push to the repository. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `UNAUTHORIZED` | access to the requested resource is not authorized | The access controller denied access for the operation on a resource. Often this will be accompanied by a 401 Unauthorized response status. | ###### On Failure: Missing Layer(s) ``` 400 Bad Request Content-Type: application/json; charset=utf-8 { "errors:" [{ "code": "BLOB_UNKNOWN", "message": "blob unknown to registry", "detail": { "digest": } }, ... ] } ``` One or more layers may be missing during a manifest upload. If so, the missing layers will be enumerated in the error response. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `BLOB_UNKNOWN` | blob unknown to registry | This error may be returned when a blob is unknown to the registry in a specified repository. This can be returned with a standard get or if a manifest references an unknown layer during upload. | ###### On Failure: Unauthorized ``` 401 Unauthorized WWW-Authenticate: realm="", ..." Content-Length: Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` The following headers will be returned on the response: |Name|Description| |----|-----------| |`WWW-Authenticate`|An RFC7235 compliant authentication challenge header.| |`Content-Length`|Length of the JSON error response body.| The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `UNAUTHORIZED` | access to the requested resource is not authorized | The access controller denied access for the operation on a resource. Often this will be accompanied by a 401 Unauthorized response status. | #### DELETE Manifest Delete the manifest identified by `name` and `reference` where `reference` can be a tag or digest. ``` DELETE /v2//manifests/ Host: Authorization: ``` The following parameters should be specified on the request: |Name|Kind|Description| |----|----|-----------| |`Host`|header|Standard HTTP Host Header. Should be set to the registry host.| |`Authorization`|header|An RFC7235 compliant authorization header.| |`name`|path|Name of the target repository.| |`tag`|path|Tag of the target manifiest.| ###### On Success: Accepted ``` 202 Accepted ``` ###### On Failure: Invalid Name or Tag ``` 400 Bad Request Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` The specified `name` or `tag` were invalid and the delete was unable to proceed. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `NAME_INVALID` | invalid repository name | Invalid repository name encountered either during manifest validation or any API operation. | | `TAG_INVALID` | manifest tag did not match URI | During a manifest upload, if the tag in the manifest does not match the uri tag, this error will be returned. | ###### On Failure: Unauthorized ``` 401 Unauthorized WWW-Authenticate: realm="", ..." Content-Length: Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` The following headers will be returned on the response: |Name|Description| |----|-----------| |`WWW-Authenticate`|An RFC7235 compliant authentication challenge header.| |`Content-Length`|Length of the JSON error response body.| The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `UNAUTHORIZED` | access to the requested resource is not authorized | The access controller denied access for the operation on a resource. Often this will be accompanied by a 401 Unauthorized response status. | ###### On Failure: Unknown Manifest ``` 404 Not Found Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` The specified `name` or `tag` are unknown to the registry and the delete was unable to proceed. Clients can assume the manifest was already deleted if this response is returned. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `NAME_UNKNOWN` | repository name not known to registry | This is returned if the name used during an operation is unknown to the registry. | | `MANIFEST_UNKNOWN` | manifest unknown | This error is returned when the manifest, identified by name and tag is unknown to the repository. | ### Blob Fetch the blob identified by `name` and `digest`. Used to fetch layers by tarsum digest. #### GET Blob Retrieve the blob from the registry identified by `digest`. A `HEAD` request can also be issued to this endpoint to obtain resource information without receiving all data. ##### Fetch Blob ``` GET /v2//blobs/ Host: Authorization: ``` The following parameters should be specified on the request: |Name|Kind|Description| |----|----|-----------| |`Host`|header|Standard HTTP Host Header. Should be set to the registry host.| |`Authorization`|header|An RFC7235 compliant authorization header.| |`name`|path|Name of the target repository.| |`digest`|path|Digest of desired blob.| ###### On Success: OK ``` 200 OK Content-Length: Docker-Content-Digest: Content-Type: application/octet-stream ``` The blob identified by `digest` is available. The blob content will be present in the body of the request. The following headers will be returned with the response: |Name|Description| |----|-----------| |`Content-Length`|The length of the requested blob content.| |`Docker-Content-Digest`|Digest of the targeted content for the request.| ###### On Success: Temporary Redirect ``` 307 Temporary Redirect Location: Docker-Content-Digest: ``` The blob identified by `digest` is available at the provided location. The following headers will be returned with the response: |Name|Description| |----|-----------| |`Location`|The location where the layer should be accessible.| |`Docker-Content-Digest`|Digest of the targeted content for the request.| ###### On Failure: Bad Request ``` 400 Bad Request Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` There was a problem with the request that needs to be addressed by the client, such as an invalid `name` or `tag`. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `NAME_INVALID` | invalid repository name | Invalid repository name encountered either during manifest validation or any API operation. | | `DIGEST_INVALID` | provided digest did not match uploaded content | When a blob is uploaded, the registry will check that the content matches the digest provided by the client. The error may include a detail structure with the key "digest", including the invalid digest string. This error may also be returned when a manifest includes an invalid layer digest. | ###### On Failure: Unauthorized ``` 401 Unauthorized WWW-Authenticate: realm="", ..." Content-Length: Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": "UNAUTHORIZED", "message": "access to the requested resource is not authorized", "detail": ... }, ... ] } ``` The client does not have access to the repository. The following headers will be returned on the response: |Name|Description| |----|-----------| |`WWW-Authenticate`|An RFC7235 compliant authentication challenge header.| |`Content-Length`|Length of the JSON error response body.| The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `UNAUTHORIZED` | access to the requested resource is not authorized | The access controller denied access for the operation on a resource. Often this will be accompanied by a 401 Unauthorized response status. | ###### On Failure: Not Found ``` 404 Not Found Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` The blob, identified by `name` and `digest`, is unknown to the registry. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `NAME_UNKNOWN` | repository name not known to registry | This is returned if the name used during an operation is unknown to the registry. | | `BLOB_UNKNOWN` | blob unknown to registry | This error may be returned when a blob is unknown to the registry in a specified repository. This can be returned with a standard get or if a manifest references an unknown layer during upload. | ##### Fetch Blob Part ``` GET /v2//blobs/ Host: Authorization: Range: bytes=- ``` This endpoint may also support RFC7233 compliant range requests. Support can be detected by issuing a HEAD request. If the header `Accept-Range: bytes` is returned, range requests can be used to fetch partial content. The following parameters should be specified on the request: |Name|Kind|Description| |----|----|-----------| |`Host`|header|Standard HTTP Host Header. Should be set to the registry host.| |`Authorization`|header|An RFC7235 compliant authorization header.| |`Range`|header|HTTP Range header specifying blob chunk.| |`name`|path|Name of the target repository.| |`digest`|path|Digest of desired blob.| ###### On Success: Partial Content ``` 206 Partial Content Content-Length: Content-Range: bytes -/ Content-Type: application/octet-stream ``` The blob identified by `digest` is available. The specified chunk of blob content will be present in the body of the request. The following headers will be returned with the response: |Name|Description| |----|-----------| |`Content-Length`|The length of the requested blob chunk.| |`Content-Range`|Content range of blob chunk.| ###### On Failure: Bad Request ``` 400 Bad Request Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` There was a problem with the request that needs to be addressed by the client, such as an invalid `name` or `tag`. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `NAME_INVALID` | invalid repository name | Invalid repository name encountered either during manifest validation or any API operation. | | `DIGEST_INVALID` | provided digest did not match uploaded content | When a blob is uploaded, the registry will check that the content matches the digest provided by the client. The error may include a detail structure with the key "digest", including the invalid digest string. This error may also be returned when a manifest includes an invalid layer digest. | ###### On Failure: Unauthorized ``` 401 Unauthorized WWW-Authenticate: realm="", ..." Content-Length: Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": "UNAUTHORIZED", "message": "access to the requested resource is not authorized", "detail": ... }, ... ] } ``` The client does not have access to the repository. The following headers will be returned on the response: |Name|Description| |----|-----------| |`WWW-Authenticate`|An RFC7235 compliant authentication challenge header.| |`Content-Length`|Length of the JSON error response body.| The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `UNAUTHORIZED` | access to the requested resource is not authorized | The access controller denied access for the operation on a resource. Often this will be accompanied by a 401 Unauthorized response status. | ###### On Failure: Not Found ``` 404 Not Found Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `NAME_UNKNOWN` | repository name not known to registry | This is returned if the name used during an operation is unknown to the registry. | | `BLOB_UNKNOWN` | blob unknown to registry | This error may be returned when a blob is unknown to the registry in a specified repository. This can be returned with a standard get or if a manifest references an unknown layer during upload. | ###### On Failure: Requested Range Not Satisfiable ``` 416 Requested Range Not Satisfiable ``` The range specification cannot be satisfied for the requested content. This can happen when the range is not formatted correctly or if the range is outside of the valid size of the content. ### Intiate Blob Upload Initiate a blob upload. This endpoint can be used to create resumable uploads or monolithic uploads. #### POST Intiate Blob Upload Initiate a resumable blob upload. If successful, an upload location will be provided to complete the upload. Optionally, if the `digest` parameter is present, the request body will be used to complete the upload in a single request. ##### Initiate Monolithic Blob Upload ``` POST /v2//blobs/uploads/?digest= Host: Authorization: Content-Length: Content-Type: application/octect-stream ``` Upload a blob identified by the `digest` parameter in single request. This upload will not be resumable unless a recoverable error is returned. The following parameters should be specified on the request: |Name|Kind|Description| |----|----|-----------| |`Host`|header|Standard HTTP Host Header. Should be set to the registry host.| |`Authorization`|header|An RFC7235 compliant authorization header.| |`Content-Length`|header|| |`name`|path|Name of the target repository.| |`digest`|query|Digest of uploaded blob. If present, the upload will be completed, in a single request, with contents of the request body as the resulting blob.| ###### On Success: Created ``` 201 Created Location: Content-Length: 0 Docker-Upload-UUID: ``` The blob has been created in the registry and is available at the provided location. The following headers will be returned with the response: |Name|Description| |----|-----------| |`Location`|| |`Content-Length`|The `Content-Length` header must be zero and the body must be empty.| |`Docker-Upload-UUID`|Identifies the docker upload uuid for the current request.| ###### On Failure: Invalid Name or Digest ``` 400 Bad Request ``` The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `DIGEST_INVALID` | provided digest did not match uploaded content | When a blob is uploaded, the registry will check that the content matches the digest provided by the client. The error may include a detail structure with the key "digest", including the invalid digest string. This error may also be returned when a manifest includes an invalid layer digest. | | `NAME_INVALID` | invalid repository name | Invalid repository name encountered either during manifest validation or any API operation. | ###### On Failure: Unauthorized ``` 401 Unauthorized WWW-Authenticate: realm="", ..." Content-Length: Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": "UNAUTHORIZED", "message": "access to the requested resource is not authorized", "detail": ... }, ... ] } ``` The client does not have access to push to the repository. The following headers will be returned on the response: |Name|Description| |----|-----------| |`WWW-Authenticate`|An RFC7235 compliant authentication challenge header.| |`Content-Length`|Length of the JSON error response body.| The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `UNAUTHORIZED` | access to the requested resource is not authorized | The access controller denied access for the operation on a resource. Often this will be accompanied by a 401 Unauthorized response status. | ##### Initiate Resumable Blob Upload ``` POST /v2//blobs/uploads/ Host: Authorization: Content-Length: 0 ``` Initiate a resumable blob upload with an empty request body. The following parameters should be specified on the request: |Name|Kind|Description| |----|----|-----------| |`Host`|header|Standard HTTP Host Header. Should be set to the registry host.| |`Authorization`|header|An RFC7235 compliant authorization header.| |`Content-Length`|header|The `Content-Length` header must be zero and the body must be empty.| |`name`|path|Name of the target repository.| ###### On Success: Accepted ``` 202 Accepted Content-Length: 0 Location: /v2//blobs/uploads/ Range: 0-0 Docker-Upload-UUID: ``` The upload has been created. The `Location` header must be used to complete the upload. The response should be identical to a `GET` request on the contents of the returned `Location` header. The following headers will be returned with the response: |Name|Description| |----|-----------| |`Content-Length`|The `Content-Length` header must be zero and the body must be empty.| |`Location`|The location of the created upload. Clients should use the contents verbatim to complete the upload, adding parameters where required.| |`Range`|Range header indicating the progress of the upload. When starting an upload, it will return an empty range, since no content has been received.| |`Docker-Upload-UUID`|Identifies the docker upload uuid for the current request.| ###### On Failure: Invalid Name or Digest ``` 400 Bad Request ``` The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `DIGEST_INVALID` | provided digest did not match uploaded content | When a blob is uploaded, the registry will check that the content matches the digest provided by the client. The error may include a detail structure with the key "digest", including the invalid digest string. This error may also be returned when a manifest includes an invalid layer digest. | | `NAME_INVALID` | invalid repository name | Invalid repository name encountered either during manifest validation or any API operation. | ###### On Failure: Unauthorized ``` 401 Unauthorized WWW-Authenticate: realm="", ..." Content-Length: Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": "UNAUTHORIZED", "message": "access to the requested resource is not authorized", "detail": ... }, ... ] } ``` The client does not have access to push to the repository. The following headers will be returned on the response: |Name|Description| |----|-----------| |`WWW-Authenticate`|An RFC7235 compliant authentication challenge header.| |`Content-Length`|Length of the JSON error response body.| The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `UNAUTHORIZED` | access to the requested resource is not authorized | The access controller denied access for the operation on a resource. Often this will be accompanied by a 401 Unauthorized response status. | ### Blob Upload Interact with blob uploads. Clients should never assemble URLs for this endpoint and should only take it through the `Location` header on related API requests. The `Location` header and its parameters should be preserved by clients, using the latest value returned via upload related API calls. #### GET Blob Upload Retrieve status of upload identified by `uuid`. The primary purpose of this endpoint is to resolve the current status of a resumable upload. ``` GET /v2//blobs/uploads/ Host: Authorization: ``` Retrieve the progress of the current upload, as reported by the `Range` header. The following parameters should be specified on the request: |Name|Kind|Description| |----|----|-----------| |`Host`|header|Standard HTTP Host Header. Should be set to the registry host.| |`Authorization`|header|An RFC7235 compliant authorization header.| |`name`|path|Name of the target repository.| |`uuid`|path|A uuid identifying the upload. This field can accept almost anything.| ###### On Success: Upload Progress ``` 204 No Content Range: 0- Content-Length: 0 Docker-Upload-UUID: ``` The upload is known and in progress. The last received offset is available in the `Range` header. The following headers will be returned with the response: |Name|Description| |----|-----------| |`Range`|Range indicating the current progress of the upload.| |`Content-Length`|The `Content-Length` header must be zero and the body must be empty.| |`Docker-Upload-UUID`|Identifies the docker upload uuid for the current request.| ###### On Failure: Bad Request ``` 400 Bad Request Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` There was an error processing the upload and it must be restarted. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `DIGEST_INVALID` | provided digest did not match uploaded content | When a blob is uploaded, the registry will check that the content matches the digest provided by the client. The error may include a detail structure with the key "digest", including the invalid digest string. This error may also be returned when a manifest includes an invalid layer digest. | | `NAME_INVALID` | invalid repository name | Invalid repository name encountered either during manifest validation or any API operation. | | `BLOB_UPLOAD_INVALID` | blob upload invalid | The blob upload encountered an error and can no longer proceed. | ###### On Failure: Unauthorized ``` 401 Unauthorized WWW-Authenticate: realm="", ..." Content-Length: Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": "UNAUTHORIZED", "message": "access to the requested resource is not authorized", "detail": ... }, ... ] } ``` The client does not have access to the repository. The following headers will be returned on the response: |Name|Description| |----|-----------| |`WWW-Authenticate`|An RFC7235 compliant authentication challenge header.| |`Content-Length`|Length of the JSON error response body.| The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `UNAUTHORIZED` | access to the requested resource is not authorized | The access controller denied access for the operation on a resource. Often this will be accompanied by a 401 Unauthorized response status. | ###### On Failure: Not Found ``` 404 Not Found Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` The upload is unknown to the registry. The upload must be restarted. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `BLOB_UPLOAD_UNKNOWN` | blob upload unknown to registry | If a blob upload has been cancelled or was never started, this error code may be returned. | #### PATCH Blob Upload Upload a chunk of data for the specified upload. ``` PATCH /v2//blobs/uploads/ Host: Authorization: Content-Range: - Content-Length: Content-Type: application/octet-stream ``` Upload a chunk of data to specified upload without completing the upload. The following parameters should be specified on the request: |Name|Kind|Description| |----|----|-----------| |`Host`|header|Standard HTTP Host Header. Should be set to the registry host.| |`Authorization`|header|An RFC7235 compliant authorization header.| |`Content-Range`|header|Range of bytes identifying the desired block of content represented by the body. Start must the end offset retrieved via status check plus one. Note that this is a non-standard use of the `Content-Range` header.| |`Content-Length`|header|Length of the chunk being uploaded, corresponding the length of the request body.| |`name`|path|Name of the target repository.| |`uuid`|path|A uuid identifying the upload. This field can accept almost anything.| ###### On Success: Chunk Accepted ``` 204 No Content Location: /v2//blobs/uploads/ Range: 0- Content-Length: 0 Docker-Upload-UUID: ``` The chunk of data has been accepted and the current progress is available in the range header. The updated upload location is available in the `Location` header. The following headers will be returned with the response: |Name|Description| |----|-----------| |`Location`|The location of the upload. Clients should assume this changes after each request. Clients should use the contents verbatim to complete the upload, adding parameters where required.| |`Range`|Range indicating the current progress of the upload.| |`Content-Length`|The `Content-Length` header must be zero and the body must be empty.| |`Docker-Upload-UUID`|Identifies the docker upload uuid for the current request.| ###### On Failure: Bad Request ``` 400 Bad Request Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` There was an error processing the upload and it must be restarted. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `DIGEST_INVALID` | provided digest did not match uploaded content | When a blob is uploaded, the registry will check that the content matches the digest provided by the client. The error may include a detail structure with the key "digest", including the invalid digest string. This error may also be returned when a manifest includes an invalid layer digest. | | `NAME_INVALID` | invalid repository name | Invalid repository name encountered either during manifest validation or any API operation. | | `BLOB_UPLOAD_INVALID` | blob upload invalid | The blob upload encountered an error and can no longer proceed. | ###### On Failure: Unauthorized ``` 401 Unauthorized WWW-Authenticate: realm="", ..." Content-Length: Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": "UNAUTHORIZED", "message": "access to the requested resource is not authorized", "detail": ... }, ... ] } ``` The client does not have access to push to the repository. The following headers will be returned on the response: |Name|Description| |----|-----------| |`WWW-Authenticate`|An RFC7235 compliant authentication challenge header.| |`Content-Length`|Length of the JSON error response body.| The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `UNAUTHORIZED` | access to the requested resource is not authorized | The access controller denied access for the operation on a resource. Often this will be accompanied by a 401 Unauthorized response status. | ###### On Failure: Not Found ``` 404 Not Found Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` The upload is unknown to the registry. The upload must be restarted. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `BLOB_UPLOAD_UNKNOWN` | blob upload unknown to registry | If a blob upload has been cancelled or was never started, this error code may be returned. | ###### On Failure: Requested Range Not Satisfiable ``` 416 Requested Range Not Satisfiable ``` The `Content-Range` specification cannot be accepted, either because it does not overlap with the current progress or it is invalid. #### PUT Blob Upload Complete the upload specified by `uuid`, optionally appending the body as the final chunk. ``` PUT /v2//blobs/uploads/?digest= Host: Authorization: Content-Range: - Content-Length: Content-Type: application/octet-stream ``` Complete the upload, providing the _final_ chunk of data, if necessary. This method may take a body with all the data. If the `Content-Range` header is specified, it may include the final chunk. A request without a body will just complete the upload with previously uploaded content. The following parameters should be specified on the request: |Name|Kind|Description| |----|----|-----------| |`Host`|header|Standard HTTP Host Header. Should be set to the registry host.| |`Authorization`|header|An RFC7235 compliant authorization header.| |`Content-Range`|header|Range of bytes identifying the block of content represented by the body. Start must the end offset retrieved via status check plus one. Note that this is a non-standard use of the `Content-Range` header. May be omitted if no data is provided.| |`Content-Length`|header|Length of the chunk being uploaded, corresponding to the length of the request body. May be zero if no data is provided.| |`name`|path|Name of the target repository.| |`uuid`|path|A uuid identifying the upload. This field can accept almost anything.| |`digest`|query|Digest of uploaded blob.| ###### On Success: Upload Complete ``` 204 No Content Location: Content-Range: - Content-Length: Docker-Content-Digest: ``` The upload has been completed and accepted by the registry. The canonical location will be available in the `Location` header. The following headers will be returned with the response: |Name|Description| |----|-----------| |`Location`|| |`Content-Range`|Range of bytes identifying the desired block of content represented by the body. Start must match the end of offset retrieved via status check. Note that this is a non-standard use of the `Content-Range` header.| |`Content-Length`|Length of the chunk being uploaded, corresponding the length of the request body.| |`Docker-Content-Digest`|Digest of the targeted content for the request.| ###### On Failure: Bad Request ``` 400 Bad Request Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` There was an error processing the upload and it must be restarted. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `DIGEST_INVALID` | provided digest did not match uploaded content | When a blob is uploaded, the registry will check that the content matches the digest provided by the client. The error may include a detail structure with the key "digest", including the invalid digest string. This error may also be returned when a manifest includes an invalid layer digest. | | `NAME_INVALID` | invalid repository name | Invalid repository name encountered either during manifest validation or any API operation. | | `BLOB_UPLOAD_INVALID` | blob upload invalid | The blob upload encountered an error and can no longer proceed. | ###### On Failure: Unauthorized ``` 401 Unauthorized WWW-Authenticate: realm="", ..." Content-Length: Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": "UNAUTHORIZED", "message": "access to the requested resource is not authorized", "detail": ... }, ... ] } ``` The client does not have access to push to the repository. The following headers will be returned on the response: |Name|Description| |----|-----------| |`WWW-Authenticate`|An RFC7235 compliant authentication challenge header.| |`Content-Length`|Length of the JSON error response body.| The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `UNAUTHORIZED` | access to the requested resource is not authorized | The access controller denied access for the operation on a resource. Often this will be accompanied by a 401 Unauthorized response status. | ###### On Failure: Not Found ``` 404 Not Found Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` The upload is unknown to the registry. The upload must be restarted. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `BLOB_UPLOAD_UNKNOWN` | blob upload unknown to registry | If a blob upload has been cancelled or was never started, this error code may be returned. | ###### On Failure: Requested Range Not Satisfiable ``` 416 Requested Range Not Satisfiable Location: /v2//blobs/uploads/ Range: 0- ``` The `Content-Range` specification cannot be accepted, either because it does not overlap with the current progress or it is invalid. The contents of the `Range` header may be used to resolve the condition. The following headers will be returned on the response: |Name|Description| |----|-----------| |`Location`|The location of the upload. Clients should assume this changes after each request. Clients should use the contents verbatim to complete the upload, adding parameters where required.| |`Range`|Range indicating the current progress of the upload.| #### DELETE Blob Upload Cancel outstanding upload processes, releasing associated resources. If this is not called, the unfinished uploads will eventually timeout. ``` DELETE /v2//blobs/uploads/ Host: Authorization: Content-Length: 0 ``` Cancel the upload specified by `uuid`. The following parameters should be specified on the request: |Name|Kind|Description| |----|----|-----------| |`Host`|header|Standard HTTP Host Header. Should be set to the registry host.| |`Authorization`|header|An RFC7235 compliant authorization header.| |`Content-Length`|header|The `Content-Length` header must be zero and the body must be empty.| |`name`|path|Name of the target repository.| |`uuid`|path|A uuid identifying the upload. This field can accept almost anything.| ###### On Success: Upload Deleted ``` 204 No Content Content-Length: 0 ``` The upload has been successfully deleted. The following headers will be returned with the response: |Name|Description| |----|-----------| |`Content-Length`|The `Content-Length` header must be zero and the body must be empty.| ###### On Failure: Bad Request ``` 400 Bad Request Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` An error was encountered processing the delete. The client may ignore this error. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `NAME_INVALID` | invalid repository name | Invalid repository name encountered either during manifest validation or any API operation. | | `BLOB_UPLOAD_INVALID` | blob upload invalid | The blob upload encountered an error and can no longer proceed. | ###### On Failure: Unauthorized ``` 401 Unauthorized WWW-Authenticate: realm="", ..." Content-Length: Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": "UNAUTHORIZED", "message": "access to the requested resource is not authorized", "detail": ... }, ... ] } ``` The client does not have access to the repository. The following headers will be returned on the response: |Name|Description| |----|-----------| |`WWW-Authenticate`|An RFC7235 compliant authentication challenge header.| |`Content-Length`|Length of the JSON error response body.| The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `UNAUTHORIZED` | access to the requested resource is not authorized | The access controller denied access for the operation on a resource. Often this will be accompanied by a 401 Unauthorized response status. | ###### On Failure: Not Found ``` 404 Not Found Content-Type: application/json; charset=utf-8 { "errors:" [ { "code": , "message": "", "detail": ... }, ... ] } ``` The upload is unknown to the registry. The client may ignore this error and assume the upload has been deleted. The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ | `BLOB_UPLOAD_UNKNOWN` | blob upload unknown to registry | If a blob upload has been cancelled or was never started, this error code may be returned. | distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/doc/spec/api.md.tmpl0000644000175000017500000006752012502424227024336 0ustar tianontianon# Docker Registry HTTP API V2 ## Introduction The _Docker Registry HTTP API_ is the protocol to facilitate distribution of images to the docker engine. It interacts with instances of the docker registry, which is a service to manage information about docker images and enable their distribution. The specification covers the operation of version 2 of this API, known as _Docker Registry HTTP API V2_. While the V1 registry protocol is usable, there are several problems with the architecture that have led to this new version. The main driver of this specification these changes to the docker the image format, covered in docker/docker#8093. The new, self-contained image manifest simplifies image definition and improves security. This specification will build on that work, leveraging new properties of the manifest format to improve performance, reduce bandwidth usage and decrease the likelihood of backend corruption. For relevant details and history leading up to this specification, please see the following issues: - [docker/docker#8093](https://github.com/docker/docker/issues/8903) - [docker/docker#9015](https://github.com/docker/docker/issues/9015) - [docker/docker-registry#612](https://github.com/docker/docker-registry/issues/612) ### Scope This specification covers the URL layout and protocols of the interaction between docker registry and docker core. This will affect the docker core registry API and the rewrite of docker-registry. Docker registry implementations may implement other API endpoints, but they are not covered by this specification. This includes the following features: - Namespace-oriented URI Layout - PUSH/PULL registry server for V2 image manifest format - Resumable layer PUSH support - V2 Client library implementation While authentication and authorization support will influence this specification, details of the protocol will be left to a future specification. Relevant header definitions and error codes are present to provide an indication of what a client may encounter. #### Future There are features that have been discussed during the process of cutting this specification. The following is an incomplete list: - Immutable image references - Multiple architecture support - Migration from v2compatibility representation These may represent features that are either out of the scope of this specification, the purview of another specification or have been deferred to a future version. ### Use Cases For the most part, the use cases of the former registry API apply to the new version. Differentiating use cases are covered below. #### Image Verification A docker engine instance would like to run verified image named "library/ubuntu", with the tag "latest". The engine contacts the registry, requesting the manifest for "library/ubuntu:latest". An untrusted registry returns a manifest. Before proceeding to download the individual layers, the engine verifies the manifest's signature, ensuring that the content was produced from a trusted source and no tampering has occured. After each layer is downloaded, the engine verifies the digest of the layer, ensuring that the content matches that specified by the manifest. #### Resumable Push Company X's build servers lose connectivity to docker registry before completing an image layer transfer. After connectivity returns, the build server attempts to re-upload the image. The registry notifies the build server that the upload has already been partially attempted. The build server responds by only sending the remaining data to complete the image file. #### Resumable Pull Company X is having more connectivity problems but this time in their deployment datacenter. When downloading an image, the connection is interrupted before completion. The client keeps the partial data and uses http `Range` requests to avoid downloading repeated data. #### Layer Upload De-duplication Company Y's build system creates two identical docker layers from build processes A and B. Build process A completes uploading the layer before B. When process B attempts to upload the layer, the registry indicates that its not necessary because the layer is already known. If process A and B upload the same layer at the same time, both operations will proceed and the first to complete will be stored in the registry (Note: we may modify this to prevent dogpile with some locking mechanism). ### Changes The V2 specification has been written to work as a living document, specifying only what is certain and leaving what is not specified open or to future changes. Only non-conflicting additions should be made to the API and accepted changes should avoid preventing future changes from happening. This section should be updated when changes are made to the specification, indicating what is different. Optionally, we may start marking parts of the specification to correspond with the versions enumerated here.
2.0.1
  • Added support for immutable manifest references in manifest endpoints.
  • Deleting a manifest by tag has been deprecated.
  • Specified `Docker-Content-Digest` header for appropriate entities.
  • Added error code for unsupported operations.
2.0
This is the baseline specification.
## Overview This section covers client flows and details of the API endpoints. The URI layout of the new API is structured to support a rich authentication and authorization model by leveraging namespaces. All endpoints will be prefixed by the API version and the repository name: /v2// For example, an API endpoint that will work with the `library/ubuntu` repository, the URI prefix will be: /v2/library/ubuntu/ This scheme provides rich access control over various operations and methods using the URI prefix and http methods that can be controlled in variety of ways. Classically, repository names have always been two path components where each path component is less than 30 characters. The V2 registry API does not enforce this. The rules for a repository name are as follows: 1. A repository name is broken up into _path components_. A component of a repository name must be at least two lowercase, alpha-numeric characters, optionally separated by periods, dashes or underscores. More strictly, it must match the regular expression `[a-z0-9]+(?:[._-][a-z0-9]+)*` and the matched result must be 2 or more characters in length. 2. The name of a repository must have at least two path components, separated by a forward slash. 3. The total length of a repository name, including slashes, must be less the 256 characters. These name requirements _only_ apply to the registry API and should accept a superset of what is supported by other docker ecosystem components. All endpoints should support aggressive http caching, compression and range headers, where appropriate. The new API attempts to leverage HTTP semantics where possible but may break from standards to implement targeted features. For detail on individual endpoints, please see the [_Detail_](#detail) section. ### Errors Actionable failure conditions, covered in detail in their relevant sections, are reported as part of 4xx responses, in a json response body. One or more errors will be returned in the following format: { "errors:" [{ "code": , "message": , "detail": }, ... ] } The `code` field will be a unique identifier, all caps with underscores by convention. The `message` field will be a human readable string. The optional `detail` field may contain arbitrary json data providing information the client can use to resolve the issue. While the client can take action on certain error codes, the registry may add new error codes over time. All client implementations should treat unknown error codes as `UNKNOWN`, allowing future error codes to be added without breaking API compatibility. For the purposes of the specification error codes will only be added and never removed. For a complete account of all error codes, please see the _Detail_ section. ### API Version Check A minimal endpoint, mounted at `/v2/` will provide version support information based on its response statuses. The request format is as follows: GET /v2/ If a `200 OK` response is returned, the registry implements the V2(.1) registry API and the client may proceed safely with other V2 operations. Optionally, the response may contain information about the supported paths in the response body. The client should be prepared to ignore this data. If a `401 Unauthorized` response is returned, the client should take action based on the contents of the "WWW-Authenticate" header and try the endpoint again. Depending on access control setup, the client may still have to authenticate against different resources, even if this check succeeds. If `404 Not Found` response status, or other unexpected status, is returned, the client should proceed with the assumption that the registry does not implement V2 of the API. ### Pulling An Image An "image" is a combination of a JSON manifest and individual layer files. The process of pulling an image centers around retrieving these two components. The first step in pulling an image is to retrieve the manifest. For reference, the relevant manifest fields for the registry are the following: field | description | ----------|------------------------------------------------| name | The name of the image. | tag | The tag for this version of the image. | fsLayers | A list of layer descriptors (including tarsum) | signature | A JWS used to verify the manifest content | For more information about the manifest format, please see [docker/docker#8093](https://github.com/docker/docker/issues/8093). When the manifest is in hand, the client must verify the signature to ensure the names and layers are valid. Once confirmed, the client will then use the tarsums to download the individual layers. Layers are stored in as blobs in the V2 registry API, keyed by their tarsum digest. #### Pulling an Image Manifest The image manifest can be fetched with the following url: ``` GET /v2//manifests/ ``` The `name` and `reference` parameter identify the image and are required. The reference may include a tag or digest. A `404 Not Found` response will be returned if the image is unknown to the registry. If the image exists and the response is successful, the image manifest will be returned, with the following format (see docker/docker#8093 for details): { "name": , "tag": , "fsLayers": [ { "blobSum": }, ... ] ], "history": , "signature": } The client should verify the returned manifest signature for authenticity before fetching layers. #### Pulling a Layer Layers are stored in the blob portion of the registry, keyed by tarsum digest. Pulling a layer is carried out by a standard http request. The URL is as follows: GET /v2//blobs/ Access to a layer will be gated by the `name` of the repository but is identified uniquely in the registry by `tarsum`. The `tarsum` parameter is an opaque field, to be interpreted by the tarsum library. This endpoint may issue a 307 (302 for /blobs/uploads/ ``` The parameters of this request are the image namespace under which the layer will be linked. Responses to this request are covered below. ##### Existing Layers The existence of a layer can be checked via a `HEAD` request to the blob store API. The request should be formatted as follows: ``` HEAD /v2//blobs/ ``` If the layer with the tarsum specified in `digest` is available, a 200 OK response will be received, with no actual body content (this is according to http specification). The response will look as follows: ``` 200 OK Content-Length: Docker-Content-Digest: ``` When this response is received, the client can assume that the layer is already available in the registry under the given name and should take no further action to upload the layer. Note that the binary digests may differ for the existing registry layer, but the tarsums will be guaranteed to match. ##### Uploading the Layer If the POST request is successful, a `202 Accepted` response will be returned with the upload URL in the `Location` header: ``` 202 Accepted Location: /v2//blobs/uploads/ Range: bytes=0- Content-Length: 0 Docker-Upload-UUID: ``` The rest of the upload process can be carried out with the returned url, called the "Upload URL" from the `Location` header. All responses to the upload url, whether sending data or getting status, will be in this format. Though the URI format (`/v2//blobs/uploads/`) for the `Location` header is specified, clients should treat it as an opaque url and should never try to assemble the it. While the `uuid` parameter may be an actual UUID, this proposal imposes no constraints on the format and clients should never impose any. If clients need to correlate local upload state with remote upload state, the contents of the `Docker-Upload-UUID` header should be used. Such an id can be used to key the last used location header when implementing resumable uploads. ##### Upload Progress The progress and chunk coordination of the upload process will be coordinated through the `Range` header. While this is a non-standard use of the `Range` header, there are examples of [similar approaches](https://developers.google.com/youtube/v3/guides/using_resumable_upload_protocol) in APIs with heavy use. For an upload that just started, for an example with a 1000 byte layer file, the `Range` header would be as follows: ``` Range: bytes=0-0 ``` To get the status of an upload, issue a GET request to the upload URL: ``` GET /v2//blobs/uploads/ Host: ``` The response will be similar to the above, except will return 204 status: ``` 204 No Content Location: /v2//blobs/uploads/ Range: bytes=0- Docker-Upload-UUID: ``` Note that the HTTP `Range` header byte ranges are inclusive and that will be honored, even in non-standard use cases. ##### Monolithic Upload A monolithic upload is simply a chunked upload with a single chunk and may be favored by clients that would like to avoided the complexity of chunking. To carry out a "monolithic" upload, one can simply put the entire content blob to the provided URL: ``` PUT /v2//blobs/uploads/?digest=[&digest=sha256:] Content-Length: Content-Type: application/octet-stream ``` The "digest" parameter must be included with the PUT request. Please see the _Completed Upload_ section for details on the parameters and expected responses. Additionally, the download can be completed with a single `POST` request to the uploads endpoint, including the "size" and "digest" parameters: ``` POST /v2//blobs/uploads/?digest=[&digest=sha256:] Content-Length: Content-Type: application/octet-stream ``` On the registry service, this should allocate a download, accept and verify the data and return the same response as the final chunk of an upload. If the POST request fails collecting the data in any way, the registry should attempt to return an error response to the client with the `Location` header providing a place to continue the download. The single `POST` method is provided for convenience and most clients should implement `POST` + `PUT` to support reliable resume of uploads. ##### Chunked Upload To carry out an upload of a chunk, the client can specify a range header and only include that part of the layer file: ``` PATCH /v2//blobs/uploads/ Content-Length: Content-Range: - Content-Type: application/octet-stream ``` There is no enforcement on layer chunk splits other than that the server must receive them in order. The server may enforce a minimum chunk size. If the server cannot accept the chunk, a `416 Requested Range Not Satisfiable` response will be returned and will include a `Range` header indicating the current status: ``` 416 Requested Range Not Satisfiable Location: /v2//blobs/uploads/ Range: 0- Content-Length: 0 Docker-Upload-UUID: ``` If this response is received, the client should resume from the "last valid range" and upload the subsequent chunk. A 416 will be returned under the following conditions: - Invalid Content-Range header format - Out of order chunk: the range of the next chunk must start immediately after the "last valid range" from the previous response. When a chunk is accepted as part of the upload, a `202 Accepted` response will be returned, including a `Range` header with the current upload status: ``` 202 Accepted Location: /v2//blobs/uploads/ Range: bytes=0- Content-Length: 0 Docker-Upload-UUID: ``` ##### Completed Upload For an upload to be considered complete, the client must submit a `PUT` request on the upload endpoint with a digest parameter. If it is not provided, the download will not be considered complete. The format for the final chunk will be as follows: ``` PUT /v2//blob/uploads/?digest=[&digest=sha256:] Content-Length: Content-Range: - Content-Type: application/octet-stream ``` Optionally, if all chunks have already been uploaded, a `PUT` request with a `digest` parameter and zero-length body may be sent to complete and validated the upload. Multiple "digest" parameters may be provided with different digests. The server may verify none or all of them but _must_ notify the client if the content is rejected. When the last chunk is received and the layer has been validated, the client will receive a `201 Created` response: ``` 201 Created Location: /v2//blobs/ Content-Length: 0 Docker-Content-Digest: ``` The `Location` header will contain the registry URL to access the accepted layer file. The `Docker-Content-Digest` header returns the canonical digest of the uploaded blob which may differ from the provided digest. Most clients may ignore the value but if it is used, the client should verify the value against the uploaded blob data. ###### Digest Parameter The "digest" parameter is designed as an opaque parameter to support verification of a successful transfer. The initial version of the registry API will support a tarsum digest, in the standard tarsum format. For example, a HTTP URI parameter might be as follows: ``` tarsum.v1+sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b ``` Given this parameter, the registry will verify that the provided content does result in this tarsum. Optionally, the registry can support other other digest parameters for non-tarfile content stored as a layer. A regular hash digest might be specified as follows: ``` sha256:6c3c624b58dbbcd3c0dd82b4c53f04194d1247c6eebdaab7c610cf7d66709b3b ``` Such a parameter would be used to verify that the binary content (as opposed to the tar content) would be verified at the end of the upload process. For the initial version, registry servers are only required to support the tarsum format. ##### Canceling an Upload An upload can be cancelled by issuing a DELETE request to the upload endpoint. The format will be as follows: ``` DELETE /v2//blobs/uploads/ ``` After this request is issued, the upload uuid will no longer be valid and the registry server will dump all intermediate data. While uploads will time out if not completed, clients should issue this request if they encounter a fatal error but still have the ability to issue an http request. ##### Errors If an 502, 503 or 504 error is received, the client should assume that the download can proceed due to a temporary condition, honoring the appropriate retry mechanism. Other 5xx errors should be treated as terminal. If there is a problem with the upload, a 4xx error will be returned indicating the problem. After receiving a 4xx response (except 416, as called out above), the upload will be considered failed and the client should take appropriate action. Note that the upload url will not be available forever. If the upload uuid is unknown to the registry, a `404 Not Found` response will be returned and the client must restart the upload process. #### Pushing an Image Manifest Once all of the layers for an image are uploaded, the client can upload the image manifest. An image can be pushed using the following request format: PUT /v2//manifests/ { "name": , "tag": , "fsLayers": [ { "blobSum": }, ... ] ], "history": , "signature": , ... } The `name` and `reference` fields of the response body must match those specified in the URL. The `reference` field may be a "tag" or a "digest". If there is a problem with pushing the manifest, a relevant 4xx response will be returned with a JSON error message. Please see the _PUT Manifest section for details on possible error codes that may be returned. If one or more layers are unknown to the registry, `BLOB_UNKNOWN` errors are returned. The `detail` field of the error response will have a `digest` field identifying the missing blob, which will be a tarsum. An error is returned for each unknown blob. The response format is as follows: { "errors:" [{ "code": "BLOB_UNKNOWN", "message": "blob unknown to registry", "detail": { "digest": } }, ... ] } #### Listing Image Tags It may be necessary to list all of the tags under a given repository. The tags for an image repository can be retrieved with the following request: GET /v2//tags/list The response will be in the following format: 200 OK Content-Type: application/json { "name": , "tags": [ , ... ] } For repositories with a large number of tags, this response may be quite large, so care should be taken by the client when parsing the response to reduce copying. ### Deleting an Image An image may be deleted from the registry via its `name` and `reference`. A delete may be issued with the following request format: DELETE /v2//manifests/ For deletes, `reference` *must* be a digest or the delete will fail. If the image exists and has been successfully deleted, the following response will be issued: 202 Accepted Content-Length: None If the image had already been deleted or did not exist, a `404 Not Found` response will be issued instead. ## Detail > **Note**: This section is still under construction. For the purposes of > implementation, if any details below differ from the described request flows > above, the section below should be corrected. When they match, this note > should be removed. The behavior of the endpoints are covered in detail in this section, organized by route and entity. All aspects of the request and responses are covered, including headers, parameters and body formats. Examples of requests and their corresponding responses, with success and failure, are enumerated. > **Note**: The sections on endpoint detail are arranged with an example > request, a description of the request, followed by information about that > request. A list of methods and URIs are covered in the table below: |Method|Path|Entity|Description| -------|----|------|------------ {{range $route := .RouteDescriptors}}{{range $method := .Methods}}| {{$method.Method}} | `{{$route.Path|prettygorilla}}` | {{$route.Entity}} | {{$method.Description}} | {{end}}{{end}} The detail for each endpoint is covered in the following sections. ### Errors The error codes encountered via the API are enumerated in the following table: |Code|Message|Description| -------|----|------|------------ {{range $err := .ErrorDescriptors}} `{{$err.Value}}` | {{$err.Message}} | {{$err.Description|removenewlines}} {{end}} {{range $route := .RouteDescriptors}} ### {{.Entity}} {{.Description}} {{range $method := $route.Methods}} #### {{.Method}} {{$route.Entity}} {{.Description}} {{if .Requests}}{{range .Requests}}{{if .Name}} ##### {{.Name}}{{end}} ``` {{$method.Method}} {{$route.Path|prettygorilla}}{{if .QueryParameters}}?{{range .QueryParameters}}{{.Name}}={{.Format}}{{end}}{{end}}{{range .Headers}} {{.Name}}: {{.Format}}{{end}}{{if .Body.ContentType}} Content-Type: {{.Body.ContentType}}{{end}}{{if .Body.Format}} {{.Body.Format}}{{end}} ``` {{.Description}} {{if or .Headers .PathParameters .QueryParameters}} The following parameters should be specified on the request: |Name|Kind|Description| |----|----|-----------| {{range .Headers}}|`{{.Name}}`|header|{{.Description}}| {{end}}{{range .PathParameters}}|`{{.Name}}`|path|{{.Description}}| {{end}}{{range .QueryParameters}}|`{{.Name}}`|query|{{.Description}}| {{end}}{{end}} {{if .Successes}} {{range .Successes}} ###### On Success: {{if .Name}}{{.Name}}{{else}}{{.StatusCode | statustext}}{{end}} ``` {{.StatusCode}} {{.StatusCode | statustext}}{{range .Headers}} {{.Name}}: {{.Format}}{{end}}{{if .Body.ContentType}} Content-Type: {{.Body.ContentType}}{{end}}{{if .Body.Format}} {{.Body.Format}}{{end}} ``` {{.Description}} {{if .Headers}}The following headers will be returned with the response: |Name|Description| |----|-----------| {{range .Headers}}|`{{.Name}}`|{{.Description}}| {{end}}{{end}}{{end}}{{end}} {{if .Failures}} {{range .Failures}} ###### On Failure: {{if .Name}}{{.Name}}{{else}}{{.StatusCode | statustext}}{{end}} ``` {{.StatusCode}} {{.StatusCode | statustext}}{{range .Headers}} {{.Name}}: {{.Format}}{{end}}{{if .Body.ContentType}} Content-Type: {{.Body.ContentType}}{{end}}{{if .Body.Format}} {{.Body.Format}}{{end}} ``` {{.Description}} {{if .Headers}} The following headers will be returned on the response: |Name|Description| |----|-----------| {{range .Headers}}|`{{.Name}}`|{{.Description}}| {{end}}{{end}} {{if .ErrorCodes}} The error codes that may be included in the response body are enumerated below: |Code|Message|Description| -------|----|------|------------ {{range $err := .ErrorCodes}}| `{{$err}}` | {{$err.Descriptor.Message}} | {{$err.Descriptor.Description|removenewlines}} | {{end}} {{end}}{{end}}{{end}}{{end}}{{end}}{{end}} {{end}} distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/doc/deploying.md0000644000175000017500000000036712502424227023646 0ustar tianontianon# Deploying **TODO(stevvooe):** This should discuss various deployment scenarios for production-ready deployments. These may be backed by ready-made docker images but this should explain how they were created and what considerations were present.distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/doc/storagedriver/0000755000175000017500000000000012502424227024204 5ustar tianontianondistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/doc/storagedriver/filesystem.md0000644000175000017500000000051412502424227026712 0ustar tianontianonDocker-Registry Filesystem Storage Driver ========================================= An implementation of the `storagedriver.StorageDriver` interface which uses the local filesystem. ## Parameters `rootdirectory`: (optional) The root directory tree in which all registry files will be stored. Defaults to `/tmp/registry/storage`. distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/doc/storagedriver/azure.md0000644000175000017500000000140412502424227025653 0ustar tianontianon# Docker Registry Microsoft Azure Blob Storage Driver An implementation of the `storagedriver.StorageDriver` interface which uses [Microsoft Azure Blob Storage][azure-blob-storage] for object storage. ## Parameters The following parameters must be used to authenticate and configure the storage driver (case-sensitive): * `accountname`: Name of the Azure Storage Account. * `accountkey`: Primary or Secondary Key for the Storage Account. * `container`: Name of the root storage container in which all registry data will be stored. Must comply the storage container name [requirements][create-container-api]. [azure-blob-storage]: http://azure.microsoft.com/en-us/services/storage/ [create-container-api]: https://msdn.microsoft.com/en-us/library/azure/dd179468.aspxdistribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/doc/storagedriver/s3.md0000644000175000017500000000363312502424227025060 0ustar tianontianonDocker-Registry S3 Storage Driver ========================================= An implementation of the `storagedriver.StorageDriver` interface which uses Amazon S3 for object storage. ## Parameters `accesskey`: Your aws access key. `secretkey`: Your aws secret key. **Note** You can provide empty strings for your access and secret keys if you plan on running the driver on an ec2 instance and will handle authentication with the instance's credentials. `region`: The name of the aws region in which you would like to store objects (for example `us-east-1`). For a list of regions, you can look at http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-regions-availability-zones.html `bucket`: The name of your s3 bucket where you wish to store objects (needs to already be created prior to driver initialization). `encrypt`: (optional) Whether you would like your data encrypted on the server side (defaults to false if not specified). `secure`: (optional) Whether you would like to transfer data to the bucket over ssl or not. Defaults to true (meaning transfering over ssl) if not specified. Note that while setting this to false will improve performance, it is not recommended due to security concerns. `v4auth`: (optional) Whether you would like to use aws signature version 4 with your requests. This defaults to true if not specified (note that the eu-central-1 region does not work with version 2 signatures, so the driver will error out if initialized with this region and v4auth set to false) `chunksize`: (optional) The default part size for multipart uploads (performed by WriteStream) to s3. The default is 10 MB. Keep in mind that the minimum part size for s3 is 5MB. You might experience better performance for larger chunk sizes depending on the speed of your connection to s3. `rootdirectory`: (optional) The root directory tree in which all registry files will be stored. Defaults to the empty string (bucket root). distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/doc/storagedriver/inmemory.md0000644000175000017500000000050112502424227026361 0ustar tianontianonDocker-Registry In-Memory Storage Driver ========================================= An implementation of the `storagedriver.StorageDriver` interface which uses local memory for object storage. **IMPORTANT**: This storage driver *does not* persist data across runs, and primarily exists for testing. ## Parameters None distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/doc/notifications.md0000644000175000017500000000021112502424227024511 0ustar tianontianon# Notifications **TODO(stevvooe)** Cover use and deployment of webhook notifications. Link to description in architecture documentation.distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/doc/architecture.md0000644000175000017500000000022112502424227024323 0ustar tianontianon# Architecture **TODO(stevvooe):** Discuss the architecture of the registry, internally and externally, in a few different deployment scenarios.distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/doc/configuration.md0000644000175000017500000002624412502424227024525 0ustar tianontianon# Configuration Below is a comprehensive example of all possible configuration options for the registry. Some options are mutually exclusive, and each section is explained in more detail below, but this is a good starting point from which you may delete the sections you do not need to create your own configuration. A copy of this configuration can be found at config.sample.yml. ```yaml version: 0.1 loglevel: debug storage: filesystem: rootdirectory: /tmp/registry azure: accountname: accountname accountkey: base64encodedaccountkey container: containername s3: accesskey: awsaccesskey secretkey: awssecretkey region: us-west-1 bucket: bucketname encrypt: true secure: true v4auth: true chunksize: 32000 rootdirectory: /s3/object/name/prefix auth: silly: realm: silly-realm service: silly-service token: realm: token-realm service: token-service issuer: registry-token-issuer rootcertbundle: /root/certs/bundle middleware: registry: - name: ARegistryMiddleware options: foo: bar repository: - name: ARepositoryMiddleware options: foo: bar storage: - name: cloudfront options: baseurl: https://my.cloudfronted.domain.com/ privatekey: /path/to/pem keypairid: cloudfrontkeypairid duration: 3000 reporting: bugsnag: apikey: bugsnagapikey releasestage: bugsnagreleasestage endpoint: bugsnagendpoint newrelic: licensekey: newreliclicensekey name: newrelicname http: addr: localhost:5000 prefix: /my/nested/registry/ secret: asecretforlocaldevelopment tls: certificate: /path/to/x509/public key: /path/to/x509/private debug: addr: localhost:5001 notifications: endpoints: - name: alistener disabled: false url: https://my.listener.com/event headers: timeout: 500 threshold: 5 backoff: 1000 ``` N.B. In some instances a configuration option may be marked **optional** but contain child options marked as **required**. This indicates that a parent may be omitted with all its children, however, if the parent is included, the children marked **required** must be included. ## version ```yaml version: 0.1 ``` The version option is **required** and indicates the version of the configuration being used. It is expected to remain a top-level field, to allow for a consistent version check before parsing the remainder of the configuration file. N.B. The version of the registry software may be found at [/version/version.go](https://github.com/docker/distribution/blob/master/version/version.go) ## loglevel ```yaml loglevel: debug ``` The loglevel option is **required** and sets the sensitivity of logging output. Permitted values are: - ```error``` - ```warn``` - ```info``` - ```debug``` ## storage ```yaml storage: filesystem: rootdirectory: /tmp/registry azure: accountname: accountname accountkey: base64encodedaccountkey container: containername s3: accesskey: awsaccesskey secretkey: awssecretkey region: us-west-1 bucket: bucketname encrypt: true secure: true v4auth: true chunksize: 32000 rootdirectory: /s3/object/name/prefix ``` The storage option is **required** and defines which storage backend is in use. At the moment only one backend may be configured, an error is returned when the registry is started with more than one storage backend configured. The following backends may be configured, **all options for a given storage backend are required**: ### filesystem This storage backend uses the local disk to store registry files. It is ideal for development and may be appropriate for some small scale production applications. - rootdirectory: **Required** - This is the absolute path to directory in which the repository will store data. ### azure This storage backend uses Microsoft's Azure Storage platform. - accountname: **Required** - Azure account name - accountkey: **Required** - Azure account key - container: **Required** - Name of the Azure container into which data will be stored ### S3 This storage backend uses Amazon's Simple Storage Service (a.k.a. S3). - accesskey: **Required** - Your AWS Access Key - secretkey: **Required** - Your AWS Secret Key. - region: **Required** - The AWS region in which your bucket exists. For the moment, the Go AWS library in use does not use the newer DNS based bucket routing. - bucket: **Required** - The bucket name in which you want to store the registry's data. - encrypt: TODO: fill in description - secure: TODO: fill in description - v4auth: This indicates whether Version 4 of AWS's authentication should be used. Generally you will want to set this to true. - chunksize: TODO: fill in description - rootdirectory: **Optional** - This is a prefix that will be applied to all S3 keys to allow you to segment data in your bucket if necessary. ## auth ```yaml auth: silly: realm: silly-realm service: silly-service token: realm: token-realm service: token-service issuer: registry-token-issuer rootcertbundle: /root/certs/bundle ``` The auth option is **optional** as there are use cases (i.e. a mirror that only permits pulls) for which authentication may not be desired. There are currently 2 possible auth providers, "silly" and "token", only one auth provider may be configured at the moment: ### silly The "silly" auth is only for development purposes. It simply checks for the existence of the "Authorization" header in the HTTP request, with no regard for the value of the header. If the header does not exist, it will respond with a challenge response, echoing back the realm, service, and scope that access was denied for. The values of the ```realm``` and ```service``` options are used in authentication reponses, both options are **required** - realm: **Required** - The realm in which the registry server authenticates. - service: **Required** - The service being authenticated. ### token Token based authentication allows the authentication system to be decoupled from the registry. It is a well established authentication paradigm with a high degree of security. - realm: **Required** - The realm in which the registry server authenticates. - service: **Required** - The service being authenticated. - issuer: **Required** - The name of the token issuer. The issuer inserts this into the token so it must match the value configured for the issuer. - rootcertbundle: **Required** - The absolute path to the root certificate bundle containing the public part of the certificates that will be used to sign authentication tokens. For more information about Token based authentication configuration, see the [specification.](spec/auth/token.md) ## middleware The middleware option is **optional** and allows middlewares to be injected at named hook points. A requirement of all middlewares is that they implement the same interface as the object they're wrapping. This means a registry middleware must implement the `distribution.Registry` interface, repository middleware must implement `distribution.Respository`, and storage middleware must implement `driver.StorageDriver`. Currently only one middleware, cloudfront, a storage middleware, is included in the registry. ```yaml middleware: registry: - name: ARegistryMiddleware options: foo: bar repository: - name: ARepositoryMiddleware options: foo: bar storage: - name: cloudfront options: baseurl: https://my.cloudfronted.domain.com/ privatekey: /path/to/pem keypairid: cloudfrontkeypairid duration: 3000 ``` Each middleware entry has `name` and `options` entries. The `name` must correspond to the name under which the middleware registers itself. The `options` field is a map that details custom configuration required to initialize the middleware. It is treated as a map[string]interface{} and as such will support any interesting structures desired, leaving it up to the middleware initialization function to best determine how to handle the specific interpretation of the options. ### cloudfront - baseurl: **Required** - SCHEME://HOST[/PATH] at which Cloudfront is served. - privatekey: **Required** - Private Key for Cloudfront provided by AWS - keypairid: **Required** - Key Pair ID provided by AWS - duration: **Optional** - Duration for which a signed URL should be valid ## reporting ```yaml reporting: bugsnag: apikey: bugsnagapikey releasestage: bugsnagreleasestage endpoint: bugsnagendpoint newrelic: licensekey: newreliclicensekey name: newrelicname ``` The reporting option is **optional** and configures error and metrics reporting tools. At the moment only two services are supported, New Relic and Bugsnag, a valid configuration may contain both. ### bugsnag - apikey: **Required** - API Key provided by Bugsnag - releasestage: **Optional** - TODO: fill in description - endpoint: **Optional** - TODO: fill in description ### newrelic - licensekey: **Required** - License key provided by New Relic - name: **Optional** - New Relic application name ## http ```yaml http: addr: localhost:5000 prefix: /my/nested/registry/ secret: asecretforlocaldevelopment tls: certificate: /path/to/x509/public key: /path/to/x509/private debug: addr: localhost:5001 ``` The http option details the configuration for the HTTP server that hosts the registry. - addr: **Required** - The HOST:PORT for which the server should accept connections. - prefix: **Optional** - If the server will not run at the root path, this should specify the prefix (the part of the path before ```v2```). It should have both preceding and trailing slashes. - secret: A random piece of data. It is used to sign state that may be stored with the client to protect against tampering. For production use you should generate a random piece of data using a cryptographically secure random generator. ### tls The tls option within http is **optional** and allows you to configure SSL for the server. If you already have a server such as Nginx or Apache running on the same host as the registry, you may prefer to configure SSL termination there and proxy connections to the registry server. - certificate: **Required** - Absolute path to x509 cert file - key: **Required** - Absolute path to x509 private key file ### debug The debug option is **optional** and allows you to configure a debug server that can be helpful in diagnosing problems. It is of most use to contributers to the distribution repository and should generally be disabled in production deployments. - addr: **Required** - The HOST:PORT on which the debug server should accept connections. ## notifications ```yaml notifications: endpoints: - name: alistener disabled: false url: https://my.listener.com/event headers: timeout: 500 threshold: 5 backoff: 1000 ``` The notifications option is **optional** and currently may contain a single option, ```endpoints```. ### endpoints Endpoints is a list of named services (URLs) that can accept event notifications. - name: **Required** - A human readable name for the service. - disabled: **Optional** - A boolean to enable/disable notifications for a service. - url: **Required** - The URL to which events should be published. - headers: **Required** - TODO: fill in description - timeout: **Required** - TODO: fill in description - threshold: **Required** - TODO: fill in description - backoff: **Required** - TODO: fill in description distribution-d957768537c5af40e4f4cd96871f7b2bde9e2923/doc/glossary.md0000644000175000017500000000203612502424227023512 0ustar tianontianon# Glossary **TODO(stevvooe):** Define and describe distribution related terms. Ideally, we reference back to the actual documentation and specifications where appropriate. **TODO(stevvooe):** The following list is a start but woefully incomplete.
Blob
The primary unit of registry storage. A string of bytes identified by content-address, known as a _digest_.
Image
An image is a collection of content from which a docker container can be created.
Layer
A tar file representing the partial content of a filesystem. Several layers can be "stacked" to make up the root filesystem.
Manifest
Describes a collection layers that make up an image.
Registry
A registry is a collection of repositories.
Repository
A repository is a collection of docker images, made up of manifests, tags and layers. The base unit of these components are blobs.
Tag
Tag provides a common name to an image.