pax_global_header00006660000000000000000000000064133512374420014516gustar00rootroot0000000000000052 comment=9e1f00a4fbfa5e15a154586d35fd16e9162a722e suture-3.0.0/000077500000000000000000000000001335123744200130455ustar00rootroot00000000000000suture-3.0.0/.travis.yml000066400000000000000000000001111335123744200151470ustar00rootroot00000000000000language: go go: - 1.1 - 1.2 - 1.3 - 1.4 - 1.5 - 1.6 - tip suture-3.0.0/LICENSE000066400000000000000000000020611335123744200140510ustar00rootroot00000000000000Copyright (c) 2014-2015 Barracuda Networks, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. suture-3.0.0/README.md000066400000000000000000000101311335123744200143200ustar00rootroot00000000000000Suture ====== [![Build Status](https://travis-ci.org/thejerf/suture.png?branch=master)](https://travis-ci.org/thejerf/suture) Suture provides Erlang-ish supervisor trees for Go. "Supervisor trees" -> "sutree" -> "suture" -> holds your code together when it's trying to die. This library has hit maturity, and isn't expected to be changed radically. This can also be imported via gopkg.in/thejerf/suture.v2 . It is intended to deal gracefully with the real failure cases that can occur with supervision trees (such as burning all your CPU time endlessly restarting dead services), while also making no unnecessary demands on the "service" code, and providing hooks to perform adequate logging with in a production environment. [A blog post describing the design decisions](http://www.jerf.org/iri/post/2930) is available. This module is fully covered with [godoc](http://godoc.org/github.com/thejerf/suture), including an example, usage, and everything else you might expect from a README.md on GitHub. (DRY.) Code Signing ------------ Starting with the commit after ac7cf8591b, I will be signing this repository with the ["jerf" keybase account](https://keybase.io/jerf). If you are viewing this repository through GitHub, you should see the commits as showing as "verified" in the commit view. (Bear in mind that due to the nature of how git commit signing works, there may be runs of unverified commits; what matters is that the top one is signed.) Aspiration ---------- One of the big wins the Erlang community has with their pervasive OTP support is that it makes it easy for them to distribute libraries that easily fit into the OTP paradigm. It ought to someday be considered a good idea to distribute libraries that provide some sort of supervisor tree functionality out of the box. It is possible to provide this functionality without explicitly depending on the Suture library. Changelog --------- suture uses semantic versioning. * 3.0: * Added a default jitter of up to 50% on the restart intervals. While this is a backwards-compatible change from a source perspective, this does represent a non-trivial behavior change. It should generally be a good thing, but this is released as a major version as a warning. * 2.0.4 * Added option PassThroughPanics, to allow panics to propagate up through the supervisor. * 2.0.3 * Accepted PR #23, making the logging functions in the supervisor public. * Added a new Supervisor method RemoveAndWait, allowing you to make a best effort way to wait for a service to terminate. * Accepted PR #24, adding an optional IsCompletable interface that Services can implement that indicates they do not need to be restarted upon a normal return. * 2.0.2 * Fixed issue #21. gccgo doesn't like `case (<-c)`, with the parentheses. Of course the parens aren't doing anything useful anyhow. No behavior changes. * 2.0.1 * __Test code change only__. Addresses the possibility that one of the tests can spuriously fail if they run in a certain order. * 2.0.0 * Major version due to change to the signature of the logging methods: A race condition could occur when the Supervisor rendered the service name via fmt.Sprintf("%#v"), because fmt examines the entire object regardless of locks through reflection. 2.0.0 changes the supervisors to snapshot the Service's name once, when it is added, and to pass it to the logging methods. * Removal of use of sync/atomic due to possible brokenness in the Debian architecture. * 1.1.2 * TravisCI showed that the fix for 1.1.1 induced a deadlock in Go 1.4 and before. * If the supervisor is terminated before a service, the service goroutine could be orphaned trying the shutdown notification to the supervisor. This should no longer occur. * 1.1.1 * Per #14, the fix in 1.1.0 did not actually wait for the Supervisor to stop. * 1.1.0 * Per #12, Supervisor.stop now tries to wait for its children before returning. A careful reading of the original .Stop() contract says this is the correct behavior. * 1.0.1 * Fixed data race on the .state variable. * 1.0.0 * Initial release. suture-3.0.0/complete_test.go000066400000000000000000000022111335123744200162370ustar00rootroot00000000000000package suture import ( "fmt" "testing" ) const ( JobLimit = 2 ) type IncrementorJob struct { current int next chan int stop chan bool } func (i *IncrementorJob) Stop() { fmt.Println("Stopping the service") i.stop <- true } func (i *IncrementorJob) Serve() { for { select { case i.next <- i.current + 1: i.current++ if i.current >= JobLimit { return } case <-i.stop: // We sync here just to guarantee the output of "Stopping the service", // so this passes the test reliably. // Most services would simply "return" here. i.stop <- true return } } } func (i *IncrementorJob) Complete() bool { // fmt.Println("IncrementorJob exited as Complete()") return i.current >= JobLimit } func TestCompleteJob(t *testing.T) { supervisor := NewSimple("Supervisor") service := &IncrementorJob{0, make(chan int), make(chan bool)} supervisor.Add(service) supervisor.ServeBackground() fmt.Println("Got:", <-service.next) fmt.Println("Got:", <-service.next) <-service.stop fmt.Println("IncrementorJob exited as Complete()") supervisor.Stop() // Output: // Got: 1 // Got: 2 // Stopping the service } suture-3.0.0/doc.go000066400000000000000000000037401335123744200141450ustar00rootroot00000000000000/* Package suture provides Erlang-like supervisor trees. This implements Erlang-esque supervisor trees, as adapted for Go. This is intended to be an industrial-strength implementation, but it has not yet been deployed in a hostile environment. (It's headed there, though.) Supervisor Tree -> SuTree -> suture -> holds your code together when it's trying to fall apart. Why use Suture? * You want to write bullet-resistant services that will remain available despite unforeseen failure. * You need the code to be smart enough not to consume 100% of the CPU restarting things. * You want to easily compose multiple such services in one program. * You want the Erlang programmers to stop lording their supervision trees over you. Suture has 100% test coverage, and is golint clean. This doesn't prove it free of bugs, but it shows I care. A blog post describing the design decisions is available at http://www.jerf.org/iri/post/2930 . Using Suture To idiomatically use Suture, create a Supervisor which is your top level "application" supervisor. This will often occur in your program's "main" function. Create "Service"s, which implement the Service interface. .Add() them to your Supervisor. Supervisors are also services, so you can create a tree structure here, depending on the exact combination of restarts you want to create. As a special case, when adding Supervisors to Supervisors, the "sub" supervisor will have the "super" supervisor's Log function copied. This allows you to set one log function on the "top" supervisor, and have it propagate down to all the sub-supervisors. This also allows libraries or modules to provide Supervisors without having to commit their users to a particular logging method. Finally, as what is probably the last line of your main() function, call .Serve() on your top level supervisor. This will start all the services you've defined. See the Example for an example, using a simple service that serves out incrementing integers. */ package suture suture-3.0.0/gml000077500000000000000000000027571335123744200135650ustar00rootroot00000000000000#!/bin/bash # This command wraps up the gometalinter invocation in the pre-commit hook # so it can be used by other things. # If used in a way that the "lintclean" file is in the current working # directory, the contents of the lintclean directory will be added to # this invocation, allowing you to filter out specific failures. # Rationale: # --exclude="composite literal uses unkeyed field" \ # jbowers: I disagree with community on this, and side with the Go # creators. Keyed fields are used when you expect new fields to be # unimportant to you, and you want to keep compiling, i.e., a new # option that, since you weren't using it before, probably want to # keep not using it. By contrast, unkeyed fields are appropriate # when you expect changes to the struct to really matter to you, # i.e., it is discovered that something MUST have a bool field added # or it turns out to be logically gibberish. You can't say that # one or the other must always be used... each has their place. # # -D gocyclo # jbowers: I consider cyclomatic complexity a bit of a crock. if [ `which gometalinter` == "" ]; then echo You need to run the \"install_buildtools\" script. exit 1 fi EXTRA_ARGS= if [ -e lintclean ]; then EXTRA_ARGS=$(cat lintclean) fi gometalinter \ --exclude="composite literal uses unkeyed field" \ -j 4 \ -D gocyclo \ -D aligncheck \ -D gofmt \ -D goimports \ -D gotype \ -D structcheck \ -D varcheck \ $EXTRA_ARGS \ $* suture-3.0.0/messages.go000066400000000000000000000030371335123744200152060ustar00rootroot00000000000000package suture // sum type pattern for type-safe message passing; see // http://www.jerf.org/iri/post/2917 type supervisorMessage interface { isSupervisorMessage() } type listServices struct { c chan []Service } func (ls listServices) isSupervisorMessage() {} type removeService struct { id serviceID notification chan struct{} } func (rs removeService) isSupervisorMessage() {} func (s *Supervisor) sync() { s.control <- syncSupervisor{} } type syncSupervisor struct { } func (ss syncSupervisor) isSupervisorMessage() {} func (s *Supervisor) fail(id serviceID, err interface{}, stacktrace []byte) { s.control <- serviceFailed{id, err, stacktrace} } type serviceFailed struct { id serviceID err interface{} stacktrace []byte } func (sf serviceFailed) isSupervisorMessage() {} func (s *Supervisor) serviceEnded(id serviceID, complete bool) { s.sendControl(serviceEnded{id, complete}) } type serviceEnded struct { id serviceID complete bool } func (s serviceEnded) isSupervisorMessage() {} // added by the Add() method type addService struct { service Service name string response chan serviceID } func (as addService) isSupervisorMessage() {} type stopSupervisor struct { done chan struct{} } func (ss stopSupervisor) isSupervisorMessage() {} func (s *Supervisor) panic() { s.control <- panicSupervisor{} } type serviceTerminated struct { id serviceID } func (st serviceTerminated) isSupervisorMessage() {} type panicSupervisor struct { } func (ps panicSupervisor) isSupervisorMessage() {} suture-3.0.0/pre-commit000077500000000000000000000002571335123744200150530ustar00rootroot00000000000000#!/bin/bash # This ensures all executables build and all tests pass before a commit # goes through. set -v set -e CWD=`pwd` go test ./gml . echo Build succeeds. exit 0 suture-3.0.0/service.go000066400000000000000000000063111335123744200150350ustar00rootroot00000000000000package suture /* Service is the interface that describes a service to a Supervisor. Serve Method The Serve method is called by a Supervisor to start the service. The service should execute within the goroutine that this is called in. If this function either returns or panics, the Supervisor will call it again. A Serve method SHOULD do as much cleanup of the state as possible, to prevent any corruption in the previous state from crashing the service again. Stop Method This method is used by the supervisor to stop the service. Calling this directly on a Service given to a Supervisor will simply result in the Service being restarted; use the Supervisor's .Remove(ServiceToken) method to stop a service. A supervisor will call .Stop() only once. Thus, it may be as destructive as it likes to get the service to stop. Once Stop has been called on a Service, the Service SHOULD NOT be reused in any other supervisor! Because of the impossibility of guaranteeing that the service has actually stopped in Go, you can't prove that you won't be starting two goroutines using the exact same memory to store state, causing completely unpredictable behavior. Stop should not return until the service has actually stopped. "Stopped" here is defined as "the service will stop servicing any further requests in the future". For instance, a common implementation is to receive a message on a dedicated "stop" channel and immediately returning. Once the stop command has been processed, the service is stopped. Another common Stop implementation is to forcibly close an open socket or other resource, which will cause detectable errors to manifest in the service code. Bear in mind that to perfectly correctly use this approach requires a bit more work to handle the chance of a Stop command coming in before the resource has been created. If a service does not Stop within the supervisor's timeout duration, a log entry will be made with a descriptive string to that effect. This does not guarantee that the service is hung; it may still get around to being properly stopped in the future. Until the service is fully stopped, both the service and the spawned goroutine trying to stop it will be "leaked". Stringer Interface When a Service is added to a Supervisor, the Supervisor will create a string representation of that service used for logging. If you implement the fmt.Stringer interface, that will be used. If you do not implement the fmt.Stringer interface, a default fmt.Sprintf("%#v") will be used. Optional Interface Services may optionally implement IsCompletable, which allows a service to indicate to a supervisor that it does not need to be restarted if it has terminated. */ type Service interface { Serve() Stop() } /* IsCompletable is an optionally-implementable interface that allows a service to report to a supervisor that it does not need to be restarted because it has terminated normally. When a Service is going to be restarted, the supervisor will check for this method, and if Complete returns true, the service is removed from the supervisor instead of restarted. This is only executed when the service is not running because it has terminated, and has not yet been restarted. */ type IsCompletable interface { Complete() bool } suture-3.0.0/supervisor.go000077500000000000000000000465471335123744200156400ustar00rootroot00000000000000package suture import ( "errors" "fmt" "log" "math" "math/rand" "runtime" "sync" "time" ) const ( notRunning = iota normal paused ) type supervisorID uint32 type serviceID uint32 type ( // BadStopLogger is called when a service fails to properly stop BadStopLogger func(*Supervisor, Service, string) // FailureLogger is called when a service fails FailureLogger func( supervisor *Supervisor, service Service, serviceName string, currentFailures float64, failureThreshold float64, restarting bool, error interface{}, stacktrace []byte, ) // BackoffLogger is called when the supervisor enters or exits backoff mode BackoffLogger func(s *Supervisor, entering bool) ) var currentSupervisorIDL sync.Mutex var currentSupervisorID uint32 // ErrWrongSupervisor is returned by the (*Supervisor).Remove method // if you pass a ServiceToken from the wrong Supervisor. var ErrWrongSupervisor = errors.New("wrong supervisor for this service token, no service removed") // ErrTimeout is returned when an attempt to RemoveAndWait for a service to // stop has timed out. var ErrTimeout = errors.New("waiting for service to stop has timed out") // ServiceToken is an opaque identifier that can be used to terminate a service that // has been Add()ed to a Supervisor. type ServiceToken struct { id uint64 } type serviceWithName struct { Service Service name string } // Jitter returns the sum of the input duration and a random jitter. It is // compatible with the jitter functions in github.com/lthibault/jitterbug. type Jitter interface { Jitter(time.Duration) time.Duration } // NoJitter does not apply any jitter to the input duration type NoJitter struct{} // Jitter leaves the input duration d unchanged. func (NoJitter) Jitter(d time.Duration) time.Duration { return d } // DefaultJitter is the jitter function that is applied when spec.BackoffJitter // is set to nil. type DefaultJitter struct { rand *rand.Rand } // Jitter will jitter the backoff time by uniformly distributing it into // the range [FailureBackoff, 1.5 * FailureBackoff). func (dj *DefaultJitter) Jitter(d time.Duration) time.Duration { // this is only called by the core supervisor loop, so it is // single-thread safe. if dj.rand == nil { dj.rand = rand.New(rand.NewSource(time.Now().UnixNano())) } jitter := dj.rand.Float64() / 2 return d + time.Duration(float64(d)*jitter) } /* Supervisor is the core type of the module that represents a Supervisor. Supervisors should be constructed either by New or NewSimple. Once constructed, a Supervisor should be started in one of three ways: 1. Calling .Serve(). 2. Calling .ServeBackground(). 3. Adding it to an existing Supervisor. Calling Serve will cause the supervisor to run until it is shut down by an external user calling Stop() on it. If that never happens, it simply runs forever. I suggest creating your services in Supervisors, then making a Serve() call on your top-level Supervisor be the last line of your main func. Calling ServeBackground will CORRECTLY start the supervisor running in a new goroutine. You do not want to just: go supervisor.Serve() because that will briefly create a race condition as it starts up, if you try to .Add() services immediately afterward. The various Log function should only be modified while the Supervisor is not running, to prevent race conditions. */ type Supervisor struct { Name string failureDecay float64 failureThreshold float64 failureBackoff time.Duration backoffJitter Jitter timeout time.Duration log func(string) services map[serviceID]serviceWithName servicesShuttingDown map[serviceID]serviceWithName lastFail time.Time failures float64 restartQueue []serviceID serviceCounter serviceID control chan supervisorMessage liveness chan struct{} resumeTimer <-chan time.Time LogBadStop BadStopLogger LogFailure FailureLogger LogBackoff BackoffLogger // avoid a dependency on github.com/thejerf/abtime by just implementing // a minimal chunk. getNow func() time.Time getAfterChan func(time.Duration) <-chan time.Time sync.Mutex // malign leftovers id supervisorID state uint8 recoverPanics bool } // Spec is used to pass arguments to the New function to create a // supervisor. See the New function for full documentation. type Spec struct { Log func(string) FailureDecay float64 FailureThreshold float64 FailureBackoff time.Duration BackoffJitter Jitter Timeout time.Duration LogBadStop BadStopLogger LogFailure FailureLogger LogBackoff BackoffLogger PassThroughPanics bool } /* New is the full constructor function for a supervisor. The name is a friendly human name for the supervisor, used in logging. Suture does not care if this is unique, but it is good for your sanity if it is. If not set, the following values are used: * Log: A function is created that uses log.Print. * FailureDecay: 30 seconds * FailureThreshold: 5 failures * FailureBackoff: 15 seconds * Timeout: 10 seconds * BackoffJitter: DefaultJitter The Log function will be called when errors occur. Suture will log the following: * When a service has failed, with a descriptive message about the current backoff status, and whether it was immediately restarted * When the supervisor has gone into its backoff mode, and when it exits it * When a service fails to stop The failureRate, failureThreshold, and failureBackoff controls how failures are handled, in order to avoid the supervisor failure case where the program does nothing but restarting failed services. If you do not care how failures behave, the default values should be fine for the vast majority of services, but if you want the details: The supervisor tracks the number of failures that have occurred, with an exponential decay on the count. Every FailureDecay seconds, the number of failures that have occurred is cut in half. (This is done smoothly with an exponential function.) When a failure occurs, the number of failures is incremented by one. When the number of failures passes the FailureThreshold, the entire service waits for FailureBackoff seconds before attempting any further restarts, at which point it resets its failure count to zero. Timeout is how long Suture will wait for a service to properly terminate. The PassThroughPanics options can be set to let panics in services propagate and crash the program, should this be desirable. */ func New(name string, spec Spec) (s *Supervisor) { s = new(Supervisor) s.Name = name currentSupervisorIDL.Lock() currentSupervisorID++ s.id = supervisorID(currentSupervisorID) currentSupervisorIDL.Unlock() if spec.Log == nil { s.log = func(msg string) { log.Print(fmt.Sprintf("Supervisor %s: %s", s.Name, msg)) } } else { s.log = spec.Log } if spec.FailureDecay == 0 { s.failureDecay = 30 } else { s.failureDecay = spec.FailureDecay } if spec.FailureThreshold == 0 { s.failureThreshold = 5 } else { s.failureThreshold = spec.FailureThreshold } if spec.FailureBackoff == 0 { s.failureBackoff = time.Second * 15 } else { s.failureBackoff = spec.FailureBackoff } if spec.BackoffJitter == nil { s.backoffJitter = &DefaultJitter{} } else { s.backoffJitter = spec.BackoffJitter } if spec.Timeout == 0 { s.timeout = time.Second * 10 } else { s.timeout = spec.Timeout } s.recoverPanics = !spec.PassThroughPanics // overriding these allows for testing the threshold behavior s.getNow = time.Now s.getAfterChan = time.After s.control = make(chan supervisorMessage) s.liveness = make(chan struct{}) s.services = make(map[serviceID]serviceWithName) s.servicesShuttingDown = make(map[serviceID]serviceWithName) s.restartQueue = make([]serviceID, 0, 1) s.resumeTimer = make(chan time.Time) // set up the default logging handlers if spec.LogBadStop == nil { s.LogBadStop = func(sup *Supervisor, _ Service, name string) { s.log(fmt.Sprintf( "%s: Service %s failed to terminate in a timely manner", sup.Name, name, )) } } else { s.LogBadStop = spec.LogBadStop } if spec.LogFailure == nil { s.LogFailure = func( sup *Supervisor, _ Service, svcName string, f float64, thresh float64, restarting bool, err interface{}, st []byte, ) { var errString string e, canError := err.(error) if canError { errString = e.Error() } else { errString = fmt.Sprintf("%#v", err) } s.log(fmt.Sprintf( "%s: Failed service '%s' (%f failures of %f), restarting: %#v, error: %s, stacktrace: %s", sup.Name, svcName, f, thresh, restarting, errString, string(st), )) } } else { s.LogFailure = spec.LogFailure } if spec.LogBackoff == nil { s.LogBackoff = func(s *Supervisor, entering bool) { if entering { s.log("Entering the backoff state.") } else { s.log("Exiting backoff state.") } } } else { s.LogBackoff = spec.LogBackoff } return } func serviceName(service Service) (serviceName string) { stringer, canStringer := service.(fmt.Stringer) if canStringer { serviceName = stringer.String() } else { serviceName = fmt.Sprintf("%#v", service) } return } // NewSimple is a convenience function to create a service with just a name // and the sensible defaults. func NewSimple(name string) *Supervisor { return New(name, Spec{}) } /* Add adds a service to this supervisor. If the supervisor is currently running, the service will be started immediately. If the supervisor is not currently running, the service will be started when the supervisor is. The returned ServiceID may be passed to the Remove method of the Supervisor to terminate the service. As a special behavior, if the service added is itself a supervisor, the supervisor being added will copy the Log function from the Supervisor it is being added to. This allows factoring out providing a Supervisor from its logging. This unconditionally overwrites the child Supervisor's logging functions. */ func (s *Supervisor) Add(service Service) ServiceToken { if s == nil { panic("can't add service to nil *suture.Supervisor") } if supervisor, isSupervisor := service.(*Supervisor); isSupervisor { supervisor.LogBadStop = s.LogBadStop supervisor.LogFailure = s.LogFailure supervisor.LogBackoff = s.LogBackoff } s.Lock() if s.state == notRunning { id := s.serviceCounter s.serviceCounter++ s.services[id] = serviceWithName{service, serviceName(service)} s.restartQueue = append(s.restartQueue, id) s.Unlock() return ServiceToken{uint64(s.id)<<32 | uint64(id)} } s.Unlock() response := make(chan serviceID) s.control <- addService{service, serviceName(service), response} return ServiceToken{uint64(s.id)<<32 | uint64(<-response)} } // ServeBackground starts running a supervisor in its own goroutine. This // method does not return until it is safe to use .Add() on the Supervisor. func (s *Supervisor) ServeBackground() { go s.Serve() s.sync() } /* Serve starts the supervisor. You should call this on the top-level supervisor, but nothing else. */ func (s *Supervisor) Serve() { if s == nil { panic("Can't serve with a nil *suture.Supervisor") } if s.id == 0 { panic("Can't call Serve on an incorrectly-constructed *suture.Supervisor") } defer func() { s.Lock() s.state = notRunning s.Unlock() }() s.Lock() if s.state != notRunning { s.Unlock() panic("Running a supervisor while it is already running?") } s.state = normal s.Unlock() // for all the services I currently know about, start them for _, id := range s.restartQueue { namedService, present := s.services[id] if present { s.runService(namedService.Service, id) } } s.restartQueue = make([]serviceID, 0, 1) for { select { case m := <-s.control: switch msg := m.(type) { case serviceFailed: s.handleFailedService(msg.id, msg.err, msg.stacktrace) case serviceEnded: service, monitored := s.services[msg.id] if monitored { if msg.complete { delete(s.services, msg.id) go func() { service.Service.Stop() }() } else { s.handleFailedService(msg.id, fmt.Sprintf("%s returned unexpectedly", service), []byte("[unknown stack trace]")) } } case addService: id := s.serviceCounter s.serviceCounter++ s.services[id] = serviceWithName{msg.service, msg.name} s.runService(msg.service, id) msg.response <- id case removeService: s.removeService(msg.id, msg.notification, s.control) case serviceTerminated: delete(s.servicesShuttingDown, msg.id) case stopSupervisor: s.stopSupervisor() msg.done <- struct{}{} return case listServices: services := []Service{} for _, service := range s.services { services = append(services, service.Service) } msg.c <- services case syncSupervisor: // this does nothing on purpose; its sole purpose is to // introduce a sync point via the channel receive case panicSupervisor: // used only by tests panic("Panicking as requested!") } case <-s.resumeTimer: // We're resuming normal operation after a pause due to // excessive thrashing // FIXME: Ought to permit some spacing of these functions, rather // than simply hammering through them s.Lock() s.state = normal s.Unlock() s.failures = 0 s.LogBackoff(s, false) for _, id := range s.restartQueue { namedService, present := s.services[id] if present { s.runService(namedService.Service, id) } } s.restartQueue = make([]serviceID, 0, 1) } } } // Stop stops the Supervisor. // // This function will not return until either all Services have stopped, or // they timeout after the timeout value given to the Supervisor at creation. func (s *Supervisor) Stop() { done := make(chan struct{}) if s.sendControl(stopSupervisor{done}) { <-done } } func (s *Supervisor) handleFailedService(id serviceID, err interface{}, stacktrace []byte) { now := s.getNow() if s.lastFail.IsZero() { s.lastFail = now s.failures = 1.0 } else { sinceLastFail := now.Sub(s.lastFail).Seconds() intervals := sinceLastFail / s.failureDecay s.failures = s.failures*math.Pow(.5, intervals) + 1 } if s.failures > s.failureThreshold { s.Lock() s.state = paused s.Unlock() s.LogBackoff(s, true) s.resumeTimer = s.getAfterChan(s.backoffJitter.Jitter(s.failureBackoff)) } s.lastFail = now failedService, monitored := s.services[id] // It is possible for a service to be no longer monitored // by the time we get here. In that case, just ignore it. if monitored { // this may look dangerous because the state could change, but this // code is only ever run in the one goroutine that is permitted to // change the state, so nothing else will. s.Lock() curState := s.state s.Unlock() if curState == normal { s.runService(failedService.Service, id) s.LogFailure(s, failedService.Service, failedService.name, s.failures, s.failureThreshold, true, err, stacktrace) } else { // FIXME: When restarting, check that the service still // exists (it may have been stopped in the meantime) s.restartQueue = append(s.restartQueue, id) s.LogFailure(s, failedService.Service, failedService.name, s.failures, s.failureThreshold, false, err, stacktrace) } } } func (s *Supervisor) runService(service Service, id serviceID) { go func() { if s.recoverPanics { defer func() { if r := recover(); r != nil { buf := make([]byte, 65535) written := runtime.Stack(buf, false) buf = buf[:written] s.fail(id, r, buf) } }() } service.Serve() complete := false if completable, ok := service.(IsCompletable); ok && completable.Complete() { complete = true } s.serviceEnded(id, complete) }() } func (s *Supervisor) removeService(id serviceID, notificationChan chan struct{}, removedChan chan supervisorMessage) { namedService, present := s.services[id] if present { delete(s.services, id) s.servicesShuttingDown[id] = namedService go func() { successChan := make(chan struct{}) go func() { namedService.Service.Stop() close(successChan) if notificationChan != nil { notificationChan <- struct{}{} } }() select { case <-successChan: // Life is good! case <-s.getAfterChan(s.timeout): s.LogBadStop(s, namedService.Service, namedService.name) } removedChan <- serviceTerminated{id} }() } else { if notificationChan != nil { notificationChan <- struct{}{} } } } func (s *Supervisor) stopSupervisor() { notifyDone := make(chan serviceID) for id := range s.services { namedService, present := s.services[id] if present { delete(s.services, id) s.servicesShuttingDown[id] = namedService go func(sID serviceID) { namedService.Service.Stop() notifyDone <- sID }(id) } } timeout := s.getAfterChan(s.timeout) SHUTTING_DOWN_SERVICES: for len(s.servicesShuttingDown) > 0 { select { case id := <-notifyDone: delete(s.servicesShuttingDown, id) case <-timeout: for _, namedService := range s.servicesShuttingDown { s.LogBadStop(s, namedService.Service, namedService.name) } // failed remove statements will log the errors. break SHUTTING_DOWN_SERVICES } } close(s.liveness) } // String implements the fmt.Stringer interface. func (s *Supervisor) String() string { return s.Name } func (s *Supervisor) sendControl(sm supervisorMessage) bool { select { case s.control <- sm: return true case <-s.liveness: return false } } /* Remove will remove the given service from the Supervisor, and attempt to Stop() it. The ServiceID token comes from the Add() call. This returns without waiting for the service to stop. */ func (s *Supervisor) Remove(id ServiceToken) error { sID := supervisorID(id.id >> 32) if sID != s.id { return ErrWrongSupervisor } // no meaningful error handling if this is false _ = s.sendControl(removeService{serviceID(id.id & 0xffffffff), nil}) return nil } /* RemoveAndWait will remove the given service from the Supervisor and attempt to Stop() it. It will wait up to the given timeout value for the service to terminate. A timeout value of 0 means to wait forever. If a nil error is returned from this function, then the service was terminated normally. If either the supervisor terminates or the timeout passes, ErrTimeout is returned. (If this isn't even the right supervisor ErrWrongSupervisor is returned.) */ func (s *Supervisor) RemoveAndWait(id ServiceToken, timeout time.Duration) error { sID := supervisorID(id.id >> 32) if sID != s.id { return ErrWrongSupervisor } var timeoutC <-chan time.Time if timeout > 0 { timer := time.NewTimer(timeout) defer timer.Stop() timeoutC = timer.C } notificationC := make(chan struct{}) sentControl := s.sendControl(removeService{serviceID(id.id & 0xffffffff), notificationC}) if !sentControl { return ErrTimeout } select { case <-notificationC: // normal case; the service is terminated. return nil // This occurs if the entire supervisor ends without the service // having terminated, and includes the timeout the supervisor // itself waited before closing the liveness channel. case <-s.liveness: return ErrTimeout // The local timeout. case <-timeoutC: return ErrTimeout } } /* Services returns a []Service containing a snapshot of the services this Supervisor is managing. */ func (s *Supervisor) Services() []Service { ls := listServices{make(chan []Service)} if s.sendControl(ls) { return <-ls.c } return nil } suture-3.0.0/suture_simple_test.go000066400000000000000000000016471335123744200173430ustar00rootroot00000000000000package suture import "fmt" type Incrementor struct { current int next chan int stop chan bool } func (i *Incrementor) Stop() { fmt.Println("Stopping the service") i.stop <- true } func (i *Incrementor) Serve() { for { select { case i.next <- i.current: i.current++ case <-i.stop: // We sync here just to guarantee the output of "Stopping the service", // so this passes the test reliably. // Most services would simply "return" here. i.stop <- true return } } } func ExampleNew_simple() { supervisor := NewSimple("Supervisor") service := &Incrementor{0, make(chan int), make(chan bool)} supervisor.Add(service) supervisor.ServeBackground() fmt.Println("Got:", <-service.next) fmt.Println("Got:", <-service.next) supervisor.Stop() // We sync here just to guarantee the output of "Stopping the service" <-service.stop // Output: // Got: 0 // Got: 1 // Stopping the service } suture-3.0.0/suture_test.go000077500000000000000000000421621335123744200157720ustar00rootroot00000000000000package suture import ( "errors" "fmt" "reflect" "sync" "testing" "time" ) const ( Happy = iota Fail Panic Hang UseStopChan ) var everMultistarted = false // Test that supervisors work perfectly when everything is hunky dory. func TestTheHappyCase(t *testing.T) { t.Parallel() s := NewSimple("A") if s.String() != "A" { t.Fatal("Can't get name from a supervisor") } service := NewService("B") s.Add(service) go s.Serve() <-service.started // If we stop the service, it just gets restarted service.Stop() <-service.started // And it is shut down when we stop the supervisor service.take <- UseStopChan s.Stop() <-service.stop } // Test that adding to a running supervisor does indeed start the service. func TestAddingToRunningSupervisor(t *testing.T) { t.Parallel() s := NewSimple("A1") s.ServeBackground() defer s.Stop() service := NewService("B1") s.Add(service) <-service.started services := s.Services() if !reflect.DeepEqual([]Service{service}, services) { t.Fatal("Can't get list of services as expected.") } } // Test what happens when services fail. func TestFailures(t *testing.T) { t.Parallel() s := NewSimple("A2") s.failureThreshold = 3.5 go s.Serve() defer func() { // to avoid deadlocks during shutdown, we have to not try to send // things out on channels while we're shutting down (this undoes the // LogFailure overide about 25 lines down) s.LogFailure = func(*Supervisor, Service, string, float64, float64, bool, interface{}, []byte) {} s.Stop() }() s.sync() service1 := NewService("B2") service2 := NewService("C2") s.Add(service1) <-service1.started s.Add(service2) <-service2.started nowFeeder := NewNowFeeder() pastVal := time.Unix(1000000, 0) nowFeeder.appendTimes(pastVal) s.getNow = nowFeeder.getter resumeChan := make(chan time.Time) s.getAfterChan = func(d time.Duration) <-chan time.Time { return resumeChan } failNotify := make(chan bool) // use this to synchronize on here s.LogFailure = func(supervisor *Supervisor, s Service, sn string, cf float64, ft float64, r bool, error interface{}, stacktrace []byte) { failNotify <- r } // All that setup was for this: Service1, please return now. service1.take <- Fail restarted := <-failNotify <-service1.started if !restarted || s.failures != 1 || s.lastFail != pastVal { t.Fatal("Did not fail in the expected manner") } // Getting past this means the service was restarted. service1.take <- Happy // Service2, your turn. service2.take <- Fail nowFeeder.appendTimes(pastVal) restarted = <-failNotify <-service2.started if !restarted || s.failures != 2 || s.lastFail != pastVal { t.Fatal("Did not fail in the expected manner") } // And you're back. (That is, the correct service was restarted.) service2.take <- Happy // Now, one failureDecay later, is everything working correctly? oneDecayLater := time.Unix(1000030, 0) nowFeeder.appendTimes(oneDecayLater) service2.take <- Fail restarted = <-failNotify <-service2.started // playing a bit fast and loose here with floating point, but... // we get 2 by taking the current failure value of 2, decaying it // by one interval, which cuts it in half to 1, then adding 1 again, // all of which "should" be precise if !restarted || s.failures != 2 || s.lastFail != oneDecayLater { t.Fatal("Did not decay properly", s.lastFail, oneDecayLater) } // For a change of pace, service1 would you be so kind as to panic? nowFeeder.appendTimes(oneDecayLater) service1.take <- Panic restarted = <-failNotify <-service1.started if !restarted || s.failures != 3 || s.lastFail != oneDecayLater { t.Fatal("Did not correctly recover from a panic") } nowFeeder.appendTimes(oneDecayLater) backingoff := make(chan bool) s.LogBackoff = func(s *Supervisor, backingOff bool) { backingoff <- backingOff } // And with this failure, we trigger the backoff code. service1.take <- Fail backoff := <-backingoff restarted = <-failNotify if !backoff || restarted || s.failures != 4 { t.Fatal("Broke past the threshold but did not log correctly", s.failures) } if service1.existing != 0 { t.Fatal("service1 still exists according to itself?") } // service2 is still running, because we don't shut anything down in a // backoff, we just stop restarting. service2.take <- Happy var correct bool timer := time.NewTimer(time.Millisecond * 10) // verify the service has not been restarted // hard to get around race conditions here without simply using a timer... select { case service1.take <- Happy: correct = false case <-timer.C: correct = true } if !correct { t.Fatal("Restarted the service during the backoff interval") } // tell the supervisor the restart interval has passed resumeChan <- time.Time{} backoff = <-backingoff <-service1.started s.sync() if s.failures != 0 { t.Fatal("Did not reset failure count after coming back from timeout.") } nowFeeder.appendTimes(oneDecayLater) service1.take <- Fail restarted = <-failNotify <-service1.started if !restarted || backoff { t.Fatal("For some reason, got that we were backing off again.", restarted, backoff) } } func TestRunningAlreadyRunning(t *testing.T) { t.Parallel() s := NewSimple("A3") go s.Serve() defer s.Stop() // ensure the supervisor has made it to its main loop s.sync() var errored bool func() { defer func() { if r := recover(); r != nil { errored = true } }() s.Serve() }() if !errored { t.Fatal("Supervisor failed to prevent itself from double-running.") } } func TestFullConstruction(t *testing.T) { t.Parallel() s := New("Moo", Spec{ Log: func(string) {}, FailureDecay: 1, FailureThreshold: 2, FailureBackoff: 3, Timeout: time.Second * 29, }) if s.String() != "Moo" || s.failureDecay != 1 || s.failureThreshold != 2 || s.failureBackoff != 3 || s.timeout != time.Second*29 { t.Fatal("Full construction failed somehow") } } // This is mostly for coverage testing. func TestDefaultLogging(t *testing.T) { t.Parallel() s := NewSimple("A4") service := NewService("B4") s.Add(service) s.failureThreshold = .5 s.failureBackoff = time.Millisecond * 25 go s.Serve() s.sync() <-service.started resumeChan := make(chan time.Time) s.getAfterChan = func(d time.Duration) <-chan time.Time { return resumeChan } service.take <- UseStopChan service.take <- Fail <-service.stop resumeChan <- time.Time{} <-service.started service.take <- Happy name := serviceName(&BarelyService{}) s.LogBadStop(s, service, name) s.LogFailure(s, service, name, 1, 1, true, errors.New("test error"), []byte{}) s.Stop() } func TestNestedSupervisors(t *testing.T) { t.Parallel() super1 := NewSimple("Top5") super2 := NewSimple("Nested5") service := NewService("Service5") super2.LogBadStop = func(*Supervisor, Service, string) { panic("Failed to copy LogBadStop") } super1.Add(super2) super2.Add(service) // test the functions got copied from super1; if this panics, it didn't // get copied super2.LogBadStop(super2, service, "Service5") go super1.Serve() super1.sync() <-service.started service.take <- Happy super1.Stop() } func TestStoppingSupervisorStopsServices(t *testing.T) { t.Parallel() s := NewSimple("Top6") service := NewService("Service 6") s.Add(service) go s.Serve() s.sync() <-service.started service.take <- UseStopChan s.Stop() <-service.stop if s.sendControl(syncSupervisor{}) { t.Fatal("supervisor is shut down, should be returning fals for sendControl") } if s.Services() != nil { t.Fatal("Non-running supervisor is returning services list") } } // This tests that even if a service is hung, the supervisor will stop. func TestStoppingStillWorksWithHungServices(t *testing.T) { t.Parallel() s := NewSimple("Top7") service := NewService("Service WillHang7") s.Add(service) go s.Serve() <-service.started service.take <- UseStopChan service.take <- Hang resumeChan := make(chan time.Time) s.getAfterChan = func(d time.Duration) <-chan time.Time { return resumeChan } failNotify := make(chan struct{}) s.LogBadStop = func(supervisor *Supervisor, s Service, name string) { failNotify <- struct{}{} } // stop the supervisor, then immediately call time on it go s.Stop() resumeChan <- time.Time{} <-failNotify service.release <- true <-service.stop } // This tests that even if a service is hung, the supervisor can still // remove it. func TestRemovingHungService(t *testing.T) { t.Parallel() s := NewSimple("TopHungService") failNotify := make(chan struct{}) resumeChan := make(chan time.Time) s.getAfterChan = func(d time.Duration) <-chan time.Time { return resumeChan } s.LogBadStop = func(supervisor *Supervisor, s Service, name string) { failNotify <- struct{}{} } service := NewService("Service WillHang") sToken := s.Add(service) go s.Serve() <-service.started service.take <- Hang s.Remove(sToken) resumeChan <- time.Time{} <-failNotify service.release <- true } func TestRemoveService(t *testing.T) { t.Parallel() s := NewSimple("Top") service := NewService("ServiceToRemove8") id := s.Add(service) go s.Serve() <-service.started service.take <- UseStopChan err := s.Remove(id) if err != nil { t.Fatal("Removing service somehow failed") } <-service.stop err = s.Remove(ServiceToken{id.id + (1 << 32)}) if err != ErrWrongSupervisor { t.Fatal("Did not detect that the ServiceToken was wrong") } err = s.RemoveAndWait(ServiceToken{id.id + (1 << 32)}, time.Second) if err != ErrWrongSupervisor { t.Fatal("Did not detect that the ServiceToken was wrong") } } func TestFailureToConstruct(t *testing.T) { t.Parallel() var s *Supervisor panics(func() { s.Serve() }) s = new(Supervisor) panics(func() { s.Serve() }) } func TestFailingSupervisors(t *testing.T) { t.Parallel() // This is a bit of a complicated test, so let me explain what // all this is doing: // 1. Set up a top-level supervisor with a hair-trigger backoff. // 2. Add a supervisor to that. // 3. To that supervisor, add a service. // 4. Panic the supervisor in the middle, sending the top-level into // backoff. // 5. Kill the lower level service too. // 6. Verify that when the top-level service comes out of backoff, // the service ends up restarted as expected. // Ultimately, we can't have more than a best-effort recovery here. // A panic'ed supervisor can't really be trusted to have consistent state, // and without *that*, we can't trust it to do anything sensible with // the children it may have been running. So unlike Erlang, we can't // can't really expect to be able to safely restart them or anything. // Really, the "correct" answer is that the Supervisor must never panic, // but in the event that it does, this verifies that it at least tries // to get on with life. // This also tests that if a Supervisor itself panics, and one of its // monitored services goes down in the meantime, that the monitored // service also gets correctly restarted when the supervisor does. s1 := NewSimple("Top9") s2 := NewSimple("Nested9") service := NewService("Service9") s1.Add(s2) s2.Add(service) go s1.Serve() <-service.started s1.failureThreshold = .5 // let us control precisely when s1 comes back resumeChan := make(chan time.Time) s1.getAfterChan = func(d time.Duration) <-chan time.Time { return resumeChan } failNotify := make(chan string) // use this to synchronize on here s1.LogFailure = func(supervisor *Supervisor, s Service, name string, cf float64, ft float64, r bool, error interface{}, stacktrace []byte) { failNotify <- fmt.Sprintf("%s", s) } s2.panic() failing := <-failNotify // that's enough sync to guarantee this: if failing != "Nested9" || s1.state != paused { t.Fatal("Top-level supervisor did not go into backoff as expected") } service.take <- Fail resumeChan <- time.Time{} <-service.started } func TestNilSupervisorAdd(t *testing.T) { t.Parallel() var s *Supervisor defer func() { if r := recover(); r == nil { t.Fatal("did not panic as expected on nil add") } }() s.Add(s) } // https://github.com/thejerf/suture/issues/11 // // The purpose of this test is to verify that it does not cause data races, // so there are no obvious assertions. func TestIssue11(t *testing.T) { t.Parallel() s := NewSimple("main") s.ServeBackground() subsuper := NewSimple("sub") s.Add(subsuper) subsuper.Add(NewService("may cause data race")) } func TestRemoveAndWait(t *testing.T) { t.Parallel() s := NewSimple("main") s.timeout = time.Second s.ServeBackground() service := NewService("A1") token := s.Add(service) <-service.started // Normal termination case; without the useStopChan flag on the // NewService, this will just terminate. So we can freely use a long // timeout, because it should not trigger. err := s.RemoveAndWait(token, time.Second) if err != nil { t.Fatal("Happy case for RemoveAndWait failed: " + err.Error()) } // Removing already-removed service does unblock the channel err = s.RemoveAndWait(token, time.Second) if err != nil { t.Fatal("Removing already-removed service failed: " + err.Error()) } service = NewService("A2") token = s.Add(service) <-service.started service.take <- Hang // Abnormal case; the service is hung until we release it err = s.RemoveAndWait(token, time.Millisecond) if err == nil { t.Fatal("RemoveAndWait unexpectedly returning that everything is fine") } if err != ErrTimeout { // laziness; one of the unhappy results is err == nil, which will // panic here, but, hey, that's a failing test, right? t.Fatal("Unexpected result for RemoveAndWait on frozen service: " + err.Error()) } // Abnormal case: The service is hung and we get the supervisor // stopping instead. service = NewService("A3") token = s.Add(service) <-service.started s.Stop() err = s.RemoveAndWait(token, 10*time.Millisecond) if err != ErrTimeout { t.Fatal("Unexpected result for RemoveAndWait on a stopped service: " + err.Error()) } // Abnormal case: The service takes long to terminate, which takes more than the timeout of the spec, but // if the service eventually terminates, this does not hang RemoveAndWait. s = NewSimple("main") s.timeout = time.Millisecond s.ServeBackground() service = NewService("A1") token = s.Add(service) <-service.started service.take <- Hang go func() { time.Sleep(10 * time.Millisecond) service.release <- true }() err = s.RemoveAndWait(token, 0) if err != nil { t.Fatal("Unexpected result of RemoveAndWait: " + err.Error()) } } func TestCoverage(t *testing.T) { New("testing coverage", Spec{ LogBadStop: func(*Supervisor, Service, string) {}, LogFailure: func( supervisor *Supervisor, service Service, serviceName string, currentFailures float64, failureThreshold float64, restarting bool, error interface{}, stacktrace []byte, ) { }, LogBackoff: func(s *Supervisor, entering bool) {}, }) } // http://golangtutorials.blogspot.com/2011/10/gotest-unit-testing-and-benchmarking-go.html // claims test function are run in the same order as the source file... // I'm not sure if this is part of the contract, though. Especially in the // face of "t.Parallel()"... func TestEverMultistarted(t *testing.T) { if everMultistarted { t.Fatal("Seem to have multistarted a service at some point, bummer.") } } // A test service that can be induced to fail, panic, or hang on demand. func NewService(name string) *FailableService { return &FailableService{name, make(chan bool), make(chan int), make(chan bool), make(chan bool), make(chan bool), 0} } type FailableService struct { name string started chan bool take chan int shutdown chan bool release chan bool stop chan bool existing int } func (s *FailableService) Serve() { if s.existing != 0 { everMultistarted = true panic("Multi-started the same service! " + s.name) } s.existing++ s.started <- true useStopChan := false for { select { case val := <-s.take: switch val { case Happy: // Do nothing on purpose. Life is good! case Fail: s.existing-- if useStopChan { s.stop <- true } return case Panic: s.existing-- panic("Panic!") case Hang: // or more specifically, "hang until I release you" <-s.release case UseStopChan: useStopChan = true } case <-s.shutdown: s.existing-- if useStopChan { s.stop <- true } return } } } func (s *FailableService) String() string { return s.name } func (s *FailableService) Stop() { s.shutdown <- true } type NowFeeder struct { values []time.Time getter func() time.Time m sync.Mutex } // This is used to test serviceName; it's a service without a Stringer. type BarelyService struct{} func (bs *BarelyService) Serve() {} func (bs *BarelyService) Stop() {} func NewNowFeeder() (nf *NowFeeder) { nf = new(NowFeeder) nf.getter = func() time.Time { nf.m.Lock() defer nf.m.Unlock() if len(nf.values) > 0 { ret := nf.values[0] nf.values = nf.values[1:] return ret } panic("Ran out of values for NowFeeder") } return } func (nf *NowFeeder) appendTimes(t ...time.Time) { nf.m.Lock() defer nf.m.Unlock() nf.values = append(nf.values, t...) } func panics(doesItPanic func()) (panics bool) { defer func() { if r := recover(); r != nil { panics = true } }() doesItPanic() return }