diff --git a/loop.go b/loop.go new file mode 100644 index 0000000..b55a7df --- /dev/null +++ b/loop.go @@ -0,0 +1,154 @@ +// Copyright 2015 Canonical Ltd. +// Licensed under the LGPLv3, see LICENCE file for details. + +package retry + +import ( + "time" + + "github.com/juju/errors" +) + +// LoopSpec is a simple structure used to define the behaviour of the Loop +// iterator. +type LoopSpec struct { + // Attempts specifies the number of times Func should be retried before + // giving up and returning the `AttemptsExceeded` error. If a negative + // value is specified, the `Call` will retry forever. + Attempts int + + // Delay specifies how long to wait between retries. + Delay time.Duration + + // MaxDelay specifies how longest time to wait between retries. If no + // value is specified there is no maximum delay. + MaxDelay time.Duration + + // MaxDuration specifies the maximum time the `Call` function should spend + // iterating over `Func`. The duration is calculated from the start of the + // `Call` function. If the next delay time would take the total duration + // of the call over MaxDuration, then a DurationExceeded error is + // returned. If no value is specified, Call will continue until the number + // of attempts is complete. + MaxDuration time.Duration + + // BackoffFunc allows the caller to provide a function that alters the + // delay each time through the loop. If this function is not provided the + // delay is the same each iteration. Alternatively a function such as + // `retry.DoubleDelay` can be used that will provide an exponential + // backoff. The first time this function is called attempt is 1, the + // second time, attempt is 2 and so on. + BackoffFunc func(delay time.Duration, attempt int) time.Duration + + // Clock provides the mechanism for waiting. Normal program execution is + // expected to use something like clock.WallClock, and tests can override + // this to not actually sleep in tests. + Clock Clock + + // Stop is a channel that can be used to indicate that the waiting should + // be interrupted. If Stop is nil, then the Call function cannot be interrupted. + // If the channel is closed prior to the Call function being executed, the + // Func is still attempted once. + Stop <-chan struct{} +} + +// BackoffFactor returns a new LoopSpec with the backoff function set to +// scale the backoff each time by the factor specified. This is an example +// of the syntactic sugar that could be applied to the spec structures. +func (spec LoopSpec) BackoffFactor(factor int) LoopSpec { + spec.BackoffFunc = func(delay time.Duration, attempt int) time.Duration { + if attempt == 1 { + return delay + } + return delay * time.Duration(factor) + } + return spec +} + +// Validate the values are valid. The ensures that there are enough values +// set in the spec for valid iteration. +func (args *LoopSpec) Validate() error { + if args.Delay == 0 { + return errors.NotValidf("missing Delay") + } + if args.Clock == nil { + return errors.NotValidf("missing Clock") + } + // One of Attempts or MaxDuration need to be specified + if args.Attempts == 0 && args.MaxDuration == 0 && args.Stop == nil { + return errors.NotValidf("missing all of Attempts, MaxDuration or Stop") + } + return nil +} + +// Loop returns a new loop iterator. +func Loop(spec LoopSpec) *Iterator { + return &Iterator{spec: spec} +} + +// Iterator provides the abstaction around the looping and delays. +type Iterator struct { + err error + count int + start time.Time + spec LoopSpec +} + +// Error returns the error from the Next calls. If the spec validate fails, +// that is the error that is returned, otherwise it is one of the loop termination +// errors, timeout, stopped, or attempts exceeded. +func (i *Iterator) Error() error { + return i.err +} + +// Count returns the current iteration if called from within the loop, or the number of +// times the loop was executed if called outside the loop. +func (i *Iterator) Count() int { + return i.count +} + +// Next executes the validation and delay aspects of the loop. +func (i *Iterator) Next(err error) bool { + if i.count == 0 { + i.err = i.spec.Validate() + if i.err == nil { + i.count++ + i.start = i.spec.Clock.Now() + } + return i.err == nil + } + + // Could theoretically add an IsFatal error test here... + if err == nil { + // Loop has finished successfully. + return false + } + if i.spec.Attempts > 0 && i.count >= i.spec.Attempts { + i.err = errors.Wrap(err, &attemptsExceeded{err}) + return false + } + + if i.spec.BackoffFunc != nil { + delay := i.spec.BackoffFunc(i.spec.Delay, i.count) + if delay > i.spec.MaxDelay && i.spec.MaxDelay > 0 { + delay = i.spec.MaxDelay + } + i.spec.Delay = delay + } + + elapsedTime := i.spec.Clock.Now().Sub(i.start) + if i.spec.MaxDuration > 0 && (elapsedTime+i.spec.Delay) > i.spec.MaxDuration { + i.err = errors.Wrap(err, &durationExceeded{err}) + return false + } + + // Wait for the delay, and retry + select { + case <-i.spec.Clock.After(i.spec.Delay): + case <-i.spec.Stop: + i.err = errors.Wrap(err, &retryStopped{err}) + return false + } + i.count++ + return true +} diff --git a/loop_test.go b/loop_test.go new file mode 100644 index 0000000..a77bbc5 --- /dev/null +++ b/loop_test.go @@ -0,0 +1,333 @@ +// Copyright 2015 Canonical Ltd. +// Licensed under the LGPLv3, see LICENCE file for details. + +package retry_test + +import ( + "time" + + "github.com/juju/errors" + "github.com/juju/testing" + jc "github.com/juju/testing/checkers" + "github.com/juju/utils/clock" + gc "gopkg.in/check.v1" + + "github.com/juju/retry" +) + +type loopSuite struct { + testing.LoggingSuite + clock *mockClock + spec retry.LoopSpec +} + +var _ = gc.Suite(&loopSuite{}) + +func (s *loopSuite) SetUpTest(c *gc.C) { + s.LoggingSuite.SetUpTest(c) + s.clock = &mockClock{} + s.spec = retry.LoopSpec{ + Attempts: 5, + Delay: time.Minute, + Clock: s.clock, + } +} + +func success() error { + return nil +} + +func failure() error { + return errors.New("bah") +} + +type willSucceed struct { + when int + count int +} + +func (w *willSucceed) maybe() error { + w.count++ + if w.count > w.when { + return nil + } + return errors.New("bah") +} + +func (s *loopSuite) TestSimpleUsage(c *gc.C) { + var err error + what := willSucceed{when: 3} + for loop := retry.Loop(s.spec); loop.Next(err); { + err = what.maybe() + } + c.Assert(err, jc.ErrorIsNil) + c.Assert(s.clock.delays, gc.HasLen, 3) +} + +func (s *loopSuite) TestMulitipleLoops(c *gc.C) { + var err error + what := willSucceed{when: 3} + for loop := retry.Loop(s.spec); loop.Next(err); { + err = what.maybe() + } + c.Assert(err, jc.ErrorIsNil) + + what = willSucceed{when: 3} + for loop := retry.Loop(s.spec); loop.Next(err); { + err = what.maybe() + } + c.Assert(err, jc.ErrorIsNil) + + c.Assert(s.clock.delays, gc.HasLen, 6) +} + +func (s *loopSuite) TestSuccessHasNoDelay(c *gc.C) { + var err error + called := false + loop := retry.Loop(s.spec) + for loop.Next(err) { + err = success() + called = true + } + c.Assert(loop.Error(), jc.ErrorIsNil) + c.Assert(loop.Count(), gc.Equals, 1) + c.Assert(called, jc.IsTrue) + c.Assert(err, jc.ErrorIsNil) + c.Assert(s.clock.delays, gc.HasLen, 0) +} + +func (s *loopSuite) TestCalledOnceEvenIfStopped(c *gc.C) { + stop := make(chan struct{}) + called := false + close(stop) + + s.spec.Stop = stop + var err error + + loop := retry.Loop(s.spec) + for loop.Next(err) { + called = true + err = success() + } + + c.Assert(loop.Error(), jc.ErrorIsNil) + c.Assert(loop.Count(), gc.Equals, 1) + c.Assert(called, jc.IsTrue) + c.Assert(err, jc.ErrorIsNil) + c.Assert(s.clock.delays, gc.HasLen, 0) +} + +func (s *loopSuite) TestAttempts(c *gc.C) { + loop := retry.Loop(s.spec) + var err error + for loop.Next(err) { + err = failure() + } + + c.Assert(err, gc.ErrorMatches, "bah") + c.Assert(loop.Error(), gc.ErrorMatches, `attempt count exceeded: bah`) + c.Assert(loop.Error(), jc.Satisfies, retry.IsAttemptsExceeded) + c.Assert(retry.LastError(loop.Error()), gc.Equals, err) + c.Assert(loop.Count(), gc.Equals, 5) + // We delay between attempts, and don't delay after the last one. + c.Assert(s.clock.delays, jc.DeepEquals, []time.Duration{ + time.Minute, + time.Minute, + time.Minute, + time.Minute, + }) +} + +func (s *loopSuite) TestBackoffFactor(c *gc.C) { + loop := retry.Loop(s.spec.BackoffFactor(2)) + var err error + for loop.Next(err) { + err = failure() + } + + c.Assert(err, gc.ErrorMatches, "bah") + c.Assert(loop.Error(), gc.ErrorMatches, `attempt count exceeded: bah`) + c.Assert(loop.Error(), jc.Satisfies, retry.IsAttemptsExceeded) + c.Assert(retry.LastError(loop.Error()), gc.Equals, err) + c.Assert(loop.Count(), gc.Equals, 5) + c.Assert(s.clock.delays, jc.DeepEquals, []time.Duration{ + time.Minute, + time.Minute * 2, + time.Minute * 4, + time.Minute * 8, + }) +} + +func (s *loopSuite) TestStopChannel(c *gc.C) { + stop := make(chan struct{}) + s.spec.Stop = stop + loop := retry.Loop(s.spec) + var err error + for loop.Next(err) { + // Close the stop channel third time through. + if loop.Count() == 3 { + close(stop) + } + err = failure() + } + + c.Assert(loop.Error(), jc.Satisfies, retry.IsRetryStopped) + c.Assert(loop.Count(), gc.Equals, 3) + c.Assert(err, gc.ErrorMatches, "bah") + c.Assert(s.clock.delays, gc.HasLen, 3) +} + +func (s *loopSuite) TestInfiniteRetries(c *gc.C) { + // OK, we can't test infinite, but we'll go for lots. + stop := make(chan struct{}) + s.spec.Attempts = retry.UnlimitedAttempts + s.spec.Stop = stop + + loop := retry.Loop(s.spec) + var err error + for loop.Next(err) { + // Close the stop channel third time through. + if loop.Count() == 111 { + close(stop) + } + err = failure() + } + + c.Assert(loop.Error(), jc.Satisfies, retry.IsRetryStopped) + c.Assert(s.clock.delays, gc.HasLen, loop.Count()) +} + +func (s *loopSuite) TestMaxDuration(c *gc.C) { + spec := retry.LoopSpec{ + Delay: time.Minute, + MaxDuration: 5 * time.Minute, + Clock: s.clock, + } + loop := retry.Loop(spec) + var err error + for loop.Next(err) { + err = failure() + } + c.Assert(loop.Error(), jc.Satisfies, retry.IsDurationExceeded) + c.Assert(err, gc.ErrorMatches, "bah") + c.Assert(s.clock.delays, jc.DeepEquals, []time.Duration{ + time.Minute, + time.Minute, + time.Minute, + time.Minute, + time.Minute, + }) +} + +func (s *loopSuite) TestMaxDurationDoubling(c *gc.C) { + spec := retry.LoopSpec{ + Delay: time.Minute, + BackoffFunc: retry.DoubleDelay, + MaxDuration: 10 * time.Minute, + Clock: s.clock, + } + loop := retry.Loop(spec) + var err error + for loop.Next(err) { + err = failure() + } + + c.Assert(loop.Error(), jc.Satisfies, retry.IsDurationExceeded) + c.Assert(err, gc.ErrorMatches, "bah") + // Stops after seven minutes, because the next wait time + // would take it to 15 minutes. + c.Assert(s.clock.delays, jc.DeepEquals, []time.Duration{ + time.Minute, + 2 * time.Minute, + 4 * time.Minute, + }) +} + +func (s *loopSuite) TestMaxDelay(c *gc.C) { + spec := retry.LoopSpec{ + Attempts: 7, + Delay: time.Minute, + MaxDelay: 10 * time.Minute, + BackoffFunc: retry.DoubleDelay, + Clock: s.clock, + } + loop := retry.Loop(spec) + var err error + for loop.Next(err) { + err = failure() + } + + c.Assert(loop.Error(), jc.Satisfies, retry.IsAttemptsExceeded) + c.Assert(err, gc.ErrorMatches, "bah") + c.Assert(s.clock.delays, jc.DeepEquals, []time.Duration{ + time.Minute, + 2 * time.Minute, + 4 * time.Minute, + 8 * time.Minute, + 10 * time.Minute, + 10 * time.Minute, + }) +} + +func (s *loopSuite) TestWithWallClock(c *gc.C) { + var attempts []int + + s.spec.Clock = clock.WallClock + s.spec.Delay = time.Millisecond + + loop := retry.Loop(s.spec) + var err error + for loop.Next(err) { + attempts = append(attempts, loop.Count()) + err = failure() + } + + c.Assert(loop.Error(), jc.Satisfies, retry.IsAttemptsExceeded) + c.Assert(err, gc.ErrorMatches, "bah") + c.Assert(attempts, jc.DeepEquals, []int{1, 2, 3, 4, 5}) +} + +func (s *loopSuite) TestNextCallsValidate(c *gc.C) { + spec := retry.LoopSpec{ + Delay: time.Minute, + Clock: s.clock, + } + called := false + loop := retry.Loop(spec) + for loop.Next(nil) { + called = true + } + + c.Assert(called, jc.IsFalse) + c.Assert(loop.Error(), jc.Satisfies, errors.IsNotValid) +} + +func (*loopSuite) TestMissingAttemptsNotValid(c *gc.C) { + spec := retry.LoopSpec{ + Delay: time.Minute, + Clock: clock.WallClock, + } + err := spec.Validate() + c.Check(err, jc.Satisfies, errors.IsNotValid) + c.Check(err, gc.ErrorMatches, `missing all of Attempts, MaxDuration or Stop not valid`) +} + +func (*loopSuite) TestMissingDelayNotValid(c *gc.C) { + spec := retry.LoopSpec{ + Attempts: 5, + Clock: clock.WallClock, + } + err := spec.Validate() + c.Check(err, jc.Satisfies, errors.IsNotValid) + c.Check(err, gc.ErrorMatches, `missing Delay not valid`) +} + +func (*loopSuite) TestMissingClockNotValid(c *gc.C) { + spec := retry.LoopSpec{ + Attempts: 5, + Delay: time.Minute, + } + err := spec.Validate() + c.Check(err, jc.Satisfies, errors.IsNotValid) + c.Check(err, gc.ErrorMatches, `missing Clock not valid`) +} diff --git a/retry.go b/retry.go index a60b4a4..70b6d63 100644 --- a/retry.go +++ b/retry.go @@ -163,8 +163,8 @@ func (args *CallArgs) Validate() error { return errors.NotValidf("missing Clock") } // One of Attempts or MaxDuration need to be specified - if args.Attempts == 0 && args.MaxDuration == 0 { - return errors.NotValidf("missing Attempts or MaxDuration") + if args.Attempts == 0 && args.MaxDuration == 0 && args.Stop == nil { + return errors.NotValidf("missing all of Attempts, MaxDuration or Stop") } return nil } @@ -176,42 +176,32 @@ func Call(args CallArgs) error { if err != nil { return errors.Trace(err) } - start := args.Clock.Now() - for i := 1; args.Attempts <= 0 || i <= args.Attempts; i++ { + + spec := LoopSpec{ + Attempts: args.Attempts, + Delay: args.Delay, + MaxDelay: args.MaxDelay, + MaxDuration: args.MaxDuration, + BackoffFunc: args.BackoffFunc, + Clock: args.Clock, + Stop: args.Stop, + } + + loop := Loop(spec) + for loop.Next(err) { err = args.Func() - if err == nil { - return nil - } if args.IsFatalError != nil && args.IsFatalError(err) { - return errors.Trace(err) + break } if args.NotifyFunc != nil { - args.NotifyFunc(err, i) - } - if i == args.Attempts && args.Attempts > 0 { - break // don't wait before returning the error - } - - if args.BackoffFunc != nil { - delay := args.BackoffFunc(args.Delay, i) - if delay > args.MaxDelay && args.MaxDelay > 0 { - delay = args.MaxDelay - } - args.Delay = delay - } - elapsedTime := args.Clock.Now().Sub(start) - if args.MaxDuration > 0 && (elapsedTime+args.Delay) > args.MaxDuration { - return errors.Wrap(err, &durationExceeded{err}) + args.NotifyFunc(err, loop.Count()) } + } - // Wait for the delay, and retry - select { - case <-args.Clock.After(args.Delay): - case <-args.Stop: - return errors.Wrap(err, &retryStopped{err}) - } + if loop.Error() != nil { + return errors.Trace(loop.Error()) } - return errors.Wrap(err, &attemptsExceeded{err}) + return errors.Trace(err) } // DoubleDelay provides a simple function that doubles the duration passed in. diff --git a/retry_test.go b/retry_test.go index 31bc5da..daccbe4 100644 --- a/retry_test.go +++ b/retry_test.go @@ -290,7 +290,7 @@ func (*retrySuite) TestMissingAttemptsNotValid(c *gc.C) { Clock: clock.WallClock, }) c.Check(err, jc.Satisfies, errors.IsNotValid) - c.Check(err, gc.ErrorMatches, `missing Attempts or MaxDuration not valid`) + c.Check(err, gc.ErrorMatches, `missing all of Attempts, MaxDuration or Stop not valid`) } func (*retrySuite) TestMissingDelayNotValid(c *gc.C) {