Add unit test for RetryOnFail

app/jobs/dispatch_comment.go

@@ -96,7 +96,7 @@ func (dispatcher *Dispatcher) Comment(comment models.Comment) {
     }
 
     // send and retry if it fails the first time
-    run.Now(Retry{
+    run.Now(RetryOnFail{
       Pod: &person.Pod,
       Send: func() error {
         return entity.Send(endpoint, priv, pub)

app/jobs/dispatch_contact.go

@@ -59,7 +59,7 @@ func (dispatcher *Dispatcher) Contact(contact models.AspectMembership) {
   entity.SetSharing(true)
 
   // send and retry if it fails the first time
-  run.Now(Retry{
+  run.Now(RetryOnFail{
     Pod: &person.Pod,
     Send: func() error {
       return entity.Send(endpoint, priv, pub)

app/jobs/dispatch_like.go

@@ -87,7 +87,7 @@ func (dispatcher *Dispatcher) Like(like models.Like) {
     }
 
     // send and retry if it fails the first time
-    run.Now(Retry{
+    run.Now(RetryOnFail{
       Pod: &person.Pod,
       Send: func() error {
         return entity.Send(endpoint, priv, pub)

app/jobs/dispatch_post.go

@@ -110,7 +110,7 @@ func (dispatcher *Dispatcher) StatusMessage(post models.Post) {
     }
 
     // send and retry if it fails the first time
-    run.Now(Retry{
+    run.Now(RetryOnFail{
       Pod: &person.Pod,
       Send: func() error {
         return entity.Send(endpoint, priv, pub)

app/jobs/relay_comment.go

@@ -118,7 +118,7 @@ func (dispatcher *Dispatcher) RelayComment(entity federation.MessageComment) {
         continue
       }
       // send and retry if it fails the first time
-      run.Now(Retry{
+      run.Now(RetryOnFail{
         Pod: &person.Pod,
         Send: func() error {
           return translate.Send(endpoint, priv, pub)
@@ -129,7 +129,7 @@ func (dispatcher *Dispatcher) RelayComment(entity federation.MessageComment) {
       entity.SetAuthor(parentUser.Person.Author)
 
       // send and retry if it fails the first time
-      run.Now(Retry{
+      run.Now(RetryOnFail{
         Pod: &person.Pod,
         Send: func() error {
           return entity.Send(endpoint, priv, pub)

app/jobs/relay_like.go

@@ -72,7 +72,7 @@ func (dispatcher *Dispatcher) RelayLike(entity federation.MessageLike) {
     entity.SetAuthor(parentUser.Person.Author)
 
     // send and retry if it fails the first time
-    run.Now(Retry{
+    run.Now(RetryOnFail{
       Pod: &person.Pod,
       Send: func() error {
         return entity.Send(endpoint, priv, pub)

app/jobs/relay_retraction.go

@@ -132,7 +132,7 @@ func (dispatcher *Dispatcher) RelayRetraction(entity federation.MessageRetract)
     }
 
     // send and retry if it fails the first time
-    run.Now(Retry{
+    run.Now(RetryOnFail{
       Pod: &person.Pod,
       Send: func() error {
         return entity.Send(endpoint, priv, pub)

app/jobs/retry.go → app/jobs/retry_on_fail.go

@@ -24,21 +24,35 @@ import (
   "time"
 )
 
-type Retry struct {
+type RetryOnFail struct {
   Pod *models.Pod
   Send func() error
-  wait time.Duration
+  After []time.Duration
+
+  firstRun bool
 }
 
-func (retry Retry) Run() {
+func (retry RetryOnFail) Run() {
+  // set default values on first run
+  if !retry.firstRun {
+    retry.firstRun = true
+    if len(retry.After) == 0 {
+      retry.After = append(retry.After, []time.Duration{
+        time.Minute, time.Hour, 24 * time.Hour,
+      }...)
+    }
+  }
+
+  // execute job and check on errors
   err := retry.Send()
   if err != nil {
-    if retry.wait == 0 {
-      retry.wait = time.Minute
-    } else if retry.wait == time.Minute {
-      retry.wait = time.Hour
-    } else if retry.wait == time.Hour {
-      retry.wait = 24 * time.Hour
+    if len(retry.After) > 0 {
+      // repeat until timeout (empty array)
+      duration := retry.After[0]
+      retry.After = retry.After[1:]
+      // repeat until timeout (empty array)
+      revel.AppLog.Warn("Jobs Retry", "waitfor", duration, "error", err)
+      run.In(duration, retry)
     } else {
       // this server is probably down. skip it..
       revel.AppLog.Error("Jobs Retry", "error", err)
@@ -49,11 +63,6 @@ func (retry Retry) Run() {
           revel.AppLog.Error("Jobs Retry", "error", err)
         }
       }
-      return
     }
-    revel.AppLog.Warn("Jobs Retry", "waitfor", retry.wait, "error", err)
-
-    // repeat until timeout
-    run.In(retry.wait, retry)
   }
 }

tests/jobstest.go

@@ -23,6 +23,7 @@ import (
   "git.feneas.org/ganggo/ganggo/app/models"
   "git.feneas.org/ganggo/gorm"
   "time"
+  "errors"
 )
 
 type JobsTest struct {
@@ -137,3 +138,48 @@ func (t *JobsTest) TestTelegramReceiver() {
   t.Assertf(len(tgReceiverTests) == 4,
     "Expected four entries, got %d", len(tgReceiverTests))
 }
+
+func (t *JobsTest) TestRetryOnFail() {
+  tests := []struct {
+    After []time.Duration
+    Expected int
+  }{
+    {
+      After: []time.Duration{
+        time.Second, time.Second, time.Second,
+      },
+      Expected: 3,
+    },
+    {
+      After: []time.Duration{
+        time.Second, time.Second,
+      },
+      Expected: 2,
+    },
+    {
+      After: []time.Duration{
+        time.Second, time.Second, time.Second, time.Second,
+      },
+      Expected: 4,
+    },
+  }
+
+  for i, test := range tests {
+    var triggered int = 0
+    retry := jobs.RetryOnFail{
+      Send: func() error {
+        triggered += 1
+        return errors.New("testing RetryOnFail")
+      },
+      After: test.After,
+    }
+    retry.Run()
+
+    // wait for async task
+    duration := time.Duration(test.Expected) * time.Second
+    time.Sleep(duration)
+
+    t.Assertf(triggered == test.Expected,
+      "#%d: Expected %d tries, got %d", i, test.Expected, triggered)
+  }
+}