| 1 | package store |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "strings" |
| 6 | "testing" |
| 7 | ) |
| 8 | |
| 9 | func TestFailAbandonedJobsThenRetryFailed(t *testing.T) { |
| 10 | s := openTest(t) |
| 11 | ctx := context.Background() |
| 12 | tim, err := s.UpsertUser(ctx, 1, "tim", "", false) |
| 13 | if err != nil { |
| 14 | t.Fatal(err) |
| 15 | } |
| 16 | repo, err := s.CreateRepo(ctx, "user", tim.ID, "bluegit", "", "master", "public") |
| 17 | if err != nil { |
| 18 | t.Fatal(err) |
| 19 | } |
| 20 | sha := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" |
| 21 | id, err := s.CreatePipeline(ctx, repo.ID, sha, "refs/heads/master", "push", "tim", "release", []NewJob{ |
| 22 | {Name: "build", Stage: "build", Position: 0, Script: "go build"}, |
| 23 | {Name: "publish_aur", Stage: "release", Position: 1, Script: "aur", Needs: []string{"build"}, NeedsSet: true, AllowFailure: true}, |
| 24 | {Name: "self_update", Stage: "release", Position: 2, Script: "post", Needs: []string{"publish_aur"}, NeedsSet: true}, |
| 25 | }) |
| 26 | if err != nil { |
| 27 | t.Fatal(err) |
| 28 | } |
| 29 | queuedID, err := s.CreatePipeline(ctx, repo.ID, sha, "refs/heads/topic", "push", "tim", "side", []NewJob{ |
| 30 | {Name: "wait", Stage: "test", Script: "sleep"}, |
| 31 | }) |
| 32 | if err != nil { |
| 33 | t.Fatal(err) |
| 34 | } |
| 35 | softID, err := s.CreatePipeline(ctx, repo.ID, sha, "refs/heads/soft", "push", "tim", "soft", []NewJob{ |
| 36 | {Name: "build", Stage: "build", Script: "go build"}, |
| 37 | {Name: "publish_aur", Stage: "release", Script: "aur", AllowFailure: true}, |
| 38 | }) |
| 39 | if err != nil { |
| 40 | t.Fatal(err) |
| 41 | } |
| 42 | set := func(pipe int64, name, status, log string) { |
| 43 | t.Helper() |
| 44 | if _, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = ?, log = ?, started_at = ? WHERE pipeline_id = ? AND name = ?`, status, log, now(), pipe, name); err != nil { |
| 45 | t.Fatal(err) |
| 46 | } |
| 47 | } |
| 48 | set(id, "build", "success", "ok\n") |
| 49 | set(id, "publish_aur", "success", "aur ok\n") |
| 50 | set(id, "self_update", "running", "post verstuurd\n") |
| 51 | set(queuedID, "wait", "queued", "") |
| 52 | set(softID, "build", "success", "ok\n") |
| 53 | set(softID, "publish_aur", "running", "bezig\n") |
| 54 | if _, err := s.db.ExecContext(ctx, `UPDATE pipelines SET status = 'running' WHERE id IN (?, ?, ?)`, id, queuedID, softID); err != nil { |
| 55 | t.Fatal(err) |
| 56 | } |
| 57 | |
| 58 | n, err := s.FailAbandonedJobs(ctx) |
| 59 | if err != nil || n != 2 { |
| 60 | t.Fatalf("abandoned: %d %v", n, err) |
| 61 | } |
| 62 | jobs, err := s.JobsByPipeline(ctx, id) |
| 63 | if err != nil { |
| 64 | t.Fatal(err) |
| 65 | } |
| 66 | by := map[string]Job{} |
| 67 | for _, j := range jobs { |
| 68 | by[j.Name] = j |
| 69 | } |
| 70 | upd := by["self_update"] |
| 71 | if upd.Status != "failed" || upd.FinishedAt.IsZero() || upd.StartedAt.IsZero() || !strings.Contains(upd.Log, "post verstuurd\n") || !strings.Contains(upd.Log, AbandonedJobNote) { |
| 72 | t.Fatalf("self_update: status=%s log=%q started=%v finished=%v", upd.Status, upd.Log, upd.StartedAt, upd.FinishedAt) |
| 73 | } |
| 74 | if by["build"].Status != "success" || by["publish_aur"].Status != "success" { |
| 75 | t.Fatalf("andere jobs: build=%s publish=%s", by["build"].Status, by["publish_aur"].Status) |
| 76 | } |
| 77 | p, err := s.PipelineByID(ctx, repo.ID, id) |
| 78 | if err != nil || p.Status != "failed" || p.FinishedAt.IsZero() { |
| 79 | t.Fatalf("pipeline: %+v %v", p, err) |
| 80 | } |
| 81 | if got := jobStatus(t, s, queuedID)["wait"]; got != "queued" { |
| 82 | t.Fatalf("wachtende job: %s", got) |
| 83 | } |
| 84 | queuedPipe, err := s.PipelineByID(ctx, repo.ID, queuedID) |
| 85 | if err != nil || queuedPipe.Status != "running" { |
| 86 | t.Fatalf("pipeline zonder dode job: %+v %v", queuedPipe, err) |
| 87 | } |
| 88 | softJobs, err := s.JobsByPipeline(ctx, softID) |
| 89 | if err != nil { |
| 90 | t.Fatal(err) |
| 91 | } |
| 92 | for _, j := range softJobs { |
| 93 | if j.Name == "publish_aur" && (j.Status != "failed" || !strings.Contains(j.Log, AbandonedJobNote)) { |
| 94 | t.Fatalf("allow_failure job: status=%s log=%q", j.Status, j.Log) |
| 95 | } |
| 96 | } |
| 97 | softPipe, err := s.PipelineByID(ctx, repo.ID, softID) |
| 98 | if err != nil || softPipe.Status != "success" { |
| 99 | t.Fatalf("alleen een zachte mislukking: %+v %v", softPipe, err) |
| 100 | } |
| 101 | running, err := s.CountRunningJobs(ctx) |
| 102 | if err != nil || running != 0 { |
| 103 | t.Fatalf("running na herstart: %d %v", running, err) |
| 104 | } |
| 105 | again, err := s.FailAbandonedJobs(ctx) |
| 106 | if err != nil || again != 0 { |
| 107 | t.Fatalf("tweede start: %d %v", again, err) |
| 108 | } |
| 109 | |
| 110 | ok, err := s.RetryFailedJobs(ctx, id) |
| 111 | if err != nil || !ok { |
| 112 | t.Fatalf("retry: %v %v", ok, err) |
| 113 | } |
| 114 | got := jobStatus(t, s, id) |
| 115 | if got["build"] != "success" || got["publish_aur"] != "success" || got["self_update"] != "queued" { |
| 116 | t.Fatalf("mode failed: %v", got) |
| 117 | } |
| 118 | retried, err := s.JobsByPipeline(ctx, id) |
| 119 | if err != nil { |
| 120 | t.Fatal(err) |
| 121 | } |
| 122 | for _, j := range retried { |
| 123 | if j.Name == "self_update" && (j.Log != "" || !j.FinishedAt.IsZero()) { |
| 124 | t.Fatalf("retry maakt de job leeg: log=%q finished=%v", j.Log, j.FinishedAt) |
| 125 | } |
| 126 | } |
| 127 | if side := jobStatus(t, s, queuedID)["wait"]; side != "queued" { |
| 128 | t.Fatalf("retry van de release raakt de andere pipeline: %s", side) |
| 129 | } |
| 130 | } |