| 1 | package store |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "database/sql" |
| 6 | "errors" |
| 7 | "strconv" |
| 8 | "time" |
| 9 | ) |
| 10 | |
| 11 | // Verlopen handmatige jobs (stale manual). |
| 12 | // |
| 13 | // Een handmatige job (status manual) verloopt als zijn pipeline stale is: |
| 14 | // - superseded: er is een nieuwere pipeline op dezelfde ref (en hetzelfde |
| 15 | // merge request) in dezelfde repository. Een pipeline die alleen een |
| 16 | // kapotte blue-ci-cd.yml meldt, telt niet als nieuwere pipeline. |
| 17 | // - leeftijd: de pipeline wacht alleen nog op handmatige jobs (status |
| 18 | // manual) en is ouder dan N dagen. N is een instance-instelling |
| 19 | // (standaard 7) die per repository overschreven kan worden; 0 = uit. |
| 20 | // |
| 21 | // Verlopen is niet destructief: de job gaat naar canceled met een reden en een |
| 22 | // logregel; logs en artifacts blijven staan. Jobs die alleen nog op de |
| 23 | // verlopen job wachtten (needs), verlopen mee. Niet-handmatige jobs die nog |
| 24 | // lopen of in de wachtrij staan, blijven ongemoeid. |
| 25 | |
| 26 | // DefaultStaleManualDays is de standaard voor de leeftijdsregel. |
| 27 | const DefaultStaleManualDays = 7 |
| 28 | |
| 29 | const staleManualKey = "stale_manual_days" |
| 30 | |
| 31 | // StaleManualDays geeft de instance-instelling (dagen, 0 = uit). |
| 32 | func (s *Store) StaleManualDays(ctx context.Context) (int, error) { |
| 33 | var v string |
| 34 | err := s.db.QueryRowContext(ctx, `SELECT value FROM instance_settings WHERE key = ?`, staleManualKey).Scan(&v) |
| 35 | if errors.Is(err, sql.ErrNoRows) { |
| 36 | return DefaultStaleManualDays, nil |
| 37 | } |
| 38 | if err != nil { |
| 39 | return 0, err |
| 40 | } |
| 41 | n, err := strconv.Atoi(v) |
| 42 | if err != nil || n < 0 { |
| 43 | return DefaultStaleManualDays, nil |
| 44 | } |
| 45 | return n, nil |
| 46 | } |
| 47 | |
| 48 | // SetStaleManualDays zet de instance-instelling (0 = uit, max 3650). |
| 49 | func (s *Store) SetStaleManualDays(ctx context.Context, days int) error { |
| 50 | if days < 0 || days > 3650 { |
| 51 | return errors.New("stale manual days out of range") |
| 52 | } |
| 53 | _, err := s.db.ExecContext(ctx, `INSERT INTO instance_settings (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value`, staleManualKey, strconv.Itoa(days)) |
| 54 | return err |
| 55 | } |
| 56 | |
| 57 | // RepoStaleManualDays geeft de eigen instelling van een repository; |
| 58 | // -1 = de instance-instelling volgen. |
| 59 | func (s *Store) RepoStaleManualDays(ctx context.Context, repoID int64) (int, error) { |
| 60 | var n int |
| 61 | err := s.db.QueryRowContext(ctx, `SELECT stale_manual_days FROM repos WHERE id = ?`, repoID).Scan(&n) |
| 62 | return n, err |
| 63 | } |
| 64 | |
| 65 | // SetRepoStaleManualDays zet de eigen instelling (-1 = volgen, 0 = uit). |
| 66 | func (s *Store) SetRepoStaleManualDays(ctx context.Context, repoID int64, days int) error { |
| 67 | if days < -1 || days > 3650 { |
| 68 | return errors.New("stale manual days out of range") |
| 69 | } |
| 70 | _, err := s.db.ExecContext(ctx, `UPDATE repos SET stale_manual_days = ? WHERE id = ?`, days, repoID) |
| 71 | return err |
| 72 | } |
| 73 | |
| 74 | // ExpiredManual is één pipeline waarvan handmatige jobs verlopen zijn. |
| 75 | type ExpiredManual struct { |
| 76 | RepoID int64 |
| 77 | PipelineID int64 |
| 78 | Ref string |
| 79 | Jobs int |
| 80 | Reason string // superseded of stale |
| 81 | } |
| 82 | |
| 83 | // expirePipelineManual laat de handmatige jobs van één pipeline verlopen |
| 84 | // (en de jobs die alleen daarop wachten) en werkt de pipelinestatus bij. |
| 85 | func (s *Store) expirePipelineManual(ctx context.Context, pipelineID int64, reason, note string) (int, error) { |
| 86 | jobs, err := s.JobsByPipeline(ctx, pipelineID) |
| 87 | if err != nil { |
| 88 | return 0, err |
| 89 | } |
| 90 | blocked := manualBlocked(jobs) |
| 91 | line := time.Now().UTC().Format(time.RFC3339) + "\t" + note + "\n" |
| 92 | n := 0 |
| 93 | for i, j := range jobs { |
| 94 | if j.Status != "manual" && !(j.Status == "queued" && blocked[i]) { |
| 95 | continue |
| 96 | } |
| 97 | res, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = 'canceled', failure_reason = ?, log = log || ?, finished_at = ? WHERE id = ? AND status = ?`, reason, line, now(), j.ID, j.Status) |
| 98 | if err != nil { |
| 99 | return n, err |
| 100 | } |
| 101 | if c, _ := res.RowsAffected(); c == 1 { |
| 102 | n++ |
| 103 | } |
| 104 | } |
| 105 | if n > 0 { |
| 106 | if _, err := s.RefreshPipelineStatus(ctx, pipelineID); err != nil { |
| 107 | return n, err |
| 108 | } |
| 109 | } |
| 110 | return n, nil |
| 111 | } |
| 112 | |
| 113 | const noteSuperseded = "manual job expired (obsolete): a newer pipeline exists for the same ref" |
| 114 | |
| 115 | func noteStale(days int) string { |
| 116 | return "manual job expired (obsolete): the pipeline waited only for manual action for more than " + strconv.Itoa(days) + " days" |
| 117 | } |
| 118 | |
| 119 | // configFailureSQL herkent een pipeline die alleen een kapotte config meldt. |
| 120 | const configFailureSQL = `EXISTS (SELECT 1 FROM jobs cj WHERE cj.pipeline_id = n.id AND cj.name = 'blue-ci-cd' AND cj.script = 'exit 1' AND cj.image = 'alpine:3.20' AND cj.position = 0) |
| 121 | AND (SELECT COUNT(*) FROM jobs cj WHERE cj.pipeline_id = n.id) = 1` |
| 122 | |
| 123 | // ExpireSupersededManual laat handmatige jobs verlopen in pipelines op |
| 124 | // dezelfde ref (en hetzelfde MR) die ouder zijn dan keep. |
| 125 | func (s *Store) ExpireSupersededManual(ctx context.Context, repoID int64, ref string, keep int64) ([]ExpiredManual, error) { |
| 126 | rows, err := s.db.QueryContext(ctx, `SELECT DISTINCT p.id, p.ref FROM pipelines p JOIN jobs j ON j.pipeline_id = p.id |
| 127 | JOIN pipelines n ON n.id = ? |
| 128 | WHERE p.repo_id = ? AND p.ref = ? AND p.id < n.id AND p.mr_id = n.mr_id AND j.status = 'manual' |
| 129 | AND NOT (`+configFailureSQL+`)`, keep, repoID, ref) |
| 130 | if err != nil { |
| 131 | return nil, err |
| 132 | } |
| 133 | return s.expireRows(ctx, rows, repoID, "superseded", ReasonSuperseded, noteSuperseded) |
| 134 | } |
| 135 | |
| 136 | func (s *Store) expireRows(ctx context.Context, rows *sql.Rows, repoID int64, kind, reason, note string) ([]ExpiredManual, error) { |
| 137 | type item struct { |
| 138 | id int64 |
| 139 | ref string |
| 140 | } |
| 141 | var items []item |
| 142 | for rows.Next() { |
| 143 | var it item |
| 144 | if err := rows.Scan(&it.id, &it.ref); err != nil { |
| 145 | rows.Close() |
| 146 | return nil, err |
| 147 | } |
| 148 | items = append(items, it) |
| 149 | } |
| 150 | err := rows.Err() |
| 151 | rows.Close() |
| 152 | if err != nil { |
| 153 | return nil, err |
| 154 | } |
| 155 | var out []ExpiredManual |
| 156 | for _, it := range items { |
| 157 | n, err := s.expirePipelineManual(ctx, it.id, reason, note) |
| 158 | if err != nil { |
| 159 | return out, err |
| 160 | } |
| 161 | if n > 0 { |
| 162 | out = append(out, ExpiredManual{RepoID: repoID, PipelineID: it.id, Ref: it.ref, Jobs: n, Reason: kind}) |
| 163 | } |
| 164 | } |
| 165 | return out, nil |
| 166 | } |
| 167 | |
| 168 | // SweepStaleManual loopt alle repositories langs: eerst superseded |
| 169 | // (voor oude pipelines van vóór deze functie), dan de leeftijdsregel. |
| 170 | func (s *Store) SweepStaleManual(ctx context.Context, at time.Time) ([]ExpiredManual, error) { |
| 171 | def, err := s.StaleManualDays(ctx) |
| 172 | if err != nil { |
| 173 | return nil, err |
| 174 | } |
| 175 | // Superseded: pipelines met een handmatige job en een nieuwere pipeline |
| 176 | // op dezelfde ref en hetzelfde MR. |
| 177 | rows, err := s.db.QueryContext(ctx, `SELECT DISTINCT p.repo_id, p.id, p.ref FROM pipelines p JOIN jobs j ON j.pipeline_id = p.id |
| 178 | WHERE j.status = 'manual' AND EXISTS ( |
| 179 | SELECT 1 FROM pipelines n WHERE n.repo_id = p.repo_id AND n.ref = p.ref AND n.mr_id = p.mr_id AND n.id > p.id |
| 180 | AND NOT (`+configFailureSQL+`)) |
| 181 | ORDER BY p.id`) |
| 182 | if err != nil { |
| 183 | return nil, err |
| 184 | } |
| 185 | type item struct { |
| 186 | repo, id int64 |
| 187 | ref string |
| 188 | } |
| 189 | var sup []item |
| 190 | for rows.Next() { |
| 191 | var it item |
| 192 | if err := rows.Scan(&it.repo, &it.id, &it.ref); err != nil { |
| 193 | rows.Close() |
| 194 | return nil, err |
| 195 | } |
| 196 | sup = append(sup, it) |
| 197 | } |
| 198 | err = rows.Err() |
| 199 | rows.Close() |
| 200 | if err != nil { |
| 201 | return nil, err |
| 202 | } |
| 203 | var out []ExpiredManual |
| 204 | for _, it := range sup { |
| 205 | n, err := s.expirePipelineManual(ctx, it.id, ReasonSuperseded, noteSuperseded) |
| 206 | if err != nil { |
| 207 | return out, err |
| 208 | } |
| 209 | if n > 0 { |
| 210 | out = append(out, ExpiredManual{RepoID: it.repo, PipelineID: it.id, Ref: it.ref, Jobs: n, Reason: "superseded"}) |
| 211 | } |
| 212 | } |
| 213 | // Leeftijd: pipelines die alleen nog op handmatige actie wachten. |
| 214 | rows, err = s.db.QueryContext(ctx, `SELECT p.repo_id, p.id, p.ref, p.created_at, r.stale_manual_days FROM pipelines p JOIN repos r ON r.id = p.repo_id WHERE p.status = 'manual' ORDER BY p.id`) |
| 215 | if err != nil { |
| 216 | return out, err |
| 217 | } |
| 218 | type aged struct { |
| 219 | item |
| 220 | days int |
| 221 | } |
| 222 | var old []aged |
| 223 | for rows.Next() { |
| 224 | var it item |
| 225 | var created string |
| 226 | var days int |
| 227 | if err := rows.Scan(&it.repo, &it.id, &it.ref, &created, &days); err != nil { |
| 228 | rows.Close() |
| 229 | return out, err |
| 230 | } |
| 231 | if days < 0 { |
| 232 | days = def |
| 233 | } |
| 234 | if days == 0 { |
| 235 | continue |
| 236 | } |
| 237 | t := parseTime(created) |
| 238 | if t.IsZero() || at.Sub(t) < time.Duration(days)*24*time.Hour { |
| 239 | continue |
| 240 | } |
| 241 | old = append(old, aged{it, days}) |
| 242 | } |
| 243 | err = rows.Err() |
| 244 | rows.Close() |
| 245 | if err != nil { |
| 246 | return out, err |
| 247 | } |
| 248 | for _, it := range old { |
| 249 | n, err := s.expirePipelineManual(ctx, it.id, ReasonStaleManual, noteStale(it.days)) |
| 250 | if err != nil { |
| 251 | return out, err |
| 252 | } |
| 253 | if n > 0 { |
| 254 | out = append(out, ExpiredManual{RepoID: it.repo, PipelineID: it.id, Ref: it.ref, Jobs: n, Reason: "stale"}) |
| 255 | } |
| 256 | } |
| 257 | return out, nil |
| 258 | } |