BlueGit

timfa bluegit

Files

bluegit / internal / store / flow.go

flow.go

fix: stuur de console-output van elke job mee in pipeline-webhooks
timfa project_87923614_bot_3aff574f6f98414b945a8799374f9092 · pushed by timfa · 3 weeks ago
6080426e History
flow.go 35.3 KiB Download Raw
1package store
2
3import (
4 "context"
5 "database/sql"
6 "encoding/json"
7 "errors"
8 "time"
9
10 "gitlab.com/Timfa/bluegit/internal/i18n"
11 "gitlab.com/Timfa/bluegit/internal/names"
12)
13
14// Webhook is een uitgaande melding.
15type Webhook struct {
16 ID int64
17 ScopeKind string
18 ScopeID int64
19 Name string
20 Preset string
21 URLTemplate string
22 Headers map[string]string
23 BodyTemplate string
24 Events []string
25 BranchFilter string
26 Active bool
27 Method string
28 Fields string
29}
30
31// CreateWebhook bewaart een webhook.
32func (s *Store) CreateWebhook(ctx context.Context, w Webhook) (int64, error) {
33 if w.Headers == nil {
34 w.Headers = map[string]string{}
35 }
36 if w.Preset == "" {
37 w.Preset = "generic"
38 }
39 if w.Method == "" {
40 w.Method = "POST"
41 }
42 res, err := s.db.ExecContext(ctx, `INSERT INTO webhooks (scope_kind, scope_id, name, preset, url_template, headers_json, body_template, events_json, branch_filter, active, method, fields_json, created_at)
43 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, ?)`,
44 w.ScopeKind, w.ScopeID, w.Name, w.Preset, w.URLTemplate, mustJSON(w.Headers), w.BodyTemplate, mustJSON(w.Events), w.BranchFilter, w.Method, w.Fields, now())
45 if err != nil {
46 return 0, err
47 }
48 return res.LastInsertId()
49}
50
51// ListWebhooks geeft de webhooks van een repository of groep.
52func (s *Store) ListWebhooks(ctx context.Context, scopeKind string, scopeID int64) ([]Webhook, error) {
53 rows, err := s.db.QueryContext(ctx, `SELECT `+webhookCols+` FROM webhooks WHERE scope_kind = ? AND scope_id = ? ORDER BY id`, scopeKind, scopeID)
54 if err != nil {
55 return nil, err
56 }
57 defer rows.Close()
58 var out []Webhook
59 for rows.Next() {
60 w, err := scanWebhook(rows)
61 if err != nil {
62 return nil, err
63 }
64 out = append(out, w)
65 }
66 return out, rows.Err()
67}
68
69const webhookCols = `id, scope_kind, scope_id, name, preset, url_template, headers_json, body_template, events_json, branch_filter, active, method, fields_json`
70
71// Webhook leest één webhook in dit scope.
72func (s *Store) Webhook(ctx context.Context, scopeKind string, scopeID, id int64) (Webhook, error) {
73 w, err := scanWebhook(s.db.QueryRowContext(ctx, `SELECT `+webhookCols+` FROM webhooks WHERE id = ? AND scope_kind = ? AND scope_id = ?`, id, scopeKind, scopeID))
74 if errors.Is(err, sql.ErrNoRows) {
75 return Webhook{}, errNotFound
76 }
77 return w, err
78}
79
80// UpdateWebhook werkt een webhook in dit scope bij.
81func (s *Store) UpdateWebhook(ctx context.Context, w Webhook) error {
82 if w.Headers == nil {
83 w.Headers = map[string]string{}
84 }
85 if w.Preset == "" {
86 w.Preset = "generic"
87 }
88 if w.Method == "" {
89 w.Method = "POST"
90 }
91 res, err := s.db.ExecContext(ctx, `UPDATE webhooks SET name = ?, preset = ?, url_template = ?, headers_json = ?, body_template = ?, events_json = ?, branch_filter = ?, method = ?, fields_json = ? WHERE id = ? AND scope_kind = ? AND scope_id = ?`,
92 w.Name, w.Preset, w.URLTemplate, mustJSON(w.Headers), w.BodyTemplate, mustJSON(w.Events), w.BranchFilter, w.Method, w.Fields, w.ID, w.ScopeKind, w.ScopeID)
93 if err != nil {
94 return err
95 }
96 n, _ := res.RowsAffected()
97 if n == 0 {
98 return errNotFound
99 }
100 return nil
101}
102
103func scanWebhook(row interface{ Scan(...any) error }) (Webhook, error) {
104 var w Webhook
105 var headers, events string
106 var active int
107 if err := row.Scan(&w.ID, &w.ScopeKind, &w.ScopeID, &w.Name, &w.Preset, &w.URLTemplate, &headers, &w.BodyTemplate, &events, &w.BranchFilter, &active, &w.Method, &w.Fields); err != nil {
108 return Webhook{}, err
109 }
110 w.Active = active == 1
111 _ = json.Unmarshal([]byte(headers), &w.Headers)
112 _ = json.Unmarshal([]byte(events), &w.Events)
113 if w.Headers == nil {
114 w.Headers = map[string]string{}
115 }
116 return w, nil
117}
118
119// DeleteWebhook verwijdert een webhook in dit scope.
120func (s *Store) DeleteWebhook(ctx context.Context, scopeKind string, scopeID, id int64) error {
121 res, err := s.db.ExecContext(ctx, `DELETE FROM webhooks WHERE id = ? AND scope_kind = ? AND scope_id = ?`, id, scopeKind, scopeID)
122 if err != nil {
123 return err
124 }
125 n, _ := res.RowsAffected()
126 if n == 0 {
127 return errNotFound
128 }
129 return nil
130}
131
132// AddDelivery bewaart het resultaat van een poging.
133func (s *Store) AddDelivery(ctx context.Context, webhookID int64, eventType string, status int, errText string) error {
134 _, err := s.db.ExecContext(ctx, `INSERT INTO webhook_deliveries (webhook_id, event_type, status_code, error, created_at) VALUES (?, ?, ?, ?, ?)`,
135 webhookID, eventType, status, trim(errText, 500), now())
136 return err
137}
138
139// Delivery is één poging.
140type Delivery struct {
141 EventType string
142 StatusCode int
143 Error string
144 CreatedAt time.Time
145}
146
147// RecentDeliveries geeft de laatste pogingen.
148func (s *Store) RecentDeliveries(ctx context.Context, webhookID int64) ([]Delivery, error) {
149 rows, err := s.db.QueryContext(ctx, `SELECT event_type, status_code, error, created_at FROM webhook_deliveries WHERE webhook_id = ? ORDER BY id DESC LIMIT 8`, webhookID)
150 if err != nil {
151 return nil, err
152 }
153 defer rows.Close()
154 var out []Delivery
155 for rows.Next() {
156 var d Delivery
157 var created string
158 if err := rows.Scan(&d.EventType, &d.StatusCode, &d.Error, &created); err != nil {
159 return nil, err
160 }
161 d.CreatedAt = parseTime(created)
162 out = append(out, d)
163 }
164 return out, rows.Err()
165}
166
167// MergeRequest is een basisverzoek om een branch samen te voegen.
168type MergeRequest struct {
169 ID int64
170 RepoID int64
171 Number int64
172 Title string
173 Description string
174 SourceBranch string
175 TargetBranch string
176 AuthorID int64
177 Author string
178 Status string
179 CreatedAt time.Time
180 UpdatedAt time.Time
181 Draft bool
182 Labels string
183}
184
185// CreateMergeRequest nummert per repository.
186func (s *Store) CreateMergeRequest(ctx context.Context, repoID, authorID int64, title, description, source, target string) (MergeRequest, error) {
187 title = trim(title, 200)
188 if title == "" {
189 return MergeRequest{}, i18n.E("err.mrTitle")
190 }
191 tx, err := s.db.BeginTx(ctx, nil)
192 if err != nil {
193 return MergeRequest{}, err
194 }
195 defer func() { _ = tx.Rollback() }()
196 var n int64
197 if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(number), 0) FROM merge_requests WHERE repo_id = ?`, repoID).Scan(&n); err != nil {
198 return MergeRequest{}, err
199 }
200 ts := now()
201 res, err := tx.ExecContext(ctx, `INSERT INTO merge_requests (repo_id, number, title, description, source_branch, target_branch, author_id, status, created_at, updated_at)
202 VALUES (?, ?, ?, ?, ?, ?, ?, 'open', ?, ?)`, repoID, n+1, title, description, source, target, authorID, ts, ts)
203 if err != nil {
204 return MergeRequest{}, err
205 }
206 id, _ := res.LastInsertId()
207 if err := tx.Commit(); err != nil {
208 return MergeRequest{}, err
209 }
210 return s.MergeRequest(ctx, repoID, id)
211}
212
213// MergeRequest leest op interne id.
214func (s *Store) MergeRequest(ctx context.Context, repoID, id int64) (MergeRequest, error) {
215 row := s.db.QueryRowContext(ctx, mrSelect+` WHERE m.repo_id = ? AND m.id = ?`, repoID, id)
216 return scanMR(row)
217}
218
219// MergeRequestByNumber leest !nummer.
220func (s *Store) MergeRequestByNumber(ctx context.Context, repoID, number int64) (MergeRequest, error) {
221 row := s.db.QueryRowContext(ctx, mrSelect+` WHERE m.repo_id = ? AND m.number = ?`, repoID, number)
222 return scanMR(row)
223}
224
225const mrSelect = `SELECT m.id, m.repo_id, m.number, m.title, m.description, m.source_branch, m.target_branch, m.author_id, u.username, m.status, m.created_at, m.updated_at, m.draft, m.labels
226FROM merge_requests m JOIN users u ON u.id = m.author_id`
227
228func scanMR(row *sql.Row) (MergeRequest, error) {
229 var m MergeRequest
230 var created, updated string
231 var draft int
232 if err := row.Scan(&m.ID, &m.RepoID, &m.Number, &m.Title, &m.Description, &m.SourceBranch, &m.TargetBranch, &m.AuthorID, &m.Author, &m.Status, &created, &updated, &draft, &m.Labels); err != nil {
233 if errors.Is(err, sql.ErrNoRows) {
234 return MergeRequest{}, errNotFound
235 }
236 return MergeRequest{}, err
237 }
238 m.CreatedAt = parseTime(created)
239 m.UpdatedAt = parseTime(updated)
240 m.Draft = draft == 1
241 return m, nil
242}
243
244// ListMergeRequests geeft verzoeken, open eerst.
245func (s *Store) ListMergeRequests(ctx context.Context, repoID int64) ([]MergeRequest, error) {
246 rows, err := s.db.QueryContext(ctx, mrSelect+` WHERE m.repo_id = ? ORDER BY CASE m.status WHEN 'open' THEN 0 ELSE 1 END, m.number DESC`, repoID)
247 if err != nil {
248 return nil, err
249 }
250 defer rows.Close()
251 return scanMRList(rows)
252}
253
254// SetMergeRequestStatus zet open, merged of closed.
255func (s *Store) SetMergeRequestStatus(ctx context.Context, id int64, status string) error {
256 _, err := s.db.ExecContext(ctx, `UPDATE merge_requests SET status = ?, updated_at = ? WHERE id = ?`, status, now(), id)
257 return err
258}
259
260// OpenMergeRequestsBySource geeft open verzoeken waarvan de bronbranch deze is.
261func (s *Store) OpenMergeRequestsBySource(ctx context.Context, repoID int64, branch string) ([]MergeRequest, error) {
262 return s.openMergeRequests(ctx, repoID, "source_branch", branch)
263}
264
265// OpenMergeRequestsByTarget geeft open verzoeken waarvan de doelbranch deze is.
266func (s *Store) OpenMergeRequestsByTarget(ctx context.Context, repoID int64, branch string) ([]MergeRequest, error) {
267 return s.openMergeRequests(ctx, repoID, "target_branch", branch)
268}
269
270func (s *Store) openMergeRequests(ctx context.Context, repoID int64, column, branch string) ([]MergeRequest, error) {
271 if column != "source_branch" && column != "target_branch" {
272 return nil, errNotFound
273 }
274 rows, err := s.db.QueryContext(ctx, mrSelect+` WHERE m.repo_id = ? AND m.`+column+` = ? AND m.status = 'open' ORDER BY m.number`, repoID, branch)
275 if err != nil {
276 return nil, err
277 }
278 defer rows.Close()
279 return scanMRList(rows)
280}
281
282func scanMRList(rows *sql.Rows) ([]MergeRequest, error) {
283 var out []MergeRequest
284 for rows.Next() {
285 var m MergeRequest
286 var created, updated string
287 var draft int
288 if err := rows.Scan(&m.ID, &m.RepoID, &m.Number, &m.Title, &m.Description, &m.SourceBranch, &m.TargetBranch, &m.AuthorID, &m.Author, &m.Status, &created, &updated, &draft, &m.Labels); err != nil {
289 return nil, err
290 }
291 m.CreatedAt = parseTime(created)
292 m.UpdatedAt = parseTime(updated)
293 m.Draft = draft == 1
294 out = append(out, m)
295 }
296 return out, rows.Err()
297}
298
299// AttachPipelineMR koppelt een pipeline aan een merge request.
300func (s *Store) AttachPipelineMR(ctx context.Context, pipelineID, mrID int64) error {
301 _, err := s.db.ExecContext(ctx, `UPDATE pipelines SET mr_id = ? WHERE id = ?`, mrID, pipelineID)
302 return err
303}
304
305// LatestMergePipeline is de nieuwste pipeline van deze merge request.
306func (s *Store) LatestMergePipeline(ctx context.Context, repoID, mrID int64) (Pipeline, error) {
307 p, err := scanPipelineRow(s.db.QueryRowContext(ctx, pipelineCols+` WHERE repo_id = ? AND mr_id = ? ORDER BY id DESC LIMIT 1`, repoID, mrID))
308 if errors.Is(err, sql.ErrNoRows) {
309 return Pipeline{}, errNotFound
310 }
311 return p, err
312}
313
314// PipelineBlocksMerge zegt of deze jobs een merge tegenhouden.
315// Een lege tekst betekent dat elke job klaar is en geen harde fout heeft.
316// allow_failure mag mislukt zijn, maar moet wel af zijn.
317func PipelineBlocksMerge(jobs []Job) string {
318 if len(jobs) == 0 {
319 return "waiting"
320 }
321 hard, open := false, false
322 for _, j := range jobs {
323 switch j.Status {
324 case "success", "skipped":
325 case "failed", "canceled":
326 if !j.AllowFailure {
327 hard = true
328 }
329 default:
330 open = true
331 }
332 }
333 if open {
334 return "running"
335 }
336 if hard {
337 return "failed"
338 }
339 return ""
340}
341
342// NewJob is een job die nog de wachtrij in moet.
343type NewJob struct {
344 Name string
345 Stage string
346 Position int
347 Image string
348 Before string
349 Script string
350 After string
351 Needs []string
352 Optional []string
353 NeedsSet bool
354 Variables map[string]string
355 ArtifactPaths []string
356 ArtifactExpire time.Duration
357 AllowFailure bool
358 When string
359 Timeout time.Duration
360 Tags []string
361 CachePaths []string
362 CacheKey string
363 ResourceGroup string
364 RetryMax int
365 RetryWhen []string
366 Services []Service
367 Environment string
368 EnvironmentURL string
369 Interruptible bool
370}
371
372// Service is een sidecar die de runner naast de job start.
373type Service struct {
374 Name string `json:"name"`
375 Alias string `json:"alias"`
376 Command []string `json:"command,omitempty"`
377 Entrypoint []string `json:"entrypoint,omitempty"`
378 Variables map[string]string `json:"variables,omitempty"`
379}
380
381// Pipeline is een draai van blue-ci-cd.yml.
382type Pipeline struct {
383 ID int64
384 RepoID int64
385 CommitSHA string
386 Ref string
387 Status string
388 Source string
389 Actor string
390 CommitTitle string
391 CreatedAt time.Time
392 StartedAt time.Time
393 FinishedAt time.Time
394 MRID int64
395}
396
397// Job is één container in een pipeline.
398type Job struct {
399 ID int64
400 PipelineID int64
401 RepoID int64
402 Name string
403 Stage string
404 Position int
405 Status string
406 Image string
407 Before string
408 Script string
409 After string
410 Needs []string
411 Optional []string
412 NeedsSet bool
413 Variables map[string]string
414 ArtifactPaths []string
415 ArtifactExpire time.Duration
416 AllowFailure bool
417 When string
418 Timeout time.Duration
419 Tags []string
420 CachePaths []string
421 CacheKey string
422 ResourceGroup string
423 RetryMax int
424 RetryWhen []string
425 Attempt int
426 Services []Service
427 Environment string
428 EnvironmentURL string
429 Interruptible bool
430 Log string
431 CommitSHA string
432 Ref string
433 StartedAt time.Time
434 FinishedAt time.Time
435}
436
437// RecordConfigFailure bewaart een pipeline die niet kon starten.
438func (s *Store) RecordConfigFailure(ctx context.Context, repoID int64, sha, ref, message string) (int64, error) {
439 tx, err := s.db.BeginTx(ctx, nil)
440 if err != nil {
441 return 0, err
442 }
443 defer func() { _ = tx.Rollback() }()
444 ts := now()
445 res, err := tx.ExecContext(ctx, `INSERT INTO pipelines (repo_id, commit_sha, ref, status, created_at, finished_at) VALUES (?, ?, ?, 'failed', ?, ?)`, repoID, sha, ref, ts, ts)
446 if err != nil {
447 return 0, err
448 }
449 id, _ := res.LastInsertId()
450 _, err = tx.ExecContext(ctx, `INSERT INTO jobs (pipeline_id, repo_id, name, stage, position, status, image, script, needs_json, needs_set, variables_json, artifact_paths, artifact_expire_seconds, allow_failure, log, finished_at)
451 VALUES (?, ?, 'blue-ci-cd', 'test', 0, 'failed', 'alpine:3.20', 'exit 1', '[]', 0, '{}', '[]', 0, 0, ?, ?)`, id, repoID, message, ts)
452 if err != nil {
453 return 0, err
454 }
455 if err := tx.Commit(); err != nil {
456 return 0, err
457 }
458 return id, nil
459}
460
461// CreatePipeline maakt een pipeline met jobs in de wachtrij.
462func (s *Store) CreatePipeline(ctx context.Context, repoID int64, sha, ref, source, actor, title string, jobs []NewJob) (int64, error) {
463 tx, err := s.db.BeginTx(ctx, nil)
464 if err != nil {
465 return 0, err
466 }
467 defer func() { _ = tx.Rollback() }()
468 res, err := tx.ExecContext(ctx, `INSERT INTO pipelines (repo_id, commit_sha, ref, status, source, actor, commit_title, created_at) VALUES (?, ?, ?, 'queued', ?, ?, ?, ?)`, repoID, sha, ref, source, actor, title, now())
469 if err != nil {
470 return 0, err
471 }
472 id, _ := res.LastInsertId()
473 for _, j := range jobs {
474 if j.Variables == nil {
475 j.Variables = map[string]string{}
476 }
477 if j.Needs == nil {
478 j.Needs = []string{}
479 }
480 if j.ArtifactPaths == nil {
481 j.ArtifactPaths = []string{}
482 }
483 if j.Optional == nil {
484 j.Optional = []string{}
485 }
486 if j.Tags == nil {
487 j.Tags = []string{}
488 }
489 if j.CachePaths == nil {
490 j.CachePaths = []string{}
491 }
492 if j.When == "" {
493 j.When = "on_success"
494 }
495 if j.RetryWhen == nil {
496 j.RetryWhen = []string{}
497 }
498 if j.Services == nil {
499 j.Services = []Service{}
500 }
501 status := "queued"
502 if j.When == "manual" {
503 status = "manual"
504 }
505 _, err := tx.ExecContext(ctx, `INSERT INTO jobs (pipeline_id, repo_id, name, stage, position, status, image, script, needs_json, needs_set, variables_json, artifact_paths, artifact_expire_seconds, allow_failure, log, before_script, after_script, when_mode, timeout_seconds, tags_json, cache_paths, cache_key, resource_group, optional_json, retry_max, retry_when, services_json, environment_name, environment_url, interruptible)
506 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, '', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
507 id, repoID, j.Name, j.Stage, j.Position, status, j.Image, j.Script, mustJSON(j.Needs), boolInt(j.NeedsSet), mustJSON(j.Variables), mustJSON(j.ArtifactPaths), int(j.ArtifactExpire/time.Second), boolInt(j.AllowFailure),
508 j.Before, j.After, j.When, int(j.Timeout/time.Second), mustJSON(j.Tags), mustJSON(j.CachePaths), j.CacheKey, j.ResourceGroup, mustJSON(j.Optional),
509 j.RetryMax, mustJSON(j.RetryWhen), mustJSON(j.Services), j.Environment, j.EnvironmentURL, boolInt(j.Interruptible))
510 if err != nil {
511 return 0, err
512 }
513 }
514 if err := tx.Commit(); err != nil {
515 return 0, err
516 }
517 return id, nil
518}
519
520// ListPipelines geeft de nieuwste pipelines.
521func (s *Store) ListPipelines(ctx context.Context, repoID int64) ([]Pipeline, error) {
522 rows, err := s.db.QueryContext(ctx, pipelineCols+` WHERE repo_id = ? ORDER BY id DESC LIMIT 50`, repoID)
523 if err != nil {
524 return nil, err
525 }
526 defer rows.Close()
527 var out []Pipeline
528 for rows.Next() {
529 p, err := scanPipeline(rows)
530 if err != nil {
531 return nil, err
532 }
533 out = append(out, p)
534 }
535 return out, rows.Err()
536}
537
538const pipelineCols = `SELECT id, repo_id, commit_sha, ref, status, created_at, finished_at, source, actor, commit_title, started_at, mr_id FROM pipelines`
539
540func scanPipeline(rows *sql.Rows) (Pipeline, error) {
541 return scanPipelineRow(rows)
542}
543
544func scanPipelineRow(sc scanner) (Pipeline, error) {
545 var p Pipeline
546 var created, finished, started sql.NullString
547 if err := sc.Scan(&p.ID, &p.RepoID, &p.CommitSHA, &p.Ref, &p.Status, &created, &finished, &p.Source, &p.Actor, &p.CommitTitle, &started, &p.MRID); err != nil {
548 return Pipeline{}, err
549 }
550 p.CreatedAt = parseTime(created.String)
551 p.FinishedAt = parseTime(finished.String)
552 p.StartedAt = parseTime(started.String)
553 return p, nil
554}
555
556type scanner interface {
557 Scan(dest ...any) error
558}
559
560// LatestPipelineByCommit geeft de nieuwste pipeline van deze commit.
561func (s *Store) LatestPipelineByCommit(ctx context.Context, repoID int64, sha string) (Pipeline, error) {
562 p, err := scanPipelineRow(s.db.QueryRowContext(ctx, pipelineCols+` WHERE repo_id = ? AND commit_sha = ? ORDER BY id DESC LIMIT 1`, repoID, sha))
563 if errors.Is(err, sql.ErrNoRows) {
564 return Pipeline{}, errNotFound
565 }
566 return p, err
567}
568
569// PipelineByID leest één pipeline.
570func (s *Store) PipelineByID(ctx context.Context, repoID, id int64) (Pipeline, error) {
571 p, err := scanPipelineRow(s.db.QueryRowContext(ctx, pipelineCols+` WHERE repo_id = ? AND id = ?`, repoID, id))
572 if errors.Is(err, sql.ErrNoRows) {
573 return Pipeline{}, errNotFound
574 }
575 return p, err
576}
577
578// JobsByPipeline geeft de jobs in bestandsvolgorde.
579func (s *Store) JobsByPipeline(ctx context.Context, pipelineID int64) ([]Job, error) {
580 rows, err := s.db.QueryContext(ctx, jobSelect+` WHERE j.pipeline_id = ? ORDER BY j.position, j.id`, pipelineID)
581 if err != nil {
582 return nil, err
583 }
584 defer rows.Close()
585 var out []Job
586 for rows.Next() {
587 j, err := scanJob(rows)
588 if err != nil {
589 return nil, err
590 }
591 out = append(out, j)
592 }
593 return out, rows.Err()
594}
595
596const jobSelect = `SELECT j.id, j.pipeline_id, j.repo_id, j.name, j.stage, j.position, j.status, j.image, j.script, j.needs_json, j.needs_set, j.variables_json, j.artifact_paths, j.artifact_expire_seconds, j.allow_failure, j.log, p.commit_sha, p.ref, j.before_script, j.after_script, j.when_mode, j.timeout_seconds, j.tags_json, j.cache_paths, j.cache_key, j.resource_group, j.optional_json, j.started_at, j.finished_at, j.retry_max, j.retry_when, j.attempt, j.services_json, j.environment_name, j.environment_url, j.interruptible
597FROM jobs j JOIN pipelines p ON p.id = j.pipeline_id`
598
599func scanJob(rows *sql.Rows) (Job, error) {
600 var j Job
601 var needs, vars, paths, tags, cache, optional, retryWhen, services string
602 var needsSet, allow, expire, timeout, interruptible int
603 var started, finished sql.NullString
604 if err := rows.Scan(&j.ID, &j.PipelineID, &j.RepoID, &j.Name, &j.Stage, &j.Position, &j.Status, &j.Image, &j.Script, &needs, &needsSet, &vars, &paths, &expire, &allow, &j.Log, &j.CommitSHA, &j.Ref, &j.Before, &j.After, &j.When, &timeout, &tags, &cache, &j.CacheKey, &j.ResourceGroup, &optional, &started, &finished, &j.RetryMax, &retryWhen, &j.Attempt, &services, &j.Environment, &j.EnvironmentURL, &interruptible); err != nil {
605 return Job{}, err
606 }
607 j.NeedsSet = needsSet == 1
608 j.AllowFailure = allow == 1
609 j.Interruptible = interruptible == 1
610 j.ArtifactExpire = time.Duration(expire) * time.Second
611 j.Timeout = time.Duration(timeout) * time.Second
612 j.StartedAt = parseTime(started.String)
613 j.FinishedAt = parseTime(finished.String)
614 _ = json.Unmarshal([]byte(needs), &j.Needs)
615 _ = json.Unmarshal([]byte(vars), &j.Variables)
616 _ = json.Unmarshal([]byte(paths), &j.ArtifactPaths)
617 _ = json.Unmarshal([]byte(tags), &j.Tags)
618 _ = json.Unmarshal([]byte(cache), &j.CachePaths)
619 _ = json.Unmarshal([]byte(optional), &j.Optional)
620 _ = json.Unmarshal([]byte(retryWhen), &j.RetryWhen)
621 _ = json.Unmarshal([]byte(services), &j.Services)
622 return j, nil
623}
624
625// ActivePipelines zijn nog niet af.
626func (s *Store) ActivePipelines(ctx context.Context) ([]Pipeline, error) {
627 rows, err := s.db.QueryContext(ctx, pipelineCols+` WHERE status IN ('queued', 'running', 'manual') OR id IN (SELECT pipeline_id FROM jobs WHERE status IN ('queued', 'running')) ORDER BY id`)
628 if err != nil {
629 return nil, err
630 }
631 defer rows.Close()
632 var out []Pipeline
633 for rows.Next() {
634 p, err := scanPipeline(rows)
635 if err != nil {
636 return nil, err
637 }
638 out = append(out, p)
639 }
640 return out, rows.Err()
641}
642
643// MarkJobRunning zet een wachtende job op running als hij dat nog is.
644func (s *Store) MarkJobRunning(ctx context.Context, id int64) (bool, error) {
645 res, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = 'running', started_at = ? WHERE id = ? AND status = 'queued'`, now(), id)
646 if err != nil {
647 return false, err
648 }
649 n, _ := res.RowsAffected()
650 if n == 1 {
651 _, _ = s.db.ExecContext(ctx, `UPDATE pipelines SET status = 'running', started_at = COALESCE(started_at, ?) WHERE id = (SELECT pipeline_id FROM jobs WHERE id = ?)`, now(), id)
652 }
653 return n == 1, nil
654}
655
656// CountRunningJobs telt jobs die Steve nu uitvoert.
657func (s *Store) CountRunningJobs(ctx context.Context) (int, error) {
658 var n int
659 err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM jobs WHERE status = 'running'`).Scan(&n)
660 return n, err
661}
662
663// FinishJob schrijft het eindresultaat en het log.
664func (s *Store) FinishJob(ctx context.Context, id int64, status, extra string) error {
665 _, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = ?, log = log || ?, finished_at = ? WHERE id = ?`, status, extra, now(), id)
666 return err
667}
668
669// AppendJobLog voegt tekst toe terwijl de job loopt.
670func (s *Store) AppendJobLog(ctx context.Context, id int64, chunk string) error {
671 _, err := s.db.ExecContext(ctx, `UPDATE jobs SET log = log || ? WHERE id = ?`, chunk, id)
672 return err
673}
674
675// RequeueJob zet een mislukte job terug in de wachtrij voor een nieuwe poging.
676func (s *Store) RequeueJob(ctx context.Context, id int64, attempt int, note string) error {
677 _, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = 'queued', attempt = ?, log = log || ?, finished_at = NULL, started_at = NULL WHERE id = ?`, attempt, note, id)
678 return err
679}
680
681// CancelInterruptible annuleert onderbreekbare jobs van oudere pipelines op dezelfde ref.
682// Lopende jobs komen terug zodat de runner hun context kan stoppen.
683func (s *Store) CancelInterruptible(ctx context.Context, repoID int64, ref string, keep int64) (running []int64, pipes []int64, err error) {
684 rows, err := s.db.QueryContext(ctx, `SELECT j.id, j.pipeline_id, j.status FROM jobs j
685 JOIN pipelines p ON p.id = j.pipeline_id
686 WHERE p.repo_id = ? AND p.ref = ? AND p.id != ? AND j.interruptible = 1 AND j.status IN ('queued', 'running', 'manual')`, repoID, ref, keep)
687 if err != nil {
688 return nil, nil, err
689 }
690 defer rows.Close()
691 type row struct {
692 id, pipe int64
693 status string
694 }
695 var found []row
696 seenPipe := map[int64]bool{}
697 for rows.Next() {
698 var item row
699 if err := rows.Scan(&item.id, &item.pipe, &item.status); err != nil {
700 return nil, nil, err
701 }
702 found = append(found, item)
703 if !seenPipe[item.pipe] {
704 seenPipe[item.pipe] = true
705 pipes = append(pipes, item.pipe)
706 }
707 }
708 if err := rows.Err(); err != nil {
709 return nil, nil, err
710 }
711 note := time.Now().UTC().Format(time.RFC3339) + "\tcanceled because a newer pipeline started for the same ref\n"
712 for _, item := range found {
713 if item.status == "running" {
714 running = append(running, item.id)
715 continue
716 }
717 if _, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = 'canceled', log = log || ?, finished_at = ? WHERE id = ? AND status IN ('queued', 'manual')`, note, now(), item.id); err != nil {
718 return running, pipes, err
719 }
720 }
721 return running, pipes, nil
722}
723
724// RefreshPipelineStatus berekent de pipelinestatus uit de jobs.
725func (s *Store) RefreshPipelineStatus(ctx context.Context, pipelineID int64) (string, error) {
726 jobs, err := s.JobsByPipeline(ctx, pipelineID)
727 if err != nil {
728 return "", err
729 }
730 status := "success"
731 open := false
732 manual := false
733 canceled := false
734 for _, j := range jobs {
735 switch j.Status {
736 case "failed":
737 if !j.AllowFailure {
738 status = "failed"
739 }
740 case "canceled":
741 if !j.AllowFailure {
742 canceled = true
743 }
744 case "queued", "running":
745 open = true
746 case "manual":
747 manual = true
748 }
749 }
750 switch {
751 case open:
752 // Een harde mislukking wint pas als er niets meer in de wachtrij of
753 // aan het werk is. Anders blijft de runner latere jobs overslaan.
754 status = "running"
755 _, err = s.db.ExecContext(ctx, `UPDATE pipelines SET status = 'running', finished_at = NULL WHERE id = ?`, pipelineID)
756 case status == "failed":
757 _, err = s.db.ExecContext(ctx, `UPDATE pipelines SET status = 'failed', finished_at = ? WHERE id = ?`, now(), pipelineID)
758 case canceled:
759 status = "canceled"
760 _, err = s.db.ExecContext(ctx, `UPDATE pipelines SET status = 'canceled', finished_at = ? WHERE id = ?`, now(), pipelineID)
761 case manual:
762 status = "manual"
763 _, err = s.db.ExecContext(ctx, `UPDATE pipelines SET status = 'manual', finished_at = NULL WHERE id = ?`, pipelineID)
764 default:
765 _, err = s.db.ExecContext(ctx, `UPDATE pipelines SET status = 'success', finished_at = ? WHERE id = ?`, now(), pipelineID)
766 }
767 return status, err
768}
769
770// ResourceBusy meldt of een andere job in deze repository die groep bezet.
771func (s *Store) ResourceBusy(ctx context.Context, repoID int64, group string, except int64) (bool, error) {
772 if group == "" {
773 return false, nil
774 }
775 var n int
776 err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM jobs WHERE repo_id = ? AND resource_group = ? AND status = 'running' AND id != ?`, repoID, group, except).Scan(&n)
777 return n > 0, err
778}
779
780// RetryJob zet een mislukte of geannuleerde job terug in de wachtrij.
781func (s *Store) RetryJob(ctx context.Context, id int64) (bool, error) {
782 res, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = 'queued', finished_at = NULL WHERE id = ? AND status IN ('failed', 'canceled')`, id)
783 if err != nil {
784 return false, err
785 }
786 n, _ := res.RowsAffected()
787 if n == 1 {
788 _, _ = s.db.ExecContext(ctx, `UPDATE pipelines SET status = 'running', finished_at = NULL WHERE id = (SELECT pipeline_id FROM jobs WHERE id = ?)`, id)
789 }
790 return n == 1, nil
791}
792
793// CancelJob stopt een job die nog niet klaar is.
794func (s *Store) CancelJob(ctx context.Context, id int64) (bool, error) {
795 res, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = 'canceled', finished_at = ? WHERE id = ? AND status IN ('created', 'pending', 'queued', 'manual', 'running')`, now(), id)
796 if err != nil {
797 return false, err
798 }
799 n, _ := res.RowsAffected()
800 return n == 1, nil
801}
802
803// QueueJob zet een handmatige job in de wachtrij.
804func (s *Store) QueueJob(ctx context.Context, id int64) (bool, error) {
805 res, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = 'queued' WHERE id = ? AND status = 'manual'`, id)
806 if err != nil {
807 return false, err
808 }
809 n, _ := res.RowsAffected()
810 if n == 1 {
811 _, _ = s.db.ExecContext(ctx, `UPDATE pipelines SET status = 'running', finished_at = NULL WHERE id = (SELECT pipeline_id FROM jobs WHERE id = ?)`, id)
812 }
813 return n == 1, nil
814}
815
816// SkipJob markeert een job die niet meer hoeft.
817func (s *Store) SkipJob(ctx context.Context, id int64, reason string) error {
818 _, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = 'skipped', log = ?, finished_at = ? WHERE id = ? AND status IN ('queued', 'manual')`, reason, now(), id)
819 return err
820}
821
822// CreateJobToken maakt het token waarmee een lopende job de API aanroept.
823func (s *Store) CreateJobToken(ctx context.Context, jobID int64) (string, error) {
824 plain, err := newToken("bgj_")
825 if err != nil {
826 return "", err
827 }
828 _, err = s.db.ExecContext(ctx, `INSERT INTO job_tokens (job_id, token_hash, created_at) VALUES (?, ?, ?)
829 ON CONFLICT(job_id) DO UPDATE SET token_hash = excluded.token_hash, created_at = excluded.created_at`,
830 jobID, hashToken(plain), now())
831 if err != nil {
832 return "", err
833 }
834 return plain, nil
835}
836
837// JobByToken zoekt een lopende job bij een job-token.
838func (s *Store) JobByToken(ctx context.Context, token string) (Job, error) {
839 if len(token) < 4 || token[:4] != "bgj_" {
840 return Job{}, errNotFound
841 }
842 var jobID int64
843 err := s.db.QueryRowContext(ctx, `SELECT job_id FROM job_tokens WHERE token_hash = ?`, hashToken(token)).Scan(&jobID)
844 if errors.Is(err, sql.ErrNoRows) {
845 return Job{}, errNotFound
846 }
847 if err != nil {
848 return Job{}, err
849 }
850 rows, err := s.db.QueryContext(ctx, jobSelect+` WHERE j.id = ?`, jobID)
851 if err != nil {
852 return Job{}, err
853 }
854 defer rows.Close()
855 if !rows.Next() {
856 return Job{}, errNotFound
857 }
858 return scanJob(rows)
859}
860
861// JobByID leest een job van deze repository.
862func (s *Store) JobByID(ctx context.Context, repoID, id int64) (Job, error) {
863 rows, err := s.db.QueryContext(ctx, jobSelect+` WHERE j.repo_id = ? AND j.id = ?`, repoID, id)
864 if err != nil {
865 return Job{}, err
866 }
867 defer rows.Close()
868 if !rows.Next() {
869 return Job{}, errNotFound
870 }
871 return scanJob(rows)
872}
873
874// Artifact verwijst naar een bestand uit een job.
875type Artifact struct {
876 ID int64
877 JobID int64
878 Path string
879 StoragePath string
880 Size int64
881}
882
883// AddArtifact registreert een bewaard bestand.
884func (s *Store) AddArtifact(ctx context.Context, jobID int64, relPath, storagePath string, size int64, expires *time.Time) error {
885 var exp any
886 if expires != nil {
887 exp = expires.UTC().Format(time.RFC3339)
888 }
889 _, err := s.db.ExecContext(ctx, `INSERT INTO artifacts (job_id, path, storage_path, size, expires_at) VALUES (?, ?, ?, ?, ?)`,
890 jobID, relPath, storagePath, size, exp)
891 return err
892}
893
894// ListArtifacts toont de bestanden van één job.
895func (s *Store) ListArtifacts(ctx context.Context, jobID int64) ([]Artifact, error) {
896 rows, err := s.db.QueryContext(ctx, `SELECT id, job_id, path, storage_path, size FROM artifacts WHERE job_id = ? ORDER BY path`, jobID)
897 if err != nil {
898 return nil, err
899 }
900 defer rows.Close()
901 var out []Artifact
902 for rows.Next() {
903 var a Artifact
904 if err := rows.Scan(&a.ID, &a.JobID, &a.Path, &a.StoragePath, &a.Size); err != nil {
905 return nil, err
906 }
907 out = append(out, a)
908 }
909 return out, rows.Err()
910}
911
912// ArtifactPath zoekt een bestand van een job in dezelfde pipeline.
913func (s *Store) ArtifactPath(ctx context.Context, pipelineID int64, jobName, relPath string) (string, error) {
914 var storage string
915 err := s.db.QueryRowContext(ctx, `SELECT a.storage_path FROM artifacts a JOIN jobs j ON j.id = a.job_id
916 WHERE j.pipeline_id = ? AND j.name = ? AND a.path = ? AND j.status = 'success'`, pipelineID, jobName, relPath).Scan(&storage)
917 if errors.Is(err, sql.ErrNoRows) {
918 return "", errNotFound
919 }
920 return storage, err
921}
922
923// Release is een publicatie op een commit.
924type Release struct {
925 ID int64
926 RepoID int64
927 Tag string
928 CommitSHA string
929 Name string
930 Body string
931 CreatedAt time.Time
932 Assets []ReleaseAsset
933}
934
935// ReleaseAsset is een bestand bij een release.
936type ReleaseAsset struct {
937 Name string
938 StoragePath string
939 Size int64
940}
941
942// CreateRelease bewaart de release. De bestanden staan al op storagePath.
943func (s *Store) CreateRelease(ctx context.Context, repoID int64, tag, sha, name, body string, authorID, jobID int64, assets []ReleaseAsset) (int64, error) {
944 if err := names.ValidTag(tag); err != nil {
945 return 0, err
946 }
947 tx, err := s.db.BeginTx(ctx, nil)
948 if err != nil {
949 return 0, err
950 }
951 defer func() { _ = tx.Rollback() }()
952 var author, job any
953 if authorID != 0 {
954 author = authorID
955 }
956 if jobID != 0 {
957 job = jobID
958 }
959 res, err := tx.ExecContext(ctx, `INSERT INTO releases (repo_id, tag, commit_sha, name, body, author_id, job_id, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
960 repoID, tag, sha, name, body, author, job, now())
961 if err != nil {
962 if containsUnique(err) {
963 return 0, i18n.E("err.tagExists", tag)
964 }
965 return 0, err
966 }
967 id, _ := res.LastInsertId()
968 for _, a := range assets {
969 if _, err := tx.ExecContext(ctx, `INSERT INTO release_assets (release_id, name, storage_path, size) VALUES (?, ?, ?, ?)`, id, a.Name, a.StoragePath, a.Size); err != nil {
970 return 0, err
971 }
972 }
973 if err := tx.Commit(); err != nil {
974 return 0, err
975 }
976 return id, nil
977}
978
979// ListReleases geeft publicaties, nieuwste eerst.
980func (s *Store) ListReleases(ctx context.Context, repoID int64) ([]Release, error) {
981 rows, err := s.db.QueryContext(ctx, `SELECT id, repo_id, tag, commit_sha, name, body, created_at FROM releases WHERE repo_id = ? ORDER BY id DESC`, repoID)
982 if err != nil {
983 return nil, err
984 }
985 defer rows.Close()
986 var out []Release
987 for rows.Next() {
988 var r Release
989 var created string
990 if err := rows.Scan(&r.ID, &r.RepoID, &r.Tag, &r.CommitSHA, &r.Name, &r.Body, &created); err != nil {
991 return nil, err
992 }
993 r.CreatedAt = parseTime(created)
994 out = append(out, r)
995 }
996 if err := rows.Err(); err != nil {
997 return nil, err
998 }
999 for i := range out {
1000 assets, err := s.releaseAssets(ctx, out[i].ID)
1001 if err != nil {
1002 return nil, err
1003 }
1004 out[i].Assets = assets
1005 }
1006 return out, nil
1007}
1008
1009// ReleaseByTag leest één release.
1010func (s *Store) ReleaseByTag(ctx context.Context, repoID int64, tag string) (Release, error) {
1011 var r Release
1012 var created string
1013 err := s.db.QueryRowContext(ctx, `SELECT id, repo_id, tag, commit_sha, name, body, created_at FROM releases WHERE repo_id = ? AND tag = ?`, repoID, tag).
1014 Scan(&r.ID, &r.RepoID, &r.Tag, &r.CommitSHA, &r.Name, &r.Body, &created)
1015 if errors.Is(err, sql.ErrNoRows) {
1016 return Release{}, errNotFound
1017 }
1018 if err != nil {
1019 return Release{}, err
1020 }
1021 r.CreatedAt = parseTime(created)
1022 r.Assets, err = s.releaseAssets(ctx, r.ID)
1023 return r, err
1024}
1025
1026func (s *Store) releaseAssets(ctx context.Context, id int64) ([]ReleaseAsset, error) {
1027 rows, err := s.db.QueryContext(ctx, `SELECT name, storage_path, size FROM release_assets WHERE release_id = ? ORDER BY name`, id)
1028 if err != nil {
1029 return nil, err
1030 }
1031 defer rows.Close()
1032 var out []ReleaseAsset
1033 for rows.Next() {
1034 var a ReleaseAsset
1035 if err := rows.Scan(&a.Name, &a.StoragePath, &a.Size); err != nil {
1036 return nil, err
1037 }
1038 out = append(out, a)
1039 }
1040 return out, rows.Err()
1041}
1042
1043func trim(s string, n int) string {
1044 s = string([]rune(s))
1045 if len([]rune(s)) > n {
1046 return string([]rune(s)[:n])
1047 }
1048 return s
1049}
1050
1051func containsUnique(err error) bool {
1052 return err != nil && (errors.Is(err, err) && (len(err.Error()) > 0 && (contains(err.Error(), "UNIQUE") || contains(err.Error(), "unique"))))
1053}
1054
1055func contains(s, sub string) bool {
1056 return len(s) >= len(sub) && (s == sub || len(sub) == 0 || (len(s) > 0 && (indexOf(s, sub) >= 0)))
1057}
1058
1059func indexOf(s, sub string) int {
1060 for i := 0; i+len(sub) <= len(s); i++ {
1061 if s[i:i+len(sub)] == sub {
1062 return i
1063 }
1064 }
1065 return -1
1066}