BlueGit

timfa bluegit

Files

bluegit / internal / runner / runner.go

runner.go

fix: sla latere stages over na een harde mislukking
timfa project_87923614_bot_3aff574f6f98414b945a8799374f9092 · pushed by timfa · 3 weeks ago
ceb7aa19 History
runner.go 16.5 KiB Download Raw
1// Package runner voert pipelinejobs uit, elk in een eigen Docker-container.
2package runner
3
4import (
5 "bytes"
6 "context"
7 "fmt"
8 "io"
9 "log/slog"
10 "os"
11 "os/exec"
12 "path/filepath"
13 "strings"
14 "sync"
15 "time"
16
17 "gitlab.com/Timfa/bluegit/internal/access"
18 "gitlab.com/Timfa/bluegit/internal/ci"
19 "gitlab.com/Timfa/bluegit/internal/gitrepo"
20 "gitlab.com/Timfa/bluegit/internal/store"
21 "gitlab.com/Timfa/bluegit/internal/webhook"
22)
23
24// Runner plant jobs en stuurt het resultaat door naar webhooks.
25type Runner struct {
26 Store *store.Store
27 GitBin string
28 Docker string
29 Storage string
30 BaseURL string
31 Network string
32 Timeout time.Duration
33 Log *slog.Logger
34 OnEvent func(repoID int64, ev webhook.Event)
35 slots chan struct{}
36 mu sync.RWMutex
37}
38
39// SetStorage wijzigt het pad waarop jobs hun clone en artifacts zetten.
40func (r *Runner) SetStorage(path string) {
41 r.mu.Lock()
42 r.Storage = path
43 r.mu.Unlock()
44}
45
46func (r *Runner) storage() string {
47 r.mu.RLock()
48 defer r.mu.RUnlock()
49 return r.Storage
50}
51
52// Loop blijft jobs oppakken tot ctx afloopt.
53func (r *Runner) Loop(ctx context.Context, concurrency int) {
54 if concurrency < 1 {
55 concurrency = 1
56 }
57 r.slots = make(chan struct{}, concurrency)
58 tick := time.NewTicker(2 * time.Second)
59 defer tick.Stop()
60 for {
61 select {
62 case <-ctx.Done():
63 return
64 case <-tick.C:
65 r.Tick(ctx)
66 }
67 }
68}
69
70// Tick bekijkt alle open pipelines één keer.
71// Zonder opslagpad gebeurt er niets.
72func (r *Runner) Tick(ctx context.Context) {
73 if r.storage() == "" {
74 return
75 }
76 pipes, err := r.Store.ActivePipelines(ctx)
77 if err != nil {
78 r.Log.Error("pipelines", "err", err)
79 return
80 }
81 for _, p := range pipes {
82 r.schedule(ctx, p)
83 }
84}
85
86func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
87 r.settle(ctx, p.ID)
88 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
89 if err != nil {
90 r.Log.Error("jobs", "err", err)
91 return
92 }
93 specs := make([]ci.Job, len(jobs))
94 status := map[string]string{}
95 for i, j := range jobs {
96 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
97 status[j.Name] = j.Status
98 }
99 for _, j := range jobs {
100 if j.Status != "queued" && j.Status != "manual" {
101 continue
102 }
103 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
104 case "skip":
105 _ = r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n")
106 status[j.Name] = "skipped"
107 case "fail":
108 _ = r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n")
109 status[j.Name] = "failed"
110 case "ready":
111 if j.Status != "queued" {
112 continue
113 }
114 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
115 continue
116 }
117 ok, err := r.Store.MarkJobRunning(ctx, j.ID)
118 if err != nil || !ok {
119 continue
120 }
121 status[j.Name] = "running"
122 job := j
123 pipe := p
124 go r.execute(ctx, pipe, job)
125 }
126 }
127}
128
129// settle slaat jobs over die door een harde mislukking niet meer hoeven.
130// Dat gebeurt meteen, ook als de pipelinestatus al op failed was gezet.
131func (r *Runner) settle(ctx context.Context, pipelineID int64) {
132 for {
133 jobs, err := r.Store.JobsByPipeline(ctx, pipelineID)
134 if err != nil {
135 if r.Log != nil {
136 r.Log.Error("jobs", "err", err)
137 }
138 return
139 }
140 specs := make([]ci.Job, len(jobs))
141 status := map[string]string{}
142 for i, j := range jobs {
143 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
144 status[j.Name] = j.Status
145 }
146 moved := false
147 for _, j := range jobs {
148 if j.Status != "queued" && j.Status != "manual" {
149 continue
150 }
151 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
152 case "skip":
153 if err := r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n"); err != nil && r.Log != nil {
154 r.Log.Error("skip", "err", err)
155 continue
156 }
157 status[j.Name] = "skipped"
158 moved = true
159 case "fail":
160 if err := r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n"); err != nil && r.Log != nil {
161 r.Log.Error("fail", "err", err)
162 continue
163 }
164 status[j.Name] = "failed"
165 moved = true
166 }
167 }
168 if !moved {
169 return
170 }
171 }
172}
173
174func indexOf(jobs []store.Job, name string) int {
175 for i, j := range jobs {
176 if j.Name == name {
177 return i
178 }
179 }
180 return 0
181}
182
183func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {
184 if r.slots != nil {
185 r.slots <- struct{}{}
186 defer func() { <-r.slots }()
187 }
188 before, _ := r.Store.PipelineByID(context.Background(), p.RepoID, p.ID)
189 mask, err := r.runOne(ctx, p, job)
190 status := "success"
191 extra := ""
192 if err != nil {
193 status = "failed"
194 extra = stamp(ci.Mask(err.Error(), mask))
195 }
196 if ferr := r.Store.FinishJob(context.Background(), job.ID, status, extra); ferr != nil {
197 r.Log.Error("job afronden", "err", ferr)
198 }
199 r.settle(context.Background(), p.ID)
200 final, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID)
201 if rerr != nil {
202 r.Log.Error("pipelinestatus", "err", rerr)
203 return
204 }
205 if before.Status != final && (final == "success" || final == "failed") && r.OnEvent != nil {
206 kind := webhook.EventPipelineSuccess
207 word := "geslaagd"
208 if final == "failed" {
209 kind = webhook.EventPipelineFailure
210 word = "mislukt"
211 }
212 repo, err := r.Store.RepoByID(context.Background(), p.RepoID)
213 if err != nil {
214 return
215 }
216 branch, _ := gitrepo.BranchOfRef(p.Ref)
217 if branch == "" {
218 branch = strings.TrimPrefix(p.Ref, "refs/heads/")
219 }
220 r.OnEvent(p.RepoID, webhook.Event{
221 Type: kind,
222 Project: repo.FullPath(),
223 Branch: branch,
224 Commit: p.CommitSHA,
225 Summary: fmt.Sprintf("pipeline #%d voor %s is %s", p.ID, repo.FullPath(), word),
226 URL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID),
227 PipelineID: fmt.Sprint(p.ID),
228 })
229 }
230}
231
232func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([]string, error) {
233 limit := r.Timeout
234 if job.Timeout > 0 {
235 limit = job.Timeout
236 }
237 if limit > 0 {
238 var cancel context.CancelFunc
239 ctx, cancel = context.WithTimeout(ctx, limit)
240 defer cancel()
241 }
242 repo, err := r.Store.RepoByID(ctx, p.RepoID)
243 if err != nil {
244 return nil, err
245 }
246 sink := &logSink{store: r.Store, jobID: job.ID}
247 sink.section("prepare", "Preparing the docker executor")
248 work := filepath.Join(r.storage(), "work", fmt.Sprint(p.ID), job.Name)
249 _ = os.RemoveAll(work)
250 if err := os.MkdirAll(work, 0o755); err != nil {
251 sink.closeSection()
252 return nil, err
253 }
254 bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
255 if _, err := execOut(ctx, r.GitBin, "clone", "--local", bare, work); err != nil {
256 sink.closeSection()
257 return nil, fmt.Errorf("clone: %w", err)
258 }
259 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", p.CommitSHA); err != nil {
260 sink.closeSection()
261 return nil, fmt.Errorf("checkout %s: %w", p.CommitSHA, err)
262 }
263 if err := r.restoreCache(repo.ID, job, work); err != nil {
264 sink.closeSection()
265 return nil, err
266 }
267 if err := r.materialize(ctx, p, job, work); err != nil {
268 sink.closeSection()
269 return nil, err
270 }
271 scriptPath := filepath.Join(work, ".bluegit-job.sh")
272 if err := os.WriteFile(scriptPath, []byte(shellScript(job)), 0o755); err != nil {
273 sink.closeSection()
274 return nil, err
275 }
276 token, err := r.Store.CreateJobToken(ctx, job.ID)
277 if err != nil {
278 sink.closeSection()
279 return nil, err
280 }
281 env, mask, err := r.environment(ctx, repo, p, job, token)
282 if err != nil {
283 sink.closeSection()
284 return nil, err
285 }
286 sink.secrets = mask
287 if _, err := exec.LookPath(r.Docker); err != nil {
288 sink.closeSection()
289 return mask, fmt.Errorf("docker (%s) is not available; the job only runs in a container", r.Docker)
290 }
291 sink.closeSection()
292 // -e NAAM zonder waarde laat Docker de variabele uit de omgeving van
293 // deze client overnemen. Zo blijft een meerregelige sleutel heel en
294 // staat de waarde niet in de proceslijst.
295 args := []string{"run", "--rm", "--network", r.Network, "-v", work + ":/workspace", "-w", "/workspace"}
296 cmdEnv := append([]string{}, os.Environ()...)
297 for k, v := range env {
298 args = append(args, "-e", k)
299 cmdEnv = append(cmdEnv, k+"="+v)
300 }
301 args = append(args, "--entrypoint", "sh", job.Image, "/workspace/.bluegit-job.sh")
302 cmd := exec.CommandContext(ctx, r.Docker, args...)
303 cmd.Env = cmdEnv
304 cmd.Stdout = sink
305 cmd.Stderr = sink
306 err = cmd.Run()
307 sink.flush()
308 if err != nil {
309 return mask, fmt.Errorf("container ended with an error: %w", err)
310 }
311 if err := r.collect(ctx, job, work); err != nil {
312 return mask, err
313 }
314 if err := r.saveCache(repo.ID, job, work); err != nil {
315 return mask, err
316 }
317 return mask, nil
318}
319
320func shellScript(job store.Job) string {
321 var b strings.Builder
322 b.WriteString("#!/bin/sh\ncd /workspace\nrc=0\n")
323 fmt.Fprintf(&b, "echo \"section_start:before:Preparing environment\"\n")
324 if strings.TrimSpace(job.Before) != "" {
325 b.WriteString("(\nset -e\n")
326 b.WriteString(job.Before)
327 if !strings.HasSuffix(job.Before, "\n") {
328 b.WriteByte('\n')
329 }
330 b.WriteString(")\nbrc=$?\n")
331 } else {
332 b.WriteString("brc=0\n")
333 }
334 b.WriteString("echo \"section_end:before:Preparing environment\"\n")
335 b.WriteString("echo \"section_start:script:Running script\"\n")
336 b.WriteString("if [ \"$brc\" -eq 0 ]; then\n(\nset -e\n")
337 b.WriteString(job.Script)
338 if !strings.HasSuffix(job.Script, "\n") {
339 b.WriteByte('\n')
340 }
341 b.WriteString(")\nrc=$?\nelse\nrc=$brc\nfi\n")
342 b.WriteString("echo \"section_end:script:Running script\"\n")
343 b.WriteString("echo \"section_start:after:After script\"\n")
344 if strings.TrimSpace(job.After) != "" {
345 b.WriteString("(\nset -e\n")
346 b.WriteString(job.After)
347 if !strings.HasSuffix(job.After, "\n") {
348 b.WriteByte('\n')
349 }
350 b.WriteString(")\n")
351 }
352 b.WriteString("echo \"section_end:after:After script\"\nexit \"$rc\"\n")
353 return b.String()
354}
355
356func (r *Runner) materialize(ctx context.Context, p store.Pipeline, job store.Job, work string) error {
357 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
358 if err != nil {
359 return err
360 }
361 wanted := map[string]bool{}
362 if job.NeedsSet {
363 for _, n := range job.Needs {
364 wanted[n] = true
365 }
366 } else {
367 stageOrder := map[string]int{}
368 n := 0
369 for _, j := range jobs {
370 if _, ok := stageOrder[j.Stage]; !ok {
371 stageOrder[j.Stage] = n
372 n++
373 }
374 }
375 my := stageOrder[job.Stage]
376 for _, j := range jobs {
377 if stageOrder[j.Stage] < my {
378 wanted[j.Name] = true
379 }
380 }
381 }
382 for _, j := range jobs {
383 if !wanted[j.Name] || j.Status != "success" {
384 continue
385 }
386 for _, rel := range j.ArtifactPaths {
387 src, err := r.Store.ArtifactPath(ctx, p.ID, j.Name, rel)
388 if err != nil {
389 continue
390 }
391 dst := filepath.Join(work, filepath.FromSlash(rel))
392 if err := copyFile(src, dst); err != nil {
393 return err
394 }
395 }
396 }
397 return nil
398}
399
400func (r *Runner) collect(ctx context.Context, job store.Job, work string) error {
401 var expires *time.Time
402 if job.ArtifactExpire > 0 {
403 t := time.Now().Add(job.ArtifactExpire)
404 expires = &t
405 }
406 for _, rel := range job.ArtifactPaths {
407 src := filepath.Join(work, filepath.FromSlash(rel))
408 info, err := os.Stat(src)
409 if err != nil || info.IsDir() {
410 return fmt.Errorf("artifact %s ontbreekt", rel)
411 }
412 dst := filepath.Join(r.storage(), "artifacts", fmt.Sprint(job.ID), filepath.FromSlash(rel))
413 if err := copyFile(src, dst); err != nil {
414 return err
415 }
416 if err := r.Store.AddArtifact(ctx, job.ID, rel, dst, info.Size(), expires); err != nil {
417 return err
418 }
419 }
420 return nil
421}
422
423func (r *Runner) environment(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, token string) (map[string]string, []string, error) {
424 name, _ := ci.SplitRef(p.Ref)
425 patterns, err := r.Store.ProtectedPatterns(ctx, repo.ID)
426 if err != nil {
427 return nil, nil, err
428 }
429 refProtected := access.BranchProtected(patterns, name)
430 env := map[string]string{}
431 var mask []string
432 if repo.OwnerKind == "group" {
433 groupSecrets, err := r.Store.SecretValues(ctx, "group", repo.OwnerID, refProtected)
434 if err != nil {
435 return nil, nil, err
436 }
437 for k, v := range groupSecrets {
438 env[k] = v
439 mask = append(mask, v)
440 }
441 }
442 repoSecrets, err := r.Store.SecretValues(ctx, "repo", repo.ID, refProtected)
443 if err != nil {
444 return nil, nil, err
445 }
446 for k, v := range repoSecrets {
447 env[k] = v
448 mask = append(mask, v)
449 }
450 for k, v := range job.Variables {
451 env[k] = v
452 }
453 base := strings.TrimRight(r.BaseURL, "/")
454 pre := ci.Variables(ci.Ref{
455 SHA: p.CommitSHA, FullRef: p.Ref, Title: p.CommitTitle,
456 DefaultBranch: repo.DefaultBranch, Source: p.Source,
457 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
458 APIURL: base + "/api/v1", PipelineID: p.ID,
459 })
460 for k, v := range pre {
461 env[k] = v
462 }
463 env["CI_JOB_ID"] = fmt.Sprint(job.ID)
464 env["CI_JOB_NAME"] = job.Name
465 env["CI_JOB_STAGE"] = job.Stage
466 env["CI_JOB_URL"] = base + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID) + "/jobs/" + fmt.Sprint(job.ID)
467 env["CI_JOB_TOKEN"] = token
468 mask = append(mask, token)
469 return env, mask, nil
470}
471
472func (r *Runner) restoreCache(repoID int64, job store.Job, work string) error {
473 if len(job.CachePaths) == 0 {
474 return nil
475 }
476 root := r.cacheDir(repoID, job.CacheKey)
477 for _, rel := range job.CachePaths {
478 src := filepath.Join(root, filepath.FromSlash(rel))
479 if _, err := os.Stat(src); err != nil {
480 continue
481 }
482 if err := copyFile(src, filepath.Join(work, filepath.FromSlash(rel))); err != nil {
483 return err
484 }
485 }
486 return nil
487}
488
489func (r *Runner) saveCache(repoID int64, job store.Job, work string) error {
490 if len(job.CachePaths) == 0 {
491 return nil
492 }
493 root := r.cacheDir(repoID, job.CacheKey)
494 for _, rel := range job.CachePaths {
495 src := filepath.Join(work, filepath.FromSlash(rel))
496 info, err := os.Stat(src)
497 if err != nil || info.IsDir() {
498 continue
499 }
500 if err := copyFile(src, filepath.Join(root, filepath.FromSlash(rel))); err != nil {
501 return err
502 }
503 }
504 return nil
505}
506
507func (r *Runner) cacheDir(repoID int64, key string) string {
508 if key == "" {
509 key = "default"
510 }
511 var b strings.Builder
512 for _, r := range key {
513 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
514 b.WriteRune(r)
515 }
516 }
517 safe := b.String()
518 if safe == "" {
519 safe = "default"
520 }
521 return filepath.Join(r.storage(), "cache", fmt.Sprint(repoID), safe)
522}
523
524func stamp(line string) string {
525 return time.Now().UTC().Format(time.RFC3339) + "\t" + line + "\n"
526}
527
528type logSink struct {
529 mu sync.Mutex
530 store *store.Store
531 jobID int64
532 secrets []string
533 partial string
534 current string
535 opened time.Time
536}
537
538func (l *logSink) Write(p []byte) (int, error) {
539 l.mu.Lock()
540 defer l.mu.Unlock()
541 l.partial += string(p)
542 for {
543 i := strings.IndexByte(l.partial, '\n')
544 if i < 0 {
545 break
546 }
547 l.emit(strings.TrimRight(l.partial[:i], "\r"))
548 l.partial = l.partial[i+1:]
549 }
550 return len(p), nil
551}
552
553func (l *logSink) flush() {
554 l.mu.Lock()
555 defer l.mu.Unlock()
556 if strings.TrimSpace(l.partial) != "" {
557 l.emit(l.partial)
558 }
559 l.partial = ""
560}
561
562func (l *logSink) emit(line string) {
563 line = ci.Mask(line, l.secrets)
564 if strings.HasPrefix(line, "section_start:") || strings.HasPrefix(line, "section_end:") {
565 if l.current != "" && strings.HasPrefix(line, "section_end:") {
566 dur := int(time.Since(l.opened).Seconds())
567 line = line + "\t" + fmt.Sprintf("%02d:%02d", dur/60, dur%60)
568 l.current = ""
569 }
570 if strings.HasPrefix(line, "section_start:") {
571 l.current = line
572 l.opened = time.Now()
573 }
574 }
575 _ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))
576}
577
578func (l *logSink) section(name, title string) {
579 l.emit("section_start:" + name + ":" + title)
580}
581
582func (l *logSink) closeSection() {
583 if l.current == "" {
584 return
585 }
586 name := "prepare"
587 if parts := strings.SplitN(strings.TrimPrefix(l.current, "section_start:"), ":", 2); len(parts) > 0 && parts[0] != "" {
588 name = parts[0]
589 }
590 l.emit("section_end:" + name + ":done")
591}
592
593func copyFile(src, dst string) error {
594 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
595 return err
596 }
597 in, err := os.Open(src)
598 if err != nil {
599 return err
600 }
601 defer in.Close()
602 out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
603 if err != nil {
604 return err
605 }
606 defer out.Close()
607 _, err = io.Copy(out, in)
608 return err
609}
610
611func execOut(ctx context.Context, bin string, args ...string) (string, error) {
612 if bin == "" {
613 bin = "git"
614 }
615 cmd := exec.CommandContext(ctx, bin, args...)
616 var stdout, stderr bytes.Buffer
617 cmd.Stdout = &stdout
618 cmd.Stderr = &stderr
619 if err := cmd.Run(); err != nil {
620 msg := strings.TrimSpace(stderr.String())
621 if msg == "" {
622 msg = err.Error()
623 }
624 return "", fmt.Errorf("%s", msg)
625 }
626 return stdout.String(), nil
627}