BlueGit

timfa bluegit

Files

bluegit / internal / runner / runner.go

runner.go

runner.go 35.8 KiB Download Raw
1// Package runner voert pipelinejobs uit, elk in een eigen Docker-container.
2package runner
3
4import (
5 "bytes"
6 "context"
7 "errors"
8 "fmt"
9 "io"
10 "io/fs"
11 "log/slog"
12 "os"
13 "os/exec"
14 "path/filepath"
15 "strconv"
16 "strings"
17 "sync"
18 "time"
19
20 "gitlab.com/Timfa/bluegit/internal/access"
21 "gitlab.com/Timfa/bluegit/internal/ci"
22 "gitlab.com/Timfa/bluegit/internal/gitrepo"
23 "gitlab.com/Timfa/bluegit/internal/images"
24 "gitlab.com/Timfa/bluegit/internal/runnercfg"
25 "gitlab.com/Timfa/bluegit/internal/store"
26 "gitlab.com/Timfa/bluegit/internal/webhook"
27)
28
29// Runner plant jobs en stuurt het resultaat door naar webhooks.
30type Runner struct {
31 // Hold houdt een pipeline vast, bijvoorbeeld tijdens een self-update.
32 Hold func(store.Pipeline) bool
33 Store *store.Store
34 GitBin string
35 Docker string
36 Storage string
37 BaseURL string
38 Network string
39 Name string
40 Timeout time.Duration
41 Log *slog.Logger
42 OnEvent func(repoID int64, ev webhook.Event)
43 // Presets zijn de image-aliassen uit ci.presets. Nil is de ingebouwde lijst.
44 Presets map[string]images.Override
45 // Jobs is het overzicht en de pauze (drain) voor de beheer-UI.
46 Jobs *runnercfg.Tracker
47 ctl control
48 slots chan struct{}
49 mu sync.RWMutex
50 cancelMu sync.Mutex
51 cancel map[int64]context.CancelFunc
52}
53
54// Interrupt stopt lopende jobs waarvan een nieuwere pipeline de beurt overneemt.
55func (r *Runner) Interrupt(ids []int64) {
56 r.cancelMu.Lock()
57 defer r.cancelMu.Unlock()
58 for _, id := range ids {
59 if c := r.cancel[id]; c != nil {
60 c()
61 }
62 }
63}
64
65func (r *Runner) track(id int64, cancel context.CancelFunc) {
66 r.cancelMu.Lock()
67 defer r.cancelMu.Unlock()
68 if r.cancel == nil {
69 r.cancel = map[int64]context.CancelFunc{}
70 }
71 r.cancel[id] = cancel
72}
73
74func (r *Runner) untrack(id int64) {
75 r.cancelMu.Lock()
76 defer r.cancelMu.Unlock()
77 delete(r.cancel, id)
78}
79
80// SetStorage wijzigt het pad waarop jobs hun clone en artifacts zetten.
81func (r *Runner) SetStorage(path string) {
82 r.mu.Lock()
83 r.Storage = path
84 r.mu.Unlock()
85}
86
87func (r *Runner) storage() string {
88 r.mu.RLock()
89 defer r.mu.RUnlock()
90 return r.Storage
91}
92
93// Loop blijft jobs oppakken tot ctx afloopt.
94func (r *Runner) Loop(ctx context.Context, concurrency int) {
95 if concurrency < 1 {
96 concurrency = 1
97 }
98 r.slots = make(chan struct{}, maxSlots)
99 r.mu.Lock()
100 if !r.ctl.set {
101 r.ctl.limit = concurrency
102 }
103 r.mu.Unlock()
104 tick := time.NewTicker(2 * time.Second)
105 defer tick.Stop()
106 for {
107 select {
108 case <-ctx.Done():
109 return
110 case <-tick.C:
111 r.Tick(ctx)
112 }
113 }
114}
115
116// Tick bekijkt alle open pipelines één keer.
117// Zonder opslagpad gebeurt er niets.
118func (r *Runner) Tick(ctx context.Context) {
119 if r.storage() == "" {
120 return
121 }
122 pipes, err := r.Store.ActivePipelines(ctx)
123 if err != nil {
124 r.Log.Error("pipelines", "err", err)
125 return
126 }
127 for _, p := range pipes {
128 r.schedule(ctx, p)
129 }
130 r.reapRemote(ctx)
131}
132
133func (r *Runner) schedule(ctx context.Context, p store.Pipeline) {
134 r.settle(ctx, p.ID)
135 if r.Hold != nil && r.Hold(p) {
136 return // wacht op de self-update
137 }
138 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
139 if err != nil {
140 r.Log.Error("jobs", "err", err)
141 return
142 }
143 specs := make([]ci.Job, len(jobs))
144 status := map[string]string{}
145 for i, j := range jobs {
146 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
147 status[j.Name] = j.Status
148 }
149 tierCache := map[int64][]Tier{}
150 for _, j := range jobs {
151 if j.Status != "queued" && j.Status != "manual" {
152 continue
153 }
154 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
155 case "skip":
156 _ = r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n")
157 status[j.Name] = "skipped"
158 case "fail":
159 _ = r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n")
160 status[j.Name] = "failed"
161 case "ready":
162 if j.Status != "queued" {
163 continue
164 }
165 if busy, _ := r.Store.ResourceBusy(ctx, j.RepoID, j.ResourceGroup, j.ID); busy {
166 continue
167 }
168 // Meer jobs dan ci.concurrency blijven queued, met een plek
169 // in de wachtrij. Het slot gaat pas vrij als execute klaar is.
170 free := r.acquireSlot()
171 prot := r.refProtected(ctx, j.RepoID, p.Ref)
172 if prot {
173 r.noteProtected(ctx, j.RepoID, j)
174 }
175 if tiers := r.tiersFor(ctx, tierCache, j.RepoID, prot); len(tiers) > 1 {
176 jr := r.Store.JobRun(ctx, j.ID)
177 step := Advance(tiers, jr.OfferTier, jr.OfferAt, time.Now(), free)
178 if !step.Builtin {
179 if free {
180 r.releaseSlot()
181 }
182 if step.Moved {
183 _ = r.Store.SetJobOffer(ctx, j.ID, step.Tier, time.Now())
184 }
185 continue
186 }
187 }
188 if !free {
189 continue
190 }
191 ok, err := r.Store.MarkJobRunning(ctx, j.ID)
192 if err != nil || !ok {
193 r.releaseSlot()
194 continue
195 }
196 _ = r.Store.SetJobRunner(ctx, j.ID, 0, r.BuiltinName())
197 _ = r.Store.ClearJobOffer(ctx, j.ID)
198 status[j.Name] = "running"
199 job := j
200 pipe := p
201 go r.execute(ctx, pipe, job)
202 }
203 }
204}
205
206// settle slaat jobs over die door een harde mislukking of een
207// overgeslagen need niet meer hoeven. Dat gebeurt meteen, ook als de
208// pipelinestatus al op failed was gezet.
209func (r *Runner) settle(ctx context.Context, pipelineID int64) {
210 for {
211 jobs, err := r.Store.JobsByPipeline(ctx, pipelineID)
212 if err != nil {
213 if r.Log != nil {
214 r.Log.Error("jobs", "err", err)
215 }
216 return
217 }
218 specs := make([]ci.Job, len(jobs))
219 status := map[string]string{}
220 for i, j := range jobs {
221 specs[i] = ci.Job{Name: j.Name, Stage: j.Stage, Needs: j.Needs, Optional: j.Optional, NeedsSet: j.NeedsSet, AllowFailure: j.AllowFailure, When: j.When}
222 status[j.Name] = j.Status
223 }
224 moved := false
225 for _, j := range jobs {
226 if j.Status != "queued" && j.Status != "manual" {
227 continue
228 }
229 switch ci.Gate(specs, status, specs[indexOf(jobs, j.Name)]) {
230 case "skip":
231 if err := r.Store.SkipJob(ctx, j.ID, "skipped because an earlier job failed\n"); err != nil && r.Log != nil {
232 r.Log.Error("skip", "err", err)
233 continue
234 }
235 status[j.Name] = "skipped"
236 moved = true
237 case "fail":
238 if err := r.Store.FinishJob(ctx, j.ID, "failed", "a required job is not part of this pipeline\n"); err != nil && r.Log != nil {
239 r.Log.Error("fail", "err", err)
240 continue
241 }
242 status[j.Name] = "failed"
243 moved = true
244 }
245 }
246 if !moved {
247 return
248 }
249 }
250}
251
252// BuiltinName is de naam van de ingebouwde runner in de job-UI.
253func (r *Runner) BuiltinName() string {
254 if r.Name != "" {
255 return r.Name
256 }
257 return "BlueGit runner"
258}
259
260func (r *Runner) tiersFor(ctx context.Context, cache map[int64][]Tier, repoID int64, protected bool) []Tier {
261 key := repoID * 2
262 if protected {
263 key++
264 }
265 if t, ok := cache[key]; ok {
266 return t
267 }
268 var tiers []Tier
269 if repo, err := r.Store.RepoByID(ctx, repoID); err == nil {
270 tiers, _ = Tiers(ctx, r.Store, repo, protected)
271 }
272 cache[key] = tiers
273 return tiers
274}
275
276// refProtected: draait een pipeline op een beschermde branch (en krijgt
277// hij dus beschermde geheimen)?
278func (r *Runner) refProtected(ctx context.Context, repoID int64, ref string) bool {
279 name, _ := ci.SplitRef(ref)
280 patterns, err := r.Store.ProtectedPatterns(ctx, repoID)
281 if err != nil {
282 return true // bij twijfel streng
283 }
284 return access.BranchProtected(patterns, name)
285}
286
287var protectedNoted sync.Map // job id → melding al in het log
288
289// noteProtected zet één keer een melding in het joblog als er opt-in
290// runners zijn die deze job niet mogen draaien.
291func (r *Runner) noteProtected(ctx context.Context, repoID int64, job store.Job) {
292 if _, done := protectedNoted.LoadOrStore(job.ID, true); done {
293 return
294 }
295 repo, err := r.Store.RepoByID(ctx, repoID)
296 if err != nil {
297 return
298 }
299 if names := ExcludedForProtected(ctx, r.Store, repo); len(names) > 0 {
300 _ = r.Store.AppendJobLog(ctx, job.ID, "Protected branch: this job only runs on the built-in runner, global runners, group runners and account runners of maintainers. Skipped: "+strings.Join(names, ", ")+"\n")
301 }
302}
303
304func indexOf(jobs []store.Job, name string) int {
305 for i, j := range jobs {
306 if j.Name == name {
307 return i
308 }
309 }
310 return 0
311}
312
313func (r *Runner) acquireSlot() bool {
314 if r.Paused() {
315 return false
316 }
317 if r.slots == nil {
318 return true
319 }
320 if l := r.controls().limit; l > 0 && len(r.slots) >= l {
321 return false
322 }
323 select {
324 case r.slots <- struct{}{}:
325 return true
326 default:
327 return false
328 }
329}
330
331func (r *Runner) releaseSlot() {
332 if r.slots == nil {
333 return
334 }
335 select {
336 case <-r.slots:
337 default:
338 }
339}
340
341func (r *Runner) execute(ctx context.Context, p store.Pipeline, job store.Job) {
342 defer r.releaseSlot()
343 before, _ := r.Store.PipelineByID(context.Background(), p.RepoID, p.ID)
344 base, stop := context.WithCancel(ctx)
345 r.track(job.ID, stop)
346 defer func() {
347 r.untrack(job.ID)
348 stop()
349 }()
350 runCtx := base
351 limit := r.Timeout
352 if job.Timeout > 0 {
353 limit = job.Timeout
354 }
355 if m := r.controls().timeout; m > 0 && (limit == 0 || time.Duration(m)*time.Minute < limit) {
356 limit = time.Duration(m) * time.Minute
357 }
358 r.trackStart(job)
359 if limit > 0 {
360 var cancel context.CancelFunc
361 runCtx, cancel = context.WithTimeout(base, limit)
362 defer cancel()
363 }
364 mask, err := r.runOne(runCtx, p, job)
365 status := "success"
366 extra := ""
367 kind := ""
368 switch {
369 case err == nil:
370 case ctx.Err() == nil && base.Err() != nil:
371 status = "canceled"
372 extra = stamp("canceled because a newer pipeline started for the same ref")
373 case errors.Is(runCtx.Err(), context.DeadlineExceeded):
374 status = "failed"
375 kind = "job_execution_timeout"
376 extra = stamp(ci.Mask(err.Error(), mask))
377 default:
378 status = "failed"
379 kind = classifyFailure(err)
380 extra = stamp(ci.Mask(err.Error(), mask))
381 }
382 r.trackDone(job.ID, status)
383 r.complete(p, job, before, status, extra, kind)
384}
385
386// complete rondt een job af: opnieuw proberen, afronden, pipelinestatus
387// en webhook. Gedeeld door de ingebouwde en de aparte runner.
388func (r *Runner) complete(p store.Pipeline, job store.Job, before store.Pipeline, status, extra, kind string) {
389 if status == "failed" && shouldRetry(job, kind) {
390 note := stamp(fmt.Sprintf("retrying (%d/%d) after %s", job.Attempt+1, job.RetryMax, kind))
391 _ = r.Store.ClearJobOffer(context.Background(), job.ID)
392 if ferr := r.Store.RequeueJob(context.Background(), job.ID, job.Attempt+1, note+extra); ferr != nil && r.Log != nil {
393 r.Log.Error("job opnieuw", "err", ferr)
394 } else if ferr == nil {
395 r.settle(context.Background(), p.ID)
396 if _, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID); rerr != nil && r.Log != nil {
397 r.Log.Error("pipelinestatus", "err", rerr)
398 }
399 return
400 }
401 }
402 if ferr := r.Store.FinishJob(context.Background(), job.ID, status, extra); ferr != nil {
403 r.Log.Error("job afronden", "err", ferr)
404 }
405 r.settle(context.Background(), p.ID)
406 final, rerr := r.Store.RefreshPipelineStatus(context.Background(), p.ID)
407 if rerr != nil {
408 r.Log.Error("pipelinestatus", "err", rerr)
409 return
410 }
411 if before.Status != final && (final == "success" || final == "failed") && r.OnEvent != nil {
412 kind := webhook.EventPipelineSuccess
413 word := "geslaagd"
414 if final == "failed" {
415 kind = webhook.EventPipelineFailure
416 word = "mislukt"
417 }
418 repo, err := r.Store.RepoByID(context.Background(), p.RepoID)
419 if err != nil {
420 return
421 }
422 branch, _ := gitrepo.BranchOfRef(p.Ref)
423 if branch == "" {
424 branch = strings.TrimPrefix(p.Ref, "refs/heads/")
425 }
426 r.OnEvent(p.RepoID, webhook.Event{
427 Type: kind,
428 Project: repo.FullPath(),
429 Branch: branch,
430 Commit: p.CommitSHA,
431 Summary: fmt.Sprintf("pipeline #%d voor %s is %s", p.ID, repo.FullPath(), word),
432 URL: strings.TrimRight(r.BaseURL, "/") + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID),
433 PipelineID: fmt.Sprint(p.ID),
434 })
435 }
436}
437
438func (r *Runner) runOne(ctx context.Context, p store.Pipeline, job store.Job) ([]string, error) {
439 repo, err := r.Store.RepoByID(ctx, p.RepoID)
440 if err != nil {
441 return nil, err
442 }
443 sink := &logSink{store: r.Store, jobID: job.ID}
444 sink.section("prepare", "Preparing the docker executor")
445 work := filepath.Join(r.storage(), "work", fmt.Sprint(p.ID), safeSegment(job.Name))
446 if err := clearWorkDir(work); err != nil {
447 sink.closeSection()
448 return nil, fmt.Errorf("runner_system_failure: work dir: %w", err)
449 }
450 if err := os.MkdirAll(work, 0o755); err != nil {
451 sink.closeSection()
452 return nil, err
453 }
454 bare := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
455 if _, err := execOut(ctx, r.GitBin, cloneArgs(bare, work)...); err != nil {
456 sink.closeSection()
457 return nil, fmt.Errorf("clone: %w", err)
458 }
459 // origin wijst naar de echte repo-URL, niet naar het pad van de bare repo
460 // op de host. In de container kan de job zo met CI_JOB_TOKEN fetchen.
461 if r.BaseURL != "" {
462 _, _ = execOut(ctx, r.GitBin, "-C", work, "remote", "set-url", "origin", strings.TrimRight(r.BaseURL, "/")+"/"+repo.FullPath()+".git")
463 }
464 if _, err := execOut(ctx, r.GitBin, "-C", work, "checkout", "--detach", p.CommitSHA); err != nil {
465 sink.closeSection()
466 return nil, fmt.Errorf("checkout %s: %w", p.CommitSHA, err)
467 }
468 if err := r.restoreCache(repo.ID, job, work); err != nil {
469 sink.closeSection()
470 return nil, err
471 }
472 if err := r.materialize(ctx, p, job, work); err != nil {
473 sink.closeSection()
474 return nil, err
475 }
476 scriptPath := filepath.Join(work, ".bluegit-job.sh")
477 host, _ := os.Hostname()
478 script := shellScript(job, placeNote{
479 Runner: r.Name, Host: host, WorkDir: "/workspace",
480 PipelineID: p.ID, JobID: job.ID,
481 })
482 if err := os.WriteFile(scriptPath, []byte(script), 0o755); err != nil {
483 sink.closeSection()
484 return nil, err
485 }
486 token, err := r.Store.CreateJobToken(ctx, job.ID)
487 if err != nil {
488 sink.closeSection()
489 return nil, err
490 }
491 env, mask, err := r.environment(ctx, repo, p, job, token)
492 if err != nil {
493 sink.closeSection()
494 return nil, err
495 }
496 sink.secrets = mask
497 if err := r.execContainer(ctx, sink, job, work, env); err != nil {
498 return mask, err
499 }
500 if err := r.collect(ctx, job, work); err != nil {
501 return mask, err
502 }
503 if err := r.saveCache(repo.ID, job, work); err != nil {
504 return mask, err
505 }
506 return mask, nil
507}
508
509// execContainer haalt het image op, start services en draait het
510// jobscript in de container. De sectie prepare moet open staan.
511func (r *Runner) execContainer(ctx context.Context, sink *logSink, job store.Job, work string, env map[string]string) error {
512 ref, lines, rerr := r.resolveImage(ctx, job.Image)
513 for _, line := range lines {
514 sink.emit(line)
515 }
516 if rerr != nil {
517 sink.closeSection()
518 return rerr
519 }
520 job.Image = ref
521 job.Services = append([]store.Service(nil), job.Services...)
522 for i := range job.Services {
523 sref, slines, serr := r.resolveImage(ctx, job.Services[i].Name)
524 for _, line := range slines {
525 sink.emit(line)
526 }
527 if serr != nil {
528 sink.closeSection()
529 return serr
530 }
531 job.Services[i].Name = sref
532 }
533 network := r.Network
534 if network == "" {
535 network = "bridge"
536 }
537 if len(job.Services) > 0 {
538 net, cleanup, serr := r.startServices(ctx, job)
539 if serr != nil {
540 sink.closeSection()
541 return serr
542 }
543 defer cleanup()
544 network = net
545 }
546 sink.closeSection()
547 // -e NAAM zonder waarde laat Docker de variabele uit de omgeving van
548 // deze client overnemen. Zo blijft een meerregelige sleutel heel en
549 // staat de waarde niet in de proceslijst.
550 // Een vaste naam per job: een geannuleerde of afgebroken container
551 // (de docker-client stopt dan, de container niet) wordt voor de start en
552 // na afloop opgeruimd.
553 cname := fmt.Sprintf("bluegit-job-%d", job.ID)
554 removeContainer(r.Docker, cname)
555 defer removeContainer(r.Docker, cname)
556 args := []string{"run", "--rm", "--name", cname, "--network", network, "-v", work + ":/workspace", "-w", "/workspace"}
557 ctl := r.controls()
558 args = append(args, ctl.runArgs...)
559 if ctl.pull == runnercfg.PullNever {
560 args = append(args, "--pull", "never")
561 }
562 cmdEnv := append([]string{}, os.Environ()...)
563 for k, v := range env {
564 args = append(args, "-e", k)
565 cmdEnv = append(cmdEnv, k+"="+v)
566 }
567 args = append(args, "--entrypoint", "sh", job.Image, "/workspace/.bluegit-job.sh")
568 cmd := exec.CommandContext(ctx, r.Docker, args...)
569 cmd.Env = cmdEnv
570 cmd.Stdout = sink
571 cmd.Stderr = sink
572 err := cmd.Run()
573 sink.flush()
574 if err != nil {
575 return fmt.Errorf("container ended with an error: %w", err)
576 }
577 return nil
578}
579
580func removeContainer(docker, name string) {
581 ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
582 defer cancel()
583 _, _ = execOut(ctx, docker, "rm", "-f", name)
584}
585
586// clearWorkDir maakt de werkmap leeg. Mappen zonder schrijfrecht (uit een
587// container) worden eerst beschrijfbaar gemaakt. Lukt verwijderen dan nog
588// niet (bestanden van root), dan gaat de oude map opzij, zodat de clone in
589// een lege map begint. Alleen als ook dat faalt, is het een fout.
590func clearWorkDir(work string) error {
591 if err := os.RemoveAll(work); err == nil {
592 return nil
593 }
594 _ = filepath.WalkDir(work, func(p string, d fs.DirEntry, err error) error {
595 if err == nil && d.IsDir() {
596 _ = os.Chmod(p, 0o755)
597 }
598 return nil
599 })
600 if err := os.RemoveAll(work); err == nil {
601 return nil
602 }
603 stale := fmt.Sprintf("%s.stale-%d", work, time.Now().UnixNano())
604 if err := os.Rename(work, stale); err != nil {
605 return err
606 }
607 go func() { _ = os.RemoveAll(stale) }()
608 return nil
609}
610
611// placeNote zijn de vaste regels in de sectie Preparing environment.
612type placeNote struct {
613 Runner string
614 Host string
615 WorkDir string
616 PipelineID int64
617 JobID int64
618}
619
620func (n placeNote) lines() []string {
621 runner := strings.TrimSpace(n.Runner)
622 if runner == "" {
623 runner = "Steve"
624 }
625 host := strings.TrimSpace(n.Host)
626 if host == "" {
627 host = "localhost"
628 }
629 dir := strings.TrimSpace(n.WorkDir)
630 if dir == "" {
631 dir = "/workspace"
632 }
633 return []string{
634 fmt.Sprintf("Running on %s via %s...", runner, host),
635 fmt.Sprintf("Working directory: %s", dir),
636 fmt.Sprintf("Pipeline #%d, job #%d", n.PipelineID, n.JobID),
637 }
638}
639
640func shSingle(s string) string {
641 return "'" + strings.ReplaceAll(s, "'", `'"'"'`) + "'"
642}
643
644func shellScript(job store.Job, note placeNote) string {
645 var b strings.Builder
646 b.WriteString("#!/bin/sh\ncd /workspace\nrc=0\n")
647 fmt.Fprintf(&b, "echo \"section_start:before:Preparing environment\"\n")
648 for _, line := range note.lines() {
649 fmt.Fprintf(&b, "echo %s\n", shSingle(line))
650 }
651 if strings.TrimSpace(job.Before) != "" {
652 b.WriteString("(\nset -e\n")
653 b.WriteString(job.Before)
654 if !strings.HasSuffix(job.Before, "\n") {
655 b.WriteByte('\n')
656 }
657 b.WriteString(")\nbrc=$?\n")
658 } else {
659 b.WriteString("brc=0\n")
660 }
661 b.WriteString("echo \"section_end:before:Preparing environment\"\n")
662 b.WriteString("echo \"section_start:script:Running script\"\n")
663 b.WriteString("if [ \"$brc\" -eq 0 ]; then\n(\nset -e\n")
664 b.WriteString(job.Script)
665 if !strings.HasSuffix(job.Script, "\n") {
666 b.WriteByte('\n')
667 }
668 b.WriteString(")\nrc=$?\nelse\nrc=$brc\nfi\n")
669 b.WriteString("echo \"section_end:script:Running script\"\n")
670 b.WriteString("echo \"section_start:after:After script\"\n")
671 if strings.TrimSpace(job.After) != "" {
672 b.WriteString("(\nset -e\n")
673 b.WriteString(job.After)
674 if !strings.HasSuffix(job.After, "\n") {
675 b.WriteByte('\n')
676 }
677 b.WriteString(")\n")
678 }
679 b.WriteString("echo \"section_end:after:After script\"\nexit \"$rc\"\n")
680 return b.String()
681}
682
683// resolveImage kiest het image waarmee de container start.
684// Een alias zoals bluegit/node wordt gebouwd. node:22 blijft zichzelf.
685func (r *Runner) resolveImage(ctx context.Context, image string) (string, []string, error) {
686 image = strings.TrimSpace(image)
687 set, err := images.Effective(r.presetSet())
688 if err != nil {
689 return image, nil, fmt.Errorf("runner_system_failure: %w", err)
690 }
691 missing := fmt.Errorf("runner_system_failure: docker (%s) is not available; the job only runs in a container", r.Docker)
692 if _, lookErr := exec.LookPath(r.Docker); lookErr != nil {
693 if p, ok := images.Find(set, image); ok {
694 return "", imageLog(p.Name, p.From, p.From, "", nil, nil), missing
695 }
696 return image, dockerImageLines(image, "", nil), missing
697 }
698 p, ok := images.Find(set, image)
699 if !ok {
700 if r.controls().pull == runnercfg.PullAlways {
701 _, _ = execOut(ctx, r.Docker, "pull", "--quiet", image)
702 }
703 id, digests := r.inspectImage(ctx, image)
704 return image, dockerImageLines(image, id, digests), nil
705 }
706 ref, built, err := p.Ensure(ctx, r.Docker)
707 if err != nil {
708 return "", imageLog(p.Name, p.From, "", "", nil, nil), fmt.Errorf("runner_system_failure: preset %s: %w", p.Name, err)
709 }
710 var lines []string
711 if built {
712 lines = append(lines, fmt.Sprintf("Building image preset %s ...", p.Name))
713 }
714 id, digests := r.inspectImage(ctx, ref)
715 var baseDigests []string
716 if ref != p.From {
717 _, baseDigests = r.inspectImage(ctx, p.From)
718 }
719 lines = append(lines, imageLog(p.Name, p.From, ref, id, digests, baseDigests)...)
720 return ref, lines, nil
721}
722
723func imageLog(preset, base, ref, id string, digests, baseDigests []string) []string {
724 var lines []string
725 if preset != "" {
726 lines = append(lines, fmt.Sprintf("Using image preset %s based on %s", preset, base))
727 }
728 if ref == "" {
729 return lines
730 }
731 lines = append(lines, dockerImageLines(ref, id, digests)...)
732 if preset != "" && ref != base {
733 digest := matchingDigest(base, baseDigests)
734 if digest == "" && len(baseDigests) > 0 {
735 digest = strings.TrimSpace(baseDigests[0])
736 }
737 if digest != "" {
738 lines = append(lines, fmt.Sprintf("Underlying image %s with digest %s ...", base, digest))
739 } else {
740 lines = append(lines, fmt.Sprintf("Underlying image %s ...", base))
741 }
742 }
743 return lines
744}
745
746func (r *Runner) inspectImage(ctx context.Context, image string) (string, []string) {
747 if strings.TrimSpace(image) == "" || r.Docker == "" {
748 return "", nil
749 }
750 out, err := execOut(ctx, r.Docker, "image", "inspect", "--format", "{{.Id}}\n{{range .RepoDigests}}{{.}}\n{{end}}", image)
751 if err != nil {
752 return "", nil
753 }
754 return parseImageInspect(out)
755}
756
757func parseImageInspect(out string) (string, []string) {
758 lines := strings.Split(out, "\n")
759 id := ""
760 if len(lines) > 0 {
761 id = strings.TrimSpace(lines[0])
762 }
763 var digests []string
764 for _, line := range lines[1:] {
765 line = strings.TrimSpace(line)
766 if line != "" {
767 digests = append(digests, line)
768 }
769 }
770 return id, digests
771}
772
773func dockerImageLines(image, id string, digests []string) []string {
774 image = strings.TrimSpace(image)
775 lines := []string{fmt.Sprintf("Using docker image %s ...", image)}
776 id = strings.TrimSpace(id)
777 digest := matchingDigest(image, digests)
778 switch {
779 case id != "" && digest != "":
780 lines = append(lines, fmt.Sprintf("Using docker image %s for %s with digest %s ...", id, image, digest))
781 case digest != "":
782 lines = append(lines, fmt.Sprintf("Using digest %s for %s ...", digest, image))
783 case id != "":
784 lines = append(lines, fmt.Sprintf("Using docker image %s for %s ...", id, image))
785 }
786 return lines
787}
788
789func matchingDigest(image string, digests []string) string {
790 name := imageName(image)
791 fallback := ""
792 for _, digest := range digests {
793 digest = strings.TrimSpace(digest)
794 at := strings.LastIndex(digest, "@")
795 if digest == "" || at <= 0 {
796 continue
797 }
798 if fallback == "" {
799 fallback = digest
800 }
801 repo := digest[:at]
802 if repo == name || strings.HasSuffix(repo, "/"+name) || imageName(repo) == name {
803 return digest
804 }
805 }
806 return fallback
807}
808
809func imageName(image string) string {
810 image = strings.TrimSpace(image)
811 if i := strings.LastIndex(image, "@"); i >= 0 {
812 image = image[:i]
813 }
814 slash := strings.LastIndex(image, "/")
815 name := image
816 if slash >= 0 {
817 name = image[slash+1:]
818 }
819 if i := strings.Index(name, ":"); i >= 0 {
820 name = name[:i]
821 }
822 return name
823}
824
825func (r *Runner) materialize(ctx context.Context, p store.Pipeline, job store.Job, work string) error {
826 jobs, err := r.Store.JobsByPipeline(ctx, p.ID)
827 if err != nil {
828 return err
829 }
830 wanted := map[string]bool{}
831 if job.NeedsSet {
832 for _, n := range job.Needs {
833 wanted[n] = true
834 }
835 } else {
836 stageOrder := map[string]int{}
837 n := 0
838 for _, j := range jobs {
839 if _, ok := stageOrder[j.Stage]; !ok {
840 stageOrder[j.Stage] = n
841 n++
842 }
843 }
844 my := stageOrder[job.Stage]
845 for _, j := range jobs {
846 if stageOrder[j.Stage] < my {
847 wanted[j.Name] = true
848 }
849 }
850 }
851 for _, j := range jobs {
852 if !wanted[j.Name] || j.Status != "success" {
853 continue
854 }
855 for _, rel := range j.ArtifactPaths {
856 src, err := r.Store.ArtifactPath(ctx, p.ID, j.Name, rel)
857 if err != nil {
858 continue
859 }
860 dst := filepath.Join(work, filepath.FromSlash(rel))
861 if err := copyFile(src, dst); err != nil {
862 return err
863 }
864 }
865 }
866 return nil
867}
868
869func (r *Runner) collect(ctx context.Context, job store.Job, work string) error {
870 var expires *time.Time
871 if job.ArtifactExpire > 0 {
872 t := time.Now().Add(job.ArtifactExpire)
873 expires = &t
874 }
875 for _, rel := range job.ArtifactPaths {
876 src := filepath.Join(work, filepath.FromSlash(rel))
877 info, err := os.Stat(src)
878 if err != nil || info.IsDir() {
879 return fmt.Errorf("artifact %s ontbreekt", rel)
880 }
881 dst := filepath.Join(r.storage(), "artifacts", fmt.Sprint(job.ID), filepath.FromSlash(rel))
882 if err := copyFile(src, dst); err != nil {
883 return err
884 }
885 if err := r.Store.AddArtifact(ctx, job.ID, rel, dst, info.Size(), expires); err != nil {
886 return err
887 }
888 }
889 return nil
890}
891
892func (r *Runner) environment(ctx context.Context, repo store.Repo, p store.Pipeline, job store.Job, token string) (map[string]string, []string, error) {
893 name, _ := ci.SplitRef(p.Ref)
894 patterns, err := r.Store.ProtectedPatterns(ctx, repo.ID)
895 if err != nil {
896 return nil, nil, err
897 }
898 refProtected := access.BranchProtected(patterns, name)
899 env := map[string]string{}
900 var mask []string
901 if repo.OwnerKind == "group" {
902 groupSecrets, err := r.Store.SecretValues(ctx, "group", repo.OwnerID, refProtected)
903 if err != nil {
904 return nil, nil, err
905 }
906 for k, v := range groupSecrets {
907 env[k] = v
908 mask = append(mask, v)
909 }
910 }
911 repoSecrets, err := r.Store.SecretValues(ctx, "repo", repo.ID, refProtected)
912 if err != nil {
913 return nil, nil, err
914 }
915 for k, v := range repoSecrets {
916 env[k] = v
917 mask = append(mask, v)
918 }
919 for k, v := range job.Variables {
920 env[k] = v
921 }
922 base := strings.TrimRight(r.BaseURL, "/")
923 dir := gitrepo.Dir(r.storage(), repo.OwnerPath, repo.Name)
924 var info *ci.MRInfo
925 if p.MRID != 0 {
926 if m, merr := r.Store.MergeRequest(ctx, repo.ID, p.MRID); merr == nil {
927 target := ""
928 if out, gerr := exec.CommandContext(ctx, r.GitBin, "-C", dir, "rev-parse", "--verify", "refs/heads/"+m.TargetBranch).Output(); gerr == nil {
929 target = strings.TrimSpace(string(out))
930 }
931 info = &ci.MRInfo{
932 ID: m.ID, IID: m.Number, Title: m.Title,
933 SourceBranch: m.SourceBranch, TargetBranch: m.TargetBranch,
934 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
935 TargetSHA: target,
936 }
937 }
938 }
939 stamp, _ := gitrepo.CommitterTimestamp(ctx, r.GitBin, dir, p.CommitSHA)
940 pre := ci.Variables(ci.Ref{
941 SHA: p.CommitSHA, FullRef: p.Ref, Title: p.CommitTitle, Timestamp: stamp,
942 DefaultBranch: repo.DefaultBranch, Source: p.Source,
943 ProjectPath: repo.FullPath(), ProjectURL: base + "/" + repo.FullPath(),
944 APIURL: base + "/api/v1", PipelineID: p.ID, MR: info,
945 })
946 for k, v := range pre {
947 env[k] = v
948 }
949 env["CI_JOB_ID"] = fmt.Sprint(job.ID)
950 env["CI_JOB_NAME"] = job.Name
951 env["CI_JOB_STAGE"] = job.Stage
952 env["CI_JOB_URL"] = base + "/" + repo.FullPath() + "/-/pipelines/" + fmt.Sprint(p.ID) + "/jobs/" + fmt.Sprint(job.ID)
953 env["CI_JOB_TOKEN"] = token
954 if job.Environment != "" {
955 env["CI_ENVIRONMENT_NAME"] = job.Environment
956 if job.EnvironmentURL != "" {
957 env["CI_ENVIRONMENT_URL"] = ci.Expand(job.EnvironmentURL, env)
958 }
959 }
960 for k, v := range job.Variables {
961 if strings.HasPrefix(k, "CI_NODE_") {
962 env[k] = v
963 }
964 }
965 // Na de jobvariabelen, zodat een eigen GIT_CONFIG_COUNT blijft staan
966 // en safe.directory er alsnog bij komt.
967 markWorkspaceSafe(env)
968 JobGitAuth(env, base+"/"+repo.FullPath(), token)
969 mask = append(mask, token)
970 return env, mask, nil
971}
972
973// JobGitAuth geeft git in de job leestoegang tot de eigen repo met het
974// job-token. De header geldt alleen voor de URL van dit project (met en
975// zonder .git) en het token vervalt als de job stopt. Via GIT_CONFIG_* werkt
976// het in elk image, zonder bestanden in de container.
977func JobGitAuth(env map[string]string, projectURL, token string) {
978 if projectURL == "" || token == "" || !strings.Contains(projectURL, "://") {
979 return
980 }
981 n := 0
982 if v, err := strconv.Atoi(strings.TrimSpace(env["GIT_CONFIG_COUNT"])); err == nil && v > 0 {
983 n = v
984 }
985 for _, u := range []string{projectURL, projectURL + ".git"} {
986 env[fmt.Sprintf("GIT_CONFIG_KEY_%d", n)] = "http." + u + ".extraHeader"
987 env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", n)] = "Authorization: Bearer " + token
988 n++
989 }
990 env["GIT_CONFIG_COUNT"] = strconv.Itoa(n)
991}
992
993// markWorkspaceSafe zet safe.directory voor elke job.
994// De checkout is van de host-uid; de container-user is dat niet.
995// Git 2.35+ weigert die repo anders. Een job die safe.directory al op
996// * of /workspace zet, blijft zoals hij is.
997func markWorkspaceSafe(env map[string]string) {
998 const key = "safe.directory"
999 n := 0
1000 if raw, ok := env["GIT_CONFIG_COUNT"]; ok {
1001 if v, err := strconv.Atoi(strings.TrimSpace(raw)); err == nil && v > 0 {
1002 n = v
1003 }
1004 }
1005 for i := 0; i < n; i++ {
1006 if env[fmt.Sprintf("GIT_CONFIG_KEY_%d", i)] != key {
1007 continue
1008 }
1009 switch env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", i)] {
1010 case "*", "/workspace":
1011 return
1012 }
1013 }
1014 env[fmt.Sprintf("GIT_CONFIG_KEY_%d", n)] = key
1015 env[fmt.Sprintf("GIT_CONFIG_VALUE_%d", n)] = "*"
1016 env["GIT_CONFIG_COUNT"] = strconv.Itoa(n + 1)
1017}
1018
1019func (r *Runner) startServices(ctx context.Context, job store.Job) (string, func(), error) {
1020 netName := fmt.Sprintf("bluegit-%d", job.ID)
1021 if _, err := execOut(ctx, r.Docker, "network", "create", netName); err != nil {
1022 return "", func() {}, fmt.Errorf("runner_system_failure: docker network: %w", err)
1023 }
1024 var names []string
1025 cleanup := func() {
1026 cctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
1027 defer cancel()
1028 for _, name := range names {
1029 _, _ = execOut(cctx, r.Docker, "rm", "-f", name)
1030 }
1031 _, _ = execOut(cctx, r.Docker, "network", "rm", netName)
1032 }
1033 for i, svc := range job.Services {
1034 cname := fmt.Sprintf("bluegit-svc-%d-%d", job.ID, i)
1035 args := []string{"run", "-d", "--name", cname, "--network", netName}
1036 if svc.Alias != "" {
1037 args = append(args, "--network-alias", svc.Alias)
1038 }
1039 cmdEnv := append([]string{}, os.Environ()...)
1040 for k, v := range svc.Variables {
1041 args = append(args, "-e", k)
1042 cmdEnv = append(cmdEnv, k+"="+v)
1043 }
1044 if len(svc.Entrypoint) > 0 {
1045 args = append(args, "--entrypoint", svc.Entrypoint[0])
1046 }
1047 args = append(args, svc.Name)
1048 if len(svc.Entrypoint) > 1 {
1049 args = append(args, svc.Entrypoint[1:]...)
1050 }
1051 args = append(args, svc.Command...)
1052 cmd := exec.CommandContext(ctx, r.Docker, args...)
1053 cmd.Env = cmdEnv
1054 var stderr bytes.Buffer
1055 cmd.Stderr = &stderr
1056 if err := cmd.Run(); err != nil {
1057 cleanup()
1058 msg := strings.TrimSpace(stderr.String())
1059 if msg == "" {
1060 msg = err.Error()
1061 }
1062 return "", func() {}, fmt.Errorf("runner_system_failure: service %s: %s", svc.Name, msg)
1063 }
1064 names = append(names, cname)
1065 }
1066 return netName, cleanup, nil
1067}
1068
1069func shouldRetry(job store.Job, kind string) bool {
1070 if job.RetryMax <= 0 || job.Attempt >= job.RetryMax || kind == "" {
1071 return false
1072 }
1073 if len(job.RetryWhen) == 0 || containsStr(job.RetryWhen, "always") {
1074 return true
1075 }
1076 if containsStr(job.RetryWhen, kind) {
1077 return true
1078 }
1079 return kind == "job_execution_timeout" && containsStr(job.RetryWhen, "stuck_or_timeout_failure")
1080}
1081
1082func classifyFailure(err error) string {
1083 msg := err.Error()
1084 if strings.Contains(msg, "runner_system_failure") || strings.Contains(msg, "is not available") {
1085 return "runner_system_failure"
1086 }
1087 if errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "deadline exceeded") {
1088 return "job_execution_timeout"
1089 }
1090 return "script_failure"
1091}
1092
1093func containsStr(list []string, s string) bool {
1094 for _, item := range list {
1095 if item == s {
1096 return true
1097 }
1098 }
1099 return false
1100}
1101
1102func safeSegment(name string) string {
1103 var b strings.Builder
1104 for _, r := range name {
1105 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
1106 b.WriteRune(r)
1107 } else {
1108 b.WriteByte('_')
1109 }
1110 }
1111 if b.Len() == 0 {
1112 return "job"
1113 }
1114 return b.String()
1115}
1116
1117func (r *Runner) restoreCache(repoID int64, job store.Job, work string) error {
1118 if len(job.CachePaths) == 0 {
1119 return nil
1120 }
1121 root := r.cacheDir(repoID, job.CacheKey)
1122 for _, rel := range job.CachePaths {
1123 src := filepath.Join(root, filepath.FromSlash(rel))
1124 if _, err := os.Stat(src); err != nil {
1125 continue
1126 }
1127 if err := copyFile(src, filepath.Join(work, filepath.FromSlash(rel))); err != nil {
1128 return err
1129 }
1130 }
1131 return nil
1132}
1133
1134func (r *Runner) saveCache(repoID int64, job store.Job, work string) error {
1135 if len(job.CachePaths) == 0 {
1136 return nil
1137 }
1138 root := r.cacheDir(repoID, job.CacheKey)
1139 for _, rel := range job.CachePaths {
1140 src := filepath.Join(work, filepath.FromSlash(rel))
1141 info, err := os.Stat(src)
1142 if err != nil || info.IsDir() {
1143 continue
1144 }
1145 if err := copyFile(src, filepath.Join(root, filepath.FromSlash(rel))); err != nil {
1146 return err
1147 }
1148 }
1149 return nil
1150}
1151
1152func (r *Runner) cacheDir(repoID int64, key string) string {
1153 if key == "" {
1154 key = "default"
1155 }
1156 var b strings.Builder
1157 for _, r := range key {
1158 if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '.' || r == '_' || r == '-' {
1159 b.WriteRune(r)
1160 }
1161 }
1162 safe := b.String()
1163 if safe == "" {
1164 safe = "default"
1165 }
1166 return filepath.Join(r.storage(), "cache", fmt.Sprint(repoID), safe)
1167}
1168
1169func stamp(line string) string {
1170 return time.Now().UTC().Format(time.RFC3339) + "\t" + line + "\n"
1171}
1172
1173type logSink struct {
1174 mu sync.Mutex
1175 store *store.Store
1176 jobID int64
1177 out func(string)
1178 secrets []string
1179 partial string
1180 current string
1181 opened time.Time
1182}
1183
1184func (l *logSink) Write(p []byte) (int, error) {
1185 l.mu.Lock()
1186 defer l.mu.Unlock()
1187 l.partial += string(p)
1188 for {
1189 i := strings.IndexByte(l.partial, '\n')
1190 if i < 0 {
1191 break
1192 }
1193 l.emit(strings.TrimRight(l.partial[:i], "\r"))
1194 l.partial = l.partial[i+1:]
1195 }
1196 return len(p), nil
1197}
1198
1199func (l *logSink) flush() {
1200 l.mu.Lock()
1201 defer l.mu.Unlock()
1202 if strings.TrimSpace(l.partial) != "" {
1203 l.emit(l.partial)
1204 }
1205 l.partial = ""
1206}
1207
1208func (l *logSink) emit(line string) {
1209 line = ci.Mask(line, l.secrets)
1210 if strings.HasPrefix(line, "section_start:") || strings.HasPrefix(line, "section_end:") {
1211 if l.current != "" && strings.HasPrefix(line, "section_end:") {
1212 dur := int(time.Since(l.opened).Seconds())
1213 line = line + "\t" + fmt.Sprintf("%02d:%02d", dur/60, dur%60)
1214 l.current = ""
1215 }
1216 if strings.HasPrefix(line, "section_start:") {
1217 l.current = line
1218 l.opened = time.Now()
1219 }
1220 }
1221 if l.out != nil {
1222 l.out(stamp(line))
1223 return
1224 }
1225 _ = l.store.AppendJobLog(context.Background(), l.jobID, stamp(line))
1226}
1227
1228func (l *logSink) section(name, title string) {
1229 l.emit("section_start:" + name + ":" + title)
1230}
1231
1232func (l *logSink) closeSection() {
1233 if l.current == "" {
1234 return
1235 }
1236 name := "prepare"
1237 if parts := strings.SplitN(strings.TrimPrefix(l.current, "section_start:"), ":", 2); len(parts) > 0 && parts[0] != "" {
1238 name = parts[0]
1239 }
1240 l.emit("section_end:" + name + ":done")
1241}
1242
1243func copyFile(src, dst string) error {
1244 if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
1245 return err
1246 }
1247 in, err := os.Open(src)
1248 if err != nil {
1249 return err
1250 }
1251 defer in.Close()
1252 out, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o644)
1253 if err != nil {
1254 return err
1255 }
1256 defer out.Close()
1257 _, err = io.Copy(out, in)
1258 return err
1259}
1260
1261func execOut(ctx context.Context, bin string, args ...string) (string, error) {
1262 if bin == "" {
1263 bin = "git"
1264 }
1265 cmd := exec.CommandContext(ctx, bin, args...)
1266 var stdout, stderr bytes.Buffer
1267 cmd.Stdout = &stdout
1268 cmd.Stderr = &stderr
1269 if err := cmd.Run(); err != nil {
1270 msg := strings.TrimSpace(stderr.String())
1271 if msg == "" {
1272 msg = err.Error()
1273 }
1274 return "", fmt.Errorf("%s", msg)
1275 }
1276 return stdout.String(), nil
1277}
1278
1279// cloneArgs kopieert de objecten in plaats van hardlinks te maken: de
1280// werkmap en de bare repo kunnen andere eigenaars hebben, en met
1281// fs.protected_hardlinks geeft link() dan "Operation not permitted".
1282func cloneArgs(bare, work string) []string {
1283 return []string{"clone", "--no-hardlinks", bare, work}
1284}